dvc: bump to 1.11.1

.codecov.yml

@@ -3,7 +3,7 @@ coverage:
   status:
     project:
       default:
-        target: 90
+        target: 85
         threshold: 10%
     patch:
       default:

.coveragerc

@@ -13,4 +13,3 @@ exclude_lines =
     raise NotImplementedError
     # pragma: no cover
     #  pragma: no cover
-    \.\.\.$

.dvcignore

@@ -1,4 +0,0 @@
-/tests
-/dvc
-/.github
-/bin

.git_archival.txt

@@ -1 +0,0 @@
-ref-names: $Format:%D$

.gitattributes

@@ -1 +0,0 @@
-.git_archival.txt  export-subst

.github/CODEOWNERS

@@ -1 +0,0 @@
-*       @iterative/dvc

.github/ISSUE_TEMPLATE/bug_report.md

@@ -3,62 +3,16 @@ name: "\U0001F41B Bug Report"
 about: Create a bug report to help us improve DVC
 ---
 
-# Bug Report
+## Bug Report
 
-<!--
-## Issue name
+### Please provide information about your setup
 
-Issue names must follow the pattern `command: description` where the command is the dvc command that you are trying to run. The description should describe the consequence of the bug. 
-
-Example: `repro: doesn't detect input changes`
--->
-
-## Description
-
-<!--
-A clear and concise description of what the bug is.
--->
-
-### Reproduce
-
-<!--
-Step list of how to reproduce the bug
--->
-
-<!--
-Example:
-
-1. dvc init
-2. Copy dataset.zip to the directory
-3. dvc add dataset.zip
-4. dvc run -d dataset.zip -o model ./train.sh
-5. modify dataset.zip
-6. dvc repro
--->
-
-### Expected
-
-<!--
-A clear and concise description of what you expect to happen.
--->
-
-### Environment information
-
-<!--
-This is required to ensure that we can reproduce the bug.
--->
-
-**Output of `dvc doctor`:**
+**Output of `dvc version`:**
 
 ```console
-$ dvc doctor
+$ dvc version
 ```
 
 **Additional Information (if any):**
 
-<!--
-Please check https://github.com/iterative/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
-
 If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
-If the issue is regarding the performance, please attach the profiling information and the benchmark comparisons.
--->

.github/dependabot.yml

@@ -1,16 +0,0 @@
-version: 2
-
-updates:
-  - directory: "/"
-    package-ecosystem: "pip"
-    schedule:
-      interval: "daily"
-    labels:
-      - "maintenance"
-
-  - directory: "/"
-    package-ecosystem: "github-actions"
-    schedule:
-      interval: "daily"
-    labels:
-      - "maintenance"

.github/mergify.yml

@@ -1,9 +0,0 @@
-pull_request_rules:
-  - name: backport patches to 1.11
-    conditions:
-      - base=main
-      - label=backport-1.11
-    actions:
-      backport:
-        branches:
-          - "1.11"

.github/workflows/benchmarks.yaml

@@ -1,23 +0,0 @@
-name: benchmarks
-on: [pull_request, workflow_dispatch]
-
-jobs:
-  bench:
-    name: run benchmarks
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-18.04]
-    steps:
-      - name: Cancel Previous Runs
-        uses: styfle/[email protected]
-        with:
-          access_token: ${{ github.token }}
-      - uses: actions/setup-python@v3
-        with:
-            python-version: 3.8
-      - uses: iterative/dvc-bench@main
-        with:
-            pytest_options: "-k 'test_init or test_help'"
-

.github/workflows/packages.yaml

@@ -1,157 +1,136 @@
 name: Build packages
 on:
-  push:
-    branches:
-    - main
+  push: {}
   release:
-    types: [released, prereleased]
-  workflow_dispatch:
+    types: [released]
 jobs:
-  deb:
-    runs-on: ubuntu-18.04
+  linux:
+    runs-on: ubuntu-16.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v2.3.2
+    - name: Set up Python 3.7
+      uses: actions/[email protected]
       with:
-          fetch-depth: 0
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.9
+        python-version: 3.7
     - name: Set up Ruby 2.4
-      uses: actions/setup-ruby@v1.1.3
+      uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.4
-    - name: Install deps
-      run: |
-        pip install .[all]
-        pip install -r scripts/build-requirements.txt
-        gem install --no-document fpm
-    - name: Build deb
-      run: python scripts/build.py deb
+    - name: Build
+      run: ./scripts/build_posix.sh
     - name: Publish deb
       if: github.event_name == 'release'
-      uses: actions/[email protected].2
+      uses: actions/[email protected].1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: scripts/fpm/dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_path: dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_content_type: binary/octet-stream
-  rpm:
-    runs-on: ubuntu-18.04
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.9
-    - name: Set up Ruby 2.4
-      uses: actions/[email protected]
-      with:
-        ruby-version: 2.4
-    - name: Install deps
-      run: |
-        pip install .[all]
-        pip install -r scripts/build-requirements.txt
-        gem install --no-document fpm
-    - name: Build rpm
-      run: python scripts/build.py rpm
     - name: Publish rpm
       if: github.event_name == 'release'
-      uses: actions/[email protected].2
+      uses: actions/[email protected].1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_path: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
         asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
         asset_content_type: binary/octet-stream
-  osxpkg:
+  mac:
     runs-on: macos-10.15
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v2.3.2
+    - name: Set up Python 3.7
+      uses: actions/[email protected]
       with:
-        fetch-depth: 0
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.9
+        python-version: 3.7
     - name: Set up Ruby 2.4
-      uses: actions/setup-ruby@v1.1.3
+      uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.4
-    - name: Install deps
-      run: |
-        # available lxml (webdav dependency) wheels are built with an old
-        # SDK, which breaks osxpkg notarization. Building from scratch
-        # instead.
-        pip install .[all] --no-binary lxml
-        pip install -r scripts/build-requirements.txt
-        gem install --no-document fpm
     - name: Build
-      run: python scripts/build.py osxpkg
+      run: ./scripts/build_posix.sh
     - name: Publish
       if: github.event_name == 'release'
-      uses: actions/[email protected].2
+      uses: actions/[email protected].1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}.pkg
+        asset_path: dvc-${{ github.event.release.tag_name }}.pkg
         asset_name: dvc-${{ github.event.release.tag_name }}.pkg
         asset_content_type: binary/octet-stream
-  exe:
+  windows:
     runs-on: windows-2019
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v2.3.2
+    - name: Set up Python 3.7
+      uses: actions/[email protected]
       with:
-        fetch-depth: 0
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.9
-    - name: Install deps
-      run: |
-        pip install .[all]
-        pip install -r scripts/build-requirements.txt
-    - name: Pull images
-      run: |
-        dvc pull
+        python-version: 3.7
     - name: Build
-      run: python scripts\build.py exe
+      run: scripts\build_windows.cmd
       shell: cmd
     - name: Publish
       if: github.event_name == 'release'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      uses: actions/[email protected].2
+      uses: actions/[email protected].1
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: scripts/innosetup/dvc-${{ github.event.release.tag_name }}.exe
+        asset_path: dvc-${{ github.event.release.tag_name }}.exe
         asset_name: dvc-${{ github.event.release.tag_name }}.exe
         asset_content_type: binary/octet-stream
-  pip:
-    runs-on: ubuntu-18.04
+  python:
+    runs-on: ubuntu-16.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v2.3.2
+    - name: Set up Python 3.7
+      uses: actions/[email protected]
       with:
-        fetch-depth: 0
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
-      with:
-        python-version: 3.8
+        python-version: 3.7
     - name: Install
       run: pip install ".[all,tests]"
     - name: Build packages
-      run: |
-        pip install -U pip
-        ./scripts/build_package.sh
+      run: ./scripts/build_package.sh
     - name: Publish packages to PyPI
       if: github.event_name == 'release'
       uses: pypa/gh-action-pypi-publish@master
       with:
         user: __token__
         password: ${{ secrets.PYPI_TOKEN }}
+  snap:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/[email protected]
+    - name: Build the package
+      uses: snapcore/action-build@v1
+      id: snapcraft
+    - name: Test the package
+      run: |
+        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
+        dvc version
+        sudo snap remove dvc
+    - name: Publish to the edge channel
+      uses: snapcore/action-publish@v1
+      if: github.event_name == 'push'
+      with:
+        store_login: ${{ secrets.STORE_LOGIN }}
+        snap: ${{ steps.snapcraft.outputs.snap }}
+        release: edge
+    - name: Publish to the stable channel
+      uses: snapcore/action-publish@v1
+      if: github.event_name == 'release'
+      with:
+        store_login: ${{ secrets.STORE_LOGIN }}
+        snap: ${{ steps.snapcraft.outputs.snap }}
+        release: stable
+    - name: Publish to the v1 channel
+      uses: snapcore/action-publish@v1
+      if: github.event_name == 'release'
+      with:
+        store_login: ${{ secrets.STORE_LOGIN }}
+        snap: ${{ steps.snapcraft.outputs.snap }}
+        release: v1

.github/workflows/tests.yaml

@@ -1,96 +1,86 @@
 name: Tests
 
 on:
-  pull_request: {}
-  push: {}
-  schedule:
-    - cron: '5 1 * * *'  # every day at 01:05
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
 
 env:
   DVC_TEST: "true"
   HOMEBREW_NO_AUTO_UPDATE: 1
   SHELL: /bin/bash
+  GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
 
 jobs:
   lint:
-    timeout-minutes: 10
     runs-on: ubuntu-latest
     steps:
-    - name: Cancel Previous Runs
-      uses: styfle/[email protected]
-      with:
-        access_token: ${{ github.token }}
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
+    - uses: actions/[email protected]
     - name: Set up Python 3.8
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v2.1.4
       with:
         python-version: 3.8
     - name: Install requirements
-      run: |
-        pip install --upgrade pip setuptools wheel
-        pip install collective.checkdocs==0.2
-        pip install ".[dev]" pre-commit
+      run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
     - name: Check README
       run: python setup.py checkdocs
-    - uses: pre-commit/[email protected]
+    - name: Check formatting
+      run: pre-commit run --all-files
   tests:
-    timeout-minutes: 45
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-latest, macos-latest]
-        pyv: ["3.8", "3.9", "3.10"]
+        os: [ubuntu-18.04, windows-2019, macos-10.15]
+        pyv: ["3.6", "3.7", "3.8", "3.9"]
+        include:
+          - os: "macos-11.0"
+            pyv: "3.9"
+        exclude:
+          - os: macos-10.15
+            pyv: "3.9"
+
     steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
+    - uses: actions/checkout@v2.3.2
     - name: Set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v2.1.4
       with:
         python-version: ${{ matrix.pyv }}
-    - name: get pip cache dir
-      id: pip-cache-dir
-      run: |
-        echo "::set-output name=dir::$(pip cache dir)"
-    - name: set pip cache
-      id: pip-cache
-      uses: actions/cache@v3
-      with:
-        path: ${{ steps.pip-cache-dir.outputs.dir }}
-        key: ${{ runner.os }}-pip-${{ hashFiles('setup.cfg') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
     - name: install
-      if: steps.cache.pip-cache-dir.cache-hit != 'true'
       run: |
-        pip install --upgrade pip setuptools wheel
-        pip install -e ".[dev]"
+        pip install wheel
+        pip install -e ".[all,tests]"
+    - name: setup git
+      run: |
+        git config --global user.email "[email protected]"
+        git config --global user.name "DVC Tester"
+    - name: setup gs creds
+      shell: bash
+      if: env.GS_CREDS_JSON != ''
+      run: |
+        mkdir -p scripts/ci
+        echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
     - name: run tests
-      timeout-minutes: 40
-      run: >-
-        python -m tests -n=auto
-        --cov-report=xml --cov-report=term
-        ${{ env.extra_test_args }}
+      run: python -m tests --all --cov-report=xml --cov-report=term --tap-combined
     - name: upload coverage report
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v1.0.13
       with:
         file: ./coverage.xml
         fail_ci_if_error: false
-  notify:
-    if: github.ref == 'refs/heads/main' && failure()
-    needs:
-      - lint
-      - tests
-    runs-on: ubuntu-latest
-    steps:
-    - name: Slack Notification
-      uses: rtCamp/[email protected]
-      env:
-        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
-        SLACK_COLOR: ${{ job.status }}
-        SLACK_MESSAGE: 'CI Failed on main :boom:'
-        SLACK_TITLE: CI Status
-        SLACK_USERNAME: DVC-CI
+    - name: Post TAP
+      # Running custom fork with an option, that does not fail CI on error
+      uses: skshetry/flaky-service/packages/action@master
+      if: always() && github.ref == 'refs/heads/master'
+      with:
+        file-path: ${{github.workspace}}/testresults.tap
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        os: ${{runner.os}}
+        matrix: ${{toJson(matrix)}}
+        repo-description: ${{github.event.repository.description}}
+        fail_ci_if_error: false

.gitignore

@@ -1,13 +1,10 @@
-__pycache__/
 neatlynx.conf
+neatlynx/__pycache__
 .idea
 /cache
 *.pyc
-
-.env*/
-.venv
-env/
-venv/
+.env/
+.env2.7/
 .python-version
 
 .dvc.conf.lock
@@ -16,7 +13,6 @@ build/
 dist/
 
 *.egg-info/
-.eggs
 
 hooks/*.pyc
 
@@ -29,7 +25,6 @@ innosetup/config.ini
 
 .coverage
 .coverage.*
-coverage.xml
 
 *.sw?
 
@@ -43,10 +38,3 @@ scripts/ci/gcp-creds.json
 .gcp-creds.json
 
 *~
-/dvc/_dvc_version.py
-
-.mypy_cache/
-.pytest_cache/
-.nox/
-.tox/
-htmlcov/

.pre-commit-config.yaml

@@ -1,29 +1,14 @@
-ci:
-  skip: [mypy, pylint, dvc-pre-commit]
-
 repos:
   - hooks:
       - id: black
         language_version: python3
     repo: https://github.com/ambv/black
-    rev: 22.3.0
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.2.0
-    hooks:
-      - id: check-toml
-      - id: check-yaml
-  - hooks:
-      - id: codespell
-        args: 
-          - --ignore-words-list
-          - ba,datas,fo,uptodate,cachable
-    repo: https://github.com/codespell-project/codespell
-    rev: v2.1.0
+    rev: 19.10b0
   - hooks:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 5.10.1
+    rev: 5.6.4
   - hooks:
       - id: flake8
         language_version: python3
@@ -33,15 +18,9 @@ repos:
           - flake8-debugger
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.2
+    rev: 3.8.4
   - repo: local
     hooks:
-    - id: mypy
-      name: mypy
-      entry: mypy
-      files: ^dvc/
-      language: system
-      types: [python]
     - id: pylint
       name: pylint
       entry: pylint
@@ -54,7 +33,7 @@ repos:
         id: beautysh
         language_version: python3
     repo: https://github.com/lovesegfault/beautysh
-    rev: v6.2.1
+    rev: 6.0.1
   - hooks:
       - args:
           - git-hook

.pylintrc

@@ -0,0 +1,607 @@
+[MASTER]
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-whitelist=
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Add files or directories matching the regex patterns to the blacklist. The
+# regex matches against base names, not paths.
+ignore-patterns=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+init-hook="import sys; \
+           from pathlib import Path; \
+           from pylint.config import find_pylintrc; \
+           sys.path.append(Path(find_pylintrc()).parent / 'tests');"
+
+
+# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
+# number of processors available to use.
+jobs=1
+
+# Control the amount of potential inferred values when inferring a single
+# object. This can help the performance when dealing with large functions or
+# complex, nested conditions.
+limit-inference-results=100
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=pylint_pytest,pylint_plugin_disable
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Specify a configuration file.
+#rcfile=
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages.
+suggestion-mode=yes
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
+confidence=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=format,
+        # ^ most of them are covered by flake8,
+        # and are conventions rather than warnings or errors
+        refactoring,  # will be noisy if it starts giving suggestions
+        spelling,
+        # design is too-few-* and too-many-* checks, too-much-* for us right now
+        design,
+        # following were generated by default
+        # most of them are redundant Py3 checks for Py2+3 codebase
+        print-statement,
+        parameter-unpacking,
+        unpacking-in-except,
+        old-raise-syntax,
+        backtick,
+        long-suffix,
+        old-ne-operator,
+        old-octal-literal,
+        import-star-module-level,
+        non-ascii-bytes-literal,
+        raw-checker-failed,
+        bad-inline-option,
+        locally-disabled,
+        file-ignored,
+        suppressed-message,
+        useless-suppression,
+        deprecated-pragma,
+        use-symbolic-message-instead,
+        apply-builtin,
+        basestring-builtin,
+        buffer-builtin,
+        cmp-builtin,
+        coerce-builtin,
+        execfile-builtin,
+        file-builtin,
+        long-builtin,
+        raw_input-builtin,
+        reduce-builtin,
+        standarderror-builtin,
+        unicode-builtin,
+        xrange-builtin,
+        coerce-method,
+        delslice-method,
+        getslice-method,
+        setslice-method,
+        no-absolute-import,
+        old-division,
+        dict-iter-method,
+        dict-view-method,
+        next-method-called,
+        metaclass-assignment,
+        indexing-exception,
+        raising-string,
+        reload-builtin,
+        oct-method,
+        hex-method,
+        nonzero-method,
+        cmp-method,
+        input-builtin,
+        round-builtin,
+        intern-builtin,
+        unichr-builtin,
+        map-builtin-not-iterating,
+        zip-builtin-not-iterating,
+        range-builtin-not-iterating,
+        filter-builtin-not-iterating,
+        using-cmp-argument,
+        eq-without-hash,
+        div-method,
+        idiv-method,
+        rdiv-method,
+        exception-message-attribute,
+        invalid-str-codec,
+        sys-max-int,
+        bad-python3-import,
+        deprecated-string-function,
+        deprecated-str-translate-call,
+        deprecated-itertools-function,
+        deprecated-types-field,
+        next-method-defined,
+        dict-items-not-iterating,
+        dict-keys-not-iterating,
+        dict-values-not-iterating,
+        deprecated-operator-function,
+        deprecated-urllib-function,
+        xreadlines-attribute,
+        deprecated-sys-function,
+        exception-escape,
+        comprehension-escape,
+        # custom disables start here
+        no-self-use,
+        invalid-name,
+        # already a check enabled for `wildcard-import`
+        unused-wildcard-import,
+        misplaced-comparison-constant,
+        # probably soon: https://github.com/iterative/dvc/issues/1843
+        logging-format-interpolation,
+        logging-fstring-interpolation,
+        # we don't have proper docstrings yet
+        # even if we do, we'll be doing it gradually
+        missing-function-docstring,
+        missing-module-docstring,
+        missing-class-docstring,
+        duplicate-code,
+        # too noisy, and detects issues even when import is inside a function
+        # see: https://github.com/PyCQA/pylint/issues/850
+        cyclic-import,
+        # isort should cover these:
+        wrong-import-order,
+        wrong-import-position,
+        ungrouped-imports,
+        multiple-imports,
+        # we use it to speedup/optimize
+        import-outside-toplevel,
+        fixme,
+
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=c-extension-no-member
+
+
+[REPORTS]
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details.
+#msg-template=
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio). You can also give a reporter class, e.g.
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Tells whether to display a full report or only the messages.
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+# Complete name of functions that never returns. When checking for
+# inconsistent-return-statements if a never returning function is called then
+# it will be considered as an explicit return statement and no message will be
+# printed.
+never-returning-functions=sys.exit
+
+
+[LOGGING]
+
+# Format style used to check logging format string. `old` means using %
+# formatting, while `new` is for `{}` formatting.
+logging-format-style=old
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format.
+logging-modules=logging
+
+
+[SPELLING]
+
+# Limits count of emitted suggestions for spelling mistakes.
+max-spelling-suggestions=4
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package..
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=hdfs.*,pytest.lazy_fixture,logging.TRACE,logger.trace,sys.getwindowsversion
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# Tells whether to warn about missing members when the owner of the attribute
+# is inferred to be None.
+ignore-none=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local,Dvcfile
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis. It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid defining new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,
+          _cb
+
+# A regular expression matching the name of dummy variables (i.e. expected to
+# not be used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore.
+ignored-argument-names=_.*|^ignored_|^unused_|args|kwargs
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Maximum number of characters on a single line.
+max-line-length=79
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+# List of optional constructs for which whitespace checking is disabled. `dict-
+# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
+# `trailing-comma` allows a space between comma and closing bracket: (a, ).
+# `empty-line` allows space-only lines.
+no-space-check=trailing-comma,
+               dict-separator
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[BASIC]
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Regular expression matching correct argument names. Overrides argument-
+# naming-style.
+#argument-rgx=
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Regular expression matching correct attribute names. Overrides attr-naming-
+# style.
+#attr-rgx=
+
+# Bad variable names which should always be refused, separated by a comma.
+bad-names=foo,
+          bar,
+          baz,
+          toto,
+          tutu,
+          tata
+
+# Naming style matching correct class attribute names.
+class-attribute-naming-style=any
+
+# Regular expression matching correct class attribute names. Overrides class-
+# attribute-naming-style.
+#class-attribute-rgx=
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Regular expression matching correct class names. Overrides class-naming-
+# style.
+#class-rgx=
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Regular expression matching correct constant names. Overrides const-naming-
+# style.
+#const-rgx=
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct function names. Overrides function-
+# naming-style.
+#function-rgx=
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+           j,
+           k,
+           ex,
+           Run,
+           _,
+           s3,
+           gs,
+           x, y,
+           x0, y0,
+           x1, y1,
+           dx, dy
+
+# Include a hint for the correct naming format with invalid-name.
+include-naming-hint=no
+
+# Naming style matching correct inline iteration names.
+inlinevar-naming-style=any
+
+# Regular expression matching correct inline iteration names. Overrides
+# inlinevar-naming-style.
+#inlinevar-rgx=
+
+# Naming style matching correct method names.
+method-naming-style=snake_case
+
+# Regular expression matching correct method names. Overrides method-naming-
+# style.
+#method-rgx=
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Regular expression matching correct module names. Overrides module-naming-
+# style.
+#module-rgx=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+# These decorators are taken in consideration only for invalid-name.
+property-classes=abc.abstractproperty
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
+
+# Regular expression matching correct variable names. Overrides variable-
+# naming-style.
+#variable-rgx=
+
+
+[SIMILARITIES]
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,
+      XXX,
+      TODO
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method.
+max-args=5
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of boolean expressions in an if statement.
+max-bool-expr=5
+
+# Maximum number of branch for function / method body.
+max-branches=12
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body.
+max-returns=6
+
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+                      __new__,
+                      setUp
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,
+                  _fields,
+                  _replace,
+                  _source,
+                  _make
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=cls
+
+
+[IMPORTS]
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Deprecated modules which should not be used, separated by a comma.
+deprecated-modules=optparse,tkinter.tix
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled).
+ext-import-graph=
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled).
+import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled).
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception".
+overgeneral-exceptions=Exception

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2017-2021 Iterative, Inc.
+   Copyright {yyyy} {name of copyright owner}
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

MANIFEST.in

@@ -1,3 +1,2 @@
+include fastentrypoints.py
 include LICENSE
-recursive-include requirements *.txt
-recursive-include dvc/repo/plots/templates *.json

README.rst

@@ -8,40 +8,30 @@
 • `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
-|CI| |Maintainability| |Coverage| |Donate| |DOI|
+|Release| |CI| |Maintainability| |Coverage| |Donate| |DOI|
 
 |PyPI| |Packages| |Brew| |Conda| |Choco| |Snap|
 
 |
 
-**Data Version Control** or **DVC** helps you develop reproducible machine
-learning projects:
+**Data Version Control** or **DVC** is an **open-source** tool for data science and machine
+learning projects. Key features:
 
-#. Version your data and models.
+#. Simple **command line** Git-like experience. Does not require installing and maintaining
+   any databases. Does not depend on any proprietary online services.
 
-#. Connect them to your code and parameters to form a pipeline.
+#. Management and versioning of **datasets** and **machine learning models**. Data is saved in
+   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
 
-#. Modify data, code, or parameters and run your modified pipeline.
+#. Makes projects **reproducible** and **shareable**; helping to answer questions about how
+   a model was built.
 
-#. Iterate and track each experiment.
+#. Helps manage experiments with Git tags/branches and **metrics** tracking.
 
-#. Share, compare, and reconstruct experiments.
-
-**Why DVC?**
-
-#. **Store** data and models (big) in your cloud storage, but keep their version
-   info (small) in your Git repo.
-
-#. **Iterate** fast with straightforward YAML pipelines. When you make changes,
-   only run the steps impacted by those changes.
-
-#. **Track** experiments in your local Git repo (no servers needed) and choose
-   which to keep and share.
-
-#. **Compare** any data, code, parameters, metrics, or plots and analyze however
-   you want.
-
-#. **Reconstruct** anyone's experiment with a single command.
+**DVC** aims to replace spreadsheet and document sharing tools (such as Excel or Google Docs)
+which are being used frequently as both knowledge repositories and team ledgers.
+DVC also replaces both ad-hoc scripts to track, move, and deploy different model versions;
+as well as ad-hoc data file suffixes and prefixes.
 
 .. contents:: **Contents**
   :backlinks: none
@@ -75,25 +65,25 @@ Quick start
 
 Please read `Get Started <https://dvc.org/doc/get-started>`_ guide for a full version. Common workflow commands include:
 
-+-----------------------------------+----------------------------------------------------------------------------+
-| Step                              | Command                                                                    |
-+===================================+============================================================================+
-| Track data                        | | ``$ git add train.py``                                                   |
-|                                   | | ``$ dvc add images.zip``                                                 |
-+-----------------------------------+----------------------------------------------------------------------------+
-| Connect code and data by commands | | ``$ dvc run -n prepare -d images.zip -o images/ unzip -q images.zip``    |
-|                                   | | ``$ dvc run -n train -d images/ -d train.py -o model.p python train.py`` |
-+-----------------------------------+----------------------------------------------------------------------------+
-| Make changes and reproduce        | | ``$ vi train.py``                                                        |
-|                                   | | ``$ dvc repro model.p.dvc``                                              |
-+-----------------------------------+----------------------------------------------------------------------------+
-| Share code                        | | ``$ git add .``                                                          |
-|                                   | | ``$ git commit -m 'The baseline model'``                                 |
-|                                   | | ``$ git push``                                                           |
-+-----------------------------------+----------------------------------------------------------------------------+
-| Share data and ML models          | | ``$ dvc remote add myremote -d s3://mybucket/image_cnn``                 |
-|                                   | | ``$ dvc push``                                                           |
-+-----------------------------------+----------------------------------------------------------------------------+
++-----------------------------------+-------------------------------------------------------------------+
+| Step                              | Command                                                           |
++===================================+===================================================================+
+| Track data                        | | ``$ git add train.py``                                          |
+|                                   | | ``$ dvc add images.zip``                                        |
++-----------------------------------+-------------------------------------------------------------------+
+| Connect code and data by commands | | ``$ dvc run -d images.zip -o images/ unzip -q images.zip``      |
+|                                   | | ``$ dvc run -d images/ -d train.py -o model.p python train.py`` |
++-----------------------------------+-------------------------------------------------------------------+
+| Make changes and reproduce        | | ``$ vi train.py``                                               |
+|                                   | | ``$ dvc repro model.p.dvc``                                     |
++-----------------------------------+-------------------------------------------------------------------+
+| Share code                        | | ``$ git add .``                                                 |
+|                                   | | ``$ git commit -m 'The baseline model'``                        |
+|                                   | | ``$ git push``                                                  |
++-----------------------------------+-------------------------------------------------------------------+
+| Share data and ML models          | | ``$ dvc remote add myremote -d s3://mybucket/image_cnn``        |
+|                                   | | ``$ dvc push``                                                  |
++-----------------------------------+-------------------------------------------------------------------+
 
 Installation
 ============
@@ -112,7 +102,7 @@ Snap (Snapcraft/Linux)
 
 This corresponds to the latest tagged release.
 Add ``--beta`` for the latest tagged release candidate,
-or ``--edge`` for the latest ``main`` version.
+or ``--edge`` for the latest ``master`` version.
 
 Choco (Chocolatey/Windows)
 --------------------------
@@ -139,11 +129,7 @@ Conda (Anaconda)
 
 .. code-block:: bash
 
-   conda install -c conda-forge mamba # installs much faster than conda
-   mamba install -c conda-forge dvc
-
-Depending on the remote storage type you plan to use to keep and share your data, you might need to
-install optional dependencies: `dvc-s3`, `dvc-azure`, `dvc-gdrive`, `dvc-gs`, `dvc-oss`, `dvc-ssh`.
+   conda install -c conda-forge dvc
 
 pip (PyPI)
 ----------
@@ -178,54 +164,34 @@ Ubuntu / Debian (deb)
 .. code-block:: bash
 
    sudo wget https://dvc.org/deb/dvc.list -O /etc/apt/sources.list.d/dvc.list
-   wget -qO - https://dvc.org/deb/iterative.asc | sudo apt-key add -
-   sudo apt update
-   sudo apt install dvc
+   sudo apt-get update
+   sudo apt-get install dvc
 
 Fedora / CentOS (rpm)
 ^^^^^^^^^^^^^^^^^^^^^
 .. code-block:: bash
 
    sudo wget https://dvc.org/rpm/dvc.repo -O /etc/yum.repos.d/dvc.repo
-   sudo rpm --import https://dvc.org/rpm/iterative.asc
    sudo yum update
    sudo yum install dvc
 
 Comparison to related technologies
 ==================================
 
-#. Data Engineering tools such as `AirFlow <https://airflow.apache.org/>`_,
-   `Luigi <https://github.com/spotify/luigi>`_, and others - in DVC data,
-   model and ML pipelines represent a single ML project focused on data
-   scientists' experience.  Data engineering tools orchestrate multiple data
-   projects and focus on efficient execution. A DVC project can be used from
-   existing data pipelines as a single execution step.
-
 #. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (which should
    not be in a Git repository) in a local key-value store, and uses file hardlinks/symlinks instead of
    copying/duplicating files.
 
-#. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with many
-   remote storage services (S3, Google Cloud, Azure, SSH, etc). DVC also
-   uses reflinks or hardlinks to avoid copy operations on checkouts; thus
-   handling large data files much more efficiently.
+#. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with any remote storage (S3, Google Cloud, Azure, SSH,
+   etc). DVC also uses reflinks or hardlinks to avoid copy operations on checkouts; thus handling large data files
+   much more efficiently.
 
-#. Makefile (and analogues including ad-hoc scripts) - DVC tracks
-   dependencies (in a directed acyclic graph).
+#. *Makefile* (and analogues including ad-hoc scripts) - DVC tracks dependencies (in a directed acyclic graph).
 
 #. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_ - DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
-#. `DAGsHub <https://dagshub.com/>`_ - online service to host DVC
-   projects.  It provides a useful UI around DVC repositories and integrates
-   other tools.
-
-#. `DVC Studio <https://studio.iterative.ai/>`_ - official online
-   platform for DVC projects.  It can be used to manage data and models, run
-   and track experiments, and visualize and share results.  Also, it
-   integrates with `CML (CI/CD for ML) <https://cml.dev/>`__ for training
-   models in the cloud or Kubernetes.
-
+#. `DAGsHub <https://dagshub.com/>`_ - This is a Github equivalent for DVC. Pushing Git+DVC based repositories to DAGsHub will produce in a high level project dashboard; including DVC pipelines and metrics visualizations, as well as links to any DVC-managed files present in cloud storage.
 
 Contributing
 ============
@@ -233,9 +199,39 @@ Contributing
 |Maintainability| |Donate|
 
 Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
-details. Thanks to all our contributors!
+details.
 
-|Contribs|
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
+   :alt: 0
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
+   :alt: 1
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
+   :alt: 2
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
+   :alt: 3
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
+   :alt: 4
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
+   :alt: 5
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
+   :alt: 6
+
+.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
+   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
+   :alt: 7
 
 Mailing List
 ============
@@ -258,14 +254,15 @@ Citation
 Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
 `DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
 
-Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and Source Code - Empirical Study of DVC Projects <https://mcis.cs.queensu.ca/publications/2021/saner.pdf>`_ , in Proceedings of the 28th IEEE International Conference on Software Analysis, Evolution, and Reengineering, SANER 2021. Hawaii, USA.
-
-
 .. |Banner| image:: https://dvc.org/img/logo-github-readme.png
    :target: https://dvc.org
    :alt: DVC logo
 
-.. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=main
+.. |Release| image:: https://img.shields.io/badge/release-ok-brightgreen
+   :target: https://travis-ci.com/iterative/dvc/branches
+   :alt: Release
+
+.. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=master
    :target: https://github.com/iterative/dvc/actions
    :alt: GHA Tests
 
@@ -273,7 +270,7 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
    :target: https://codeclimate.com/github/iterative/dvc
    :alt: Code Climate
 
-.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/main/graph/badge.svg
+.. |Coverage| image:: https://codecov.io/gh/iterative/dvc/branch/master/graph/badge.svg
    :target: https://codecov.io/gh/iterative/dvc
    :alt: Codecov
 
@@ -312,7 +309,3 @@ Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and S
 .. |Flowchart| image:: https://dvc.org/img/flow.gif
    :target: https://dvc.org/img/flow.gif
    :alt: how_dvc_works
-
-.. |Contribs| image:: https://contrib.rocks/image?repo=iterative/dvc
-   :target: https://github.com/iterative/dvc/graphs/contributors
-   :alt: Contributors

debian/changelog

@@ -0,0 +1,5 @@
+dvc (0.75.b1) experimental; urgency=low
+
+  * First native Debian package version.
+
+ -- Dmitry Petrov <[email protected]>  Sat, 07 Dec 2019 09:49:28 +0300

debian/compat

@@ -0,0 +1 @@
+10

debian/control

@@ -0,0 +1,33 @@
+Source: dvc
+Maintainer: Dmitry Petrov <[email protected]>
+Section: python
+Priority: optional
+Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)
+Standards-Version: 4.0.0
+Homepage: http://dvc.org
+Vcs-Git: https://github.com/iterative/dvc.git
+Vcs-browser: https://github.com/iterative/dvc
+
+Package: dvc
+Architecture: all
+Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2)
+Recommends: python3-arrow (=0.14.0),
+            python3-paramiko (>=2.5.0),
+            python3-gssapi (>= 2.5.0),
+            python3-boto3 (>=1.9.86),
+            python3-pydrive (=1.3.1),
+            python3-azure-storage-blob (=12.1.0),
+            python3-oss2 (=2.6.1),
+            python3-google-cloud-storage (=1.19.0)
+Description: Git for data scientists - manage your code and data together
+ Data Version Control or DVC is an open-source tool for data science and machine learning projects.
+ .
+ Key features:
+ .
+ 1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services.
+ .
+ 2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
+ .
+ 3. Makes projects reproducible and shareable; helping to answer questions about how a model was built.
+ .
+ 4. Helps manage experiments with Git tags/branches and metrics tracking.

debian/copyright

@@ -0,0 +1,11 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: DVC
+Upstream-Contact: Dmitry Petrov <[email protected]>
+Source: https://github.com/iterative/dvc
+
+Files: *
+Copyright: Dmitry Petrov <[email protected]>
+License: Apache 2.0
+ On Debian systems, the full text of the Apache 2.0
+ License can be found in the file
+ `/usr/share/common-licenses/Apache-2.0`.

debian/rules

@@ -0,0 +1,6 @@
+#!/usr/bin/make -f
+
+export PYBUILD_NAME=dvc
+%:
+	dh $@ --with python3 --buildsystem=pybuild
+

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

debian/watch

@@ -0,0 +1,4 @@
+# please also check http://pypi.debian.net/dvc/watch
+version=3
+opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
+http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))

dvc/__main__.py

@@ -1,7 +1,7 @@
 """Main entry point for DVC command line tool."""
 import sys
 
-from dvc.cli import main
+from dvc.main import main
 
 if __name__ == "__main__":
     sys.exit(main(sys.argv[1:]))

dvc/_debug.py

@@ -1,162 +0,0 @@
-from contextlib import ExitStack, contextmanager
-from datetime import datetime
-from typing import TYPE_CHECKING, Callable, Union
-
-if TYPE_CHECKING:
-    from argparse import Namespace
-
-
-@contextmanager
-def viztracer_profile(
-    path: Union[Callable[[], str], str],
-    max_stack_depth: int = -1,
-):
-    try:
-        import viztracer  # pylint: disable=import-error
-    except ImportError:
-        print("Failed to run profiler, viztracer is not installed")
-        yield
-        return
-
-    tracer = viztracer.VizTracer(max_stack_depth=max_stack_depth)
-
-    tracer.start()
-    yield
-    tracer.stop()
-
-    tracer.save(path() if callable(path) else path)
-
-
-@contextmanager
-def yappi_profile(
-    path: Union[Callable[[], str], str] = None, wall_clock: bool = True
-):
-    try:
-        import yappi  # pylint: disable=import-error
-    except ImportError:
-        print("Failed to run profiler, yappi is not installed")
-        yield
-        return
-
-    yappi.set_clock_type("wall" if wall_clock else "cpu")
-
-    yappi.start()
-    yield
-    yappi.stop()
-
-    # pylint:disable=no-member
-    if path:
-        stats = yappi.get_func_stats()
-        fpath = path() if callable(path) else path
-        stats.save(fpath, "callgrind")
-    else:
-        yappi.get_func_stats().print_all()
-        yappi.get_thread_stats().print_all()
-
-    yappi.clear_stats()
-
-
-@contextmanager
-def instrument(html_output=False):
-    """Run a statistical profiler"""
-    try:
-        from pyinstrument import Profiler  # pylint: disable=import-error
-    except ImportError:
-        print("Failed to run profiler, pyinstrument is not installed")
-        yield
-        return
-
-    profiler = Profiler()
-
-    profiler.start()
-    yield
-    profiler.stop()
-
-    if html_output:
-        profiler.open_in_browser()
-        return
-    print(profiler.output_text(unicode=True, color=True))
-
-
-@contextmanager
-def profile(dump_path: str = None):
-    """Run a cprofile"""
-    import cProfile
-
-    prof = cProfile.Profile()
-    prof.enable()
-
-    yield
-
-    prof.disable()
-    if not dump_path:
-        prof.print_stats(sort="cumtime")
-        return
-    prof.dump_stats(dump_path)
-
-
-@contextmanager
-def debug():
-    try:
-        yield
-    except Exception:  # pylint: disable=broad-except
-        try:
-            import ipdb as pdb  # noqa: T100, pylint: disable=import-error
-        except ImportError:
-            import pdb  # noqa: T100
-        pdb.post_mortem()
-
-        raise  # prevent from jumping ahead
-
-
-@contextmanager
-def debugtools(args: "Namespace" = None, **kwargs):
-    kw = vars(args) if args else {}
-    kw.update(kwargs)
-
-    with ExitStack() as stack:
-        if kw.get("pdb"):
-            stack.enter_context(debug())
-        if kw.get("cprofile") or kw.get("cprofile_dump"):
-            stack.enter_context(profile(kw.get("cprofile_dump")))
-        if kw.get("instrument") or kw.get("instrument_open"):
-            stack.enter_context(instrument(kw.get("instrument_open", False)))
-        if kw.get("yappi"):
-            output = "callgrind.dvc-{0:%Y%m%d}_{0:%H%M%S}.out"
-            stack.enter_context(
-                yappi_profile(path=lambda: output.format(datetime.now()))
-            )
-        if kw.get("viztracer") or kw.get("viztracer_depth"):
-            output = "viztracer.dvc-{0:%Y%m%d}_{0:%H%M%S}.json"
-            stack.enter_context(
-                viztracer_profile(
-                    path=lambda: output.format(datetime.now()),
-                    max_stack_depth=kw.get("viztracer_depth") or -1,
-                )
-            )
-        yield
-
-
-def add_debugging_flags(parser):
-    from argparse import SUPPRESS
-
-    parser.add_argument(
-        "--cprofile", action="store_true", default=False, help=SUPPRESS
-    )
-    parser.add_argument(
-        "--yappi", action="store_true", default=False, help=SUPPRESS
-    )
-    parser.add_argument(
-        "--viztracer", action="store_true", default=False, help=SUPPRESS
-    )
-    parser.add_argument("--viztracer-depth", type=int, help=SUPPRESS)
-    parser.add_argument("--cprofile-dump", help=SUPPRESS)
-    parser.add_argument(
-        "--pdb", action="store_true", default=False, help=SUPPRESS
-    )
-    parser.add_argument(
-        "--instrument", action="store_true", default=False, help=SUPPRESS
-    )
-    parser.add_argument(
-        "--instrument-open", action="store_true", default=False, help=SUPPRESS
-    )

dvc/analytics.py

@@ -1,8 +1,24 @@
 import json
 import logging
 import os
+import platform
+import sys
+import tempfile
+import uuid
 
-from .env import DVC_NO_ANALYTICS
+import distro
+import requests
+
+from dvc import __version__
+from dvc.config import Config, to_bool
+from dvc.daemon import daemon
+from dvc.exceptions import NotDvcRepoError
+from dvc.lock import Lock, LockError
+from dvc.repo import Repo
+from dvc.scm import SCM, NoSCM
+from dvc.scm.base import SCMError
+from dvc.utils import env2bool, is_binary
+from dvc.utils.fs import makedirs
 
 logger = logging.getLogger(__name__)
 
@@ -19,10 +35,6 @@ def collect_and_send_report(args=None, return_code=None):
     report as a JSON, where the _collector_ generates it and the _sender_
     removes it after sending it.
     """
-    import tempfile
-
-    from dvc.daemon import daemon
-
     report = {}
 
     # Include command execution information on the report only when available.
@@ -34,22 +46,16 @@ def collect_and_send_report(args=None, return_code=None):
 
     with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
         json.dump(report, fobj)
-    daemon(["analytics", fobj.name])
+        daemon(["analytics", fobj.name])
 
 
 def is_enabled():
-    from dvc.config import Config, to_bool
-    from dvc.utils import env2bool
-
     if env2bool("DVC_TEST"):
         return False
 
-    enabled = not os.getenv(DVC_NO_ANALYTICS)
-    if enabled:
-        enabled = to_bool(
-            Config(validate=False).get("core", {}).get("analytics", "true")
-        )
-
+    enabled = to_bool(
+        Config(validate=False).get("core", {}).get("analytics", "true")
+    )
     logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
 
     return enabled
@@ -63,12 +69,10 @@ def send(path):
     `collect_and_send_report`. Sending happens on another process,
     thus, the need of removing such file afterwards.
     """
-    import requests
-
     url = "https://analytics.dvc.org"
     headers = {"content-type": "application/json"}
 
-    with open(path, encoding="utf-8") as fobj:
+    with open(path) as fobj:
         report = json.load(fobj)
 
     report.update(_runtime_info())
@@ -82,13 +86,6 @@ def send(path):
 
 
 def _scm_in_use():
-    from scmrepo.noscm import NoSCM
-
-    from dvc.exceptions import NotDvcRepoError
-    from dvc.repo import Repo
-
-    from .scm import SCM, SCMError
-
     try:
         scm = SCM(root_dir=Repo.find_root())
         return type(scm).__name__
@@ -102,9 +99,6 @@ def _runtime_info():
     """
     Gather information from the environment where DVC runs to fill a report.
     """
-    from dvc import __version__
-    from dvc.utils import is_binary
-
     return {
         "dvc_version": __version__,
         "is_binary": is_binary(),
@@ -115,11 +109,6 @@ def _runtime_info():
 
 
 def _system_info():
-    import platform
-    import sys
-
-    import distro
-
     system = platform.system()
 
     if system == "Windows":
@@ -158,12 +147,6 @@ def _find_or_create_user_id():
 
     IDs are generated randomly with UUID.
     """
-    import uuid
-
-    from dvc.config import Config
-    from dvc.lock import Lock, LockError
-    from dvc.utils.fs import makedirs
-
     config_dir = Config.get_dir("global")
     fname = os.path.join(config_dir, "user_id")
     lockfile = os.path.join(config_dir, "user_id.lock")
@@ -175,13 +158,13 @@ def _find_or_create_user_id():
     try:
         with Lock(lockfile):
             try:
-                with open(fname, encoding="utf-8") as fobj:
+                with open(fname) as fobj:
                     user_id = json.load(fobj)["user_id"]
 
             except (FileNotFoundError, ValueError, KeyError):
                 user_id = str(uuid.uuid4())
 
-                with open(fname, "w", encoding="utf-8") as fobj:
+                with open(fname, "w") as fobj:
                     json.dump({"user_id": user_id}, fobj)
 
             return user_id

dvc/api.py

@@ -1,9 +1,16 @@
 import os
 from contextlib import _GeneratorContextManager as GCM
+from contextlib import contextmanager
 
 from funcy import reraise
 
-from dvc.exceptions import OutputNotFoundError, PathMissingError
+from dvc.exceptions import (
+    NotDvcRepoError,
+    OutputNotFoundError,
+    PathMissingError,
+)
+from dvc.external_repo import external_repo
+from dvc.path_info import PathInfo
 from dvc.repo import Repo
 
 
@@ -18,19 +25,17 @@ def get_url(path, repo=None, rev=None, remote=None):
     NOTE: This function does not check for the actual existence of the file or
     directory in the remote storage.
     """
-    with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
-        fs_path = _repo.dvcfs.from_os_path(path)
+    with _make_repo(repo, rev=rev) as _repo:
+        path_info = PathInfo(_repo.root_dir) / path
         with reraise(FileNotFoundError, PathMissingError(path, repo)):
-            info = _repo.dvcfs.info(fs_path)
+            metadata = _repo.repo_tree.metadata(path_info)
 
-        dvc_info = info.get("dvc_info")
-        if not dvc_info:
+        if not metadata.is_dvc:
             raise OutputNotFoundError(path, repo)
 
-        dvc_repo = info["repo"]
-        md5 = dvc_info["md5"]
-
-        return dvc_repo.cloud.get_url_for(remote, checksum=md5)
+        cloud = metadata.repo.cloud
+        hash_info = _repo.repo_tree.get_hash(path_info)
+        return cloud.get_url_for(remote, checksum=hash_info.value)
 
 
 def open(  # noqa, pylint: disable=redefined-builtin
@@ -73,7 +78,7 @@ class _OpenContextManager(GCM):
 
 
 def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
+    with _make_repo(repo, rev=rev) as _repo:
         with _repo.open_by_relpath(
             path, remote=remote, mode=mode, encoding=encoding
         ) as fd:
@@ -92,6 +97,19 @@ def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
         return fd.read()
 
 
+@contextmanager
+def _make_repo(repo_url=None, rev=None):
+    repo_url = repo_url or os.getcwd()
+    if rev is None and os.path.exists(repo_url):
+        try:
+            yield Repo(repo_url, subrepos=True)
+            return
+        except NotDvcRepoError:
+            pass  # fallthrough to external_repo
+    with external_repo(url=repo_url, rev=rev) as repo:
+        yield repo
+
+
 def make_checkpoint():
     """
     Signal DVC to create a checkpoint experiment.
@@ -104,21 +122,21 @@ def make_checkpoint():
     from time import sleep
 
     from dvc.env import DVC_CHECKPOINT, DVC_ROOT
-    from dvc.stage.monitor import CheckpointTask
+    from dvc.stage.run import CHECKPOINT_SIGNAL_FILE
 
     if os.getenv(DVC_CHECKPOINT) is None:
         return
 
     root_dir = os.getenv(DVC_ROOT, Repo.find_root())
     signal_file = os.path.join(
-        root_dir, Repo.DVC_DIR, "tmp", CheckpointTask.SIGNAL_FILE
+        root_dir, Repo.DVC_DIR, "tmp", CHECKPOINT_SIGNAL_FILE
     )
 
-    with builtins.open(signal_file, "w", encoding="utf-8") as fobj:
+    with builtins.open(signal_file, "w") as fobj:
         # NOTE: force flushing/writing empty file to disk, otherwise when
         # run in certain contexts (pytest) file may not actually be written
         fobj.write("")
         fobj.flush()
         os.fsync(fobj.fileno())
     while os.path.exists(signal_file):
-        sleep(0.1)
+        sleep(1)

dvc/cache/__init__.py

@@ -0,0 +1,176 @@
+"""Manages cache of a DVC repo."""
+from collections import defaultdict
+
+from ..scheme import Schemes
+
+
+def get_cloud_cache(tree):
+    from .base import CloudCache
+    from .local import LocalCache
+    from .ssh import SSHCache
+
+    if tree.scheme == Schemes.LOCAL:
+        return LocalCache(tree)
+
+    if tree.scheme == Schemes.SSH:
+        return SSHCache(tree)
+
+    return CloudCache(tree)
+
+
+def _get_cache(repo, settings):
+    from ..tree import get_cloud_tree
+
+    if not settings:
+        return None
+
+    tree = get_cloud_tree(repo, **settings)
+    return get_cloud_cache(tree)
+
+
+class Cache:
+    """Class that manages cache locations of a DVC repo.
+
+    Args:
+        repo (dvc.repo.Repo): repo instance that this cache belongs to.
+    """
+
+    CACHE_DIR = "cache"
+    CLOUD_SCHEMES = [
+        Schemes.S3,
+        Schemes.GS,
+        Schemes.SSH,
+        Schemes.HDFS,
+        Schemes.WEBHDFS,
+    ]
+
+    def __init__(self, repo):
+        self.repo = repo
+        self.config = config = repo.config["cache"]
+        self._cache = {}
+
+        local = config.get("local")
+
+        if local:
+            settings = {"name": local}
+        elif "dir" not in config:
+            settings = None
+        else:
+            from ..config import LOCAL_COMMON
+
+            settings = {"url": config["dir"]}
+            for opt in LOCAL_COMMON.keys():
+                if opt in config:
+                    settings[str(opt)] = config.get(opt)
+
+        self._cache[Schemes.LOCAL] = _get_cache(repo, settings)
+
+        for scheme in self.CLOUD_SCHEMES:
+            remote = self.config.get(scheme)
+            settings = {"name": remote} if remote else None
+            self._cache[scheme] = _get_cache(repo, settings)
+
+    def __getattr__(self, name):
+        try:
+            return self._cache[name]
+        except KeyError as exc:
+            raise AttributeError from exc
+
+    def by_scheme(self):
+        yield from self._cache.items()
+
+
+class NamedCacheItem:
+    def __init__(self):
+        self.names = set()
+        self.children = defaultdict(NamedCacheItem)
+
+    def __eq__(self, other):
+        return self.names == other.names and self.children == other.children
+
+    def child_keys(self):
+        for key, child in self.children.items():
+            yield key
+            yield from child.child_keys()
+
+    def child_names(self):
+        for key, child in self.children.items():
+            yield key, child.names
+            yield from child.child_names()
+
+    def add(self, checksum, item):
+        self.children[checksum].update(item)
+
+    def update(self, item, suffix=""):
+        if suffix:
+            self.names.update(n + suffix for n in item.names)
+        else:
+            self.names.update(item.names)
+        for checksum, child_item in item.children.items():
+            self.children[checksum].update(child_item)
+
+
+class NamedCache:
+    # pylint: disable=protected-access
+    def __init__(self):
+        self._items = defaultdict(lambda: defaultdict(NamedCacheItem))
+        self.external = defaultdict(set)
+
+    @classmethod
+    def make(cls, scheme, checksum, name):
+        cache = cls()
+        cache.add(scheme, checksum, name)
+        return cache
+
+    def __getitem__(self, key):
+        return self._items[key]
+
+    def add(self, scheme, checksum, name):
+        """Add a mapped name for the specified checksum."""
+        self._items[scheme][checksum].names.add(name)
+
+    def add_child_cache(self, checksum, cache, suffix=""):
+        """Add/update child cache for the specified checksum."""
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme][checksum].children
+            for child_checksum, item in src.items():
+                dst[child_checksum].update(item, suffix=suffix)
+
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)
+
+    def add_external(self, url, rev, path):
+        self.external[url, rev].add(path)
+
+    def update(self, cache, suffix=""):
+        for scheme, src in cache._items.items():
+            dst = self._items[scheme]
+            for checksum, item in src.items():
+                dst[checksum].update(item, suffix=suffix)
+
+        for repo_pair, files in cache.external.items():
+            self.external[repo_pair].update(files)
+
+    def scheme_keys(self, scheme):
+        """Iterate over a flat list of all keys for the specified scheme,
+        including children.
+        """
+        for key, item in self._items[scheme].items():
+            yield key
+            yield from item.child_keys()
+
+    def scheme_names(self, scheme):
+        """Iterate over a flat list of checksum, names items for the specified
+        scheme, including children.
+        """
+        for key, item in self._items[scheme].items():
+            yield key, item.names
+            yield from item.child_names()
+
+    def dir_keys(self, scheme):
+        return (
+            key for key, item in self._items[scheme].items() if item.children
+        )
+
+    def child_keys(self, scheme, checksum):
+        return self._items[scheme][checksum].child_keys()

dvc/cache/base.py

@@ -0,0 +1,937 @@
+import itertools
+import json
+import logging
+from concurrent.futures import ThreadPoolExecutor
+from copy import copy
+from typing import Optional
+
+from funcy import decorator
+from shortuuid import uuid
+
+import dvc.prompt as prompt
+from dvc.dir_info import DirInfo
+from dvc.exceptions import (
+    CacheLinkError,
+    CheckoutError,
+    ConfirmRemoveError,
+    DvcException,
+)
+from dvc.progress import Tqdm
+from dvc.remote.slow_link_detection import (  # type: ignore[attr-defined]
+    slow_link_guard,
+)
+
+from ..tree.base import RemoteActionNotImplemented
+
+logger = logging.getLogger(__name__)
+
+
+class DirCacheError(DvcException):
+    def __init__(self, hash_):
+        super().__init__(
+            f"Failed to load dir cache for hash value: '{hash_}'."
+        )
+
+
+@decorator
+def use_state(call):
+    tree = call._args[0].tree  # pylint: disable=protected-access
+    with tree.state:
+        return call()
+
+
+class CloudCache:
+    """Cloud cache class."""
+
+    DEFAULT_CACHE_TYPES = ["copy"]
+    CACHE_MODE: Optional[int] = None
+
+    def __init__(self, tree):
+        self.tree = tree
+        self.repo = tree.repo
+
+        self.cache_types = tree.config.get("type") or copy(
+            self.DEFAULT_CACHE_TYPES
+        )
+        self.cache_type_confirmed = False
+        self._dir_info = {}
+
+    def get_dir_cache(self, hash_info):
+        assert hash_info
+
+        dir_info = self._dir_info.get(hash_info.value)
+        if dir_info:
+            return dir_info
+
+        try:
+            dir_info = self.load_dir_cache(hash_info)
+        except DirCacheError:
+            dir_info = DirInfo()
+
+        self._dir_info[hash_info.value] = dir_info
+        return dir_info
+
+    def load_dir_cache(self, hash_info):
+        path_info = self.tree.hash_to_path_info(hash_info.value)
+
+        try:
+            with self.tree.open(path_info, "r") as fobj:
+                d = json.load(fobj)
+        except (ValueError, FileNotFoundError) as exc:
+            raise DirCacheError(hash_info) from exc
+
+        if not isinstance(d, list):
+            logger.error(
+                "dir cache file format error '%s' [skipping the file]",
+                path_info,
+            )
+            d = []
+
+        return DirInfo.from_list(d)
+
+    def changed(self, path_info, hash_info):
+        """Checks if data has changed.
+
+        A file is considered changed if:
+            - It doesn't exist on the working directory (was unlinked)
+            - Hash value is not computed (saving a new file)
+            - The hash value stored is different from the given one
+            - There's no file in the cache
+
+        Args:
+            path_info: dict with path information.
+            hash: expected hash value for this data.
+
+        Returns:
+            bool: True if data has changed, False otherwise.
+        """
+
+        logger.debug(
+            "checking if '%s'('%s') has changed.", path_info, hash_info
+        )
+
+        if not self.tree.exists(path_info):
+            logger.debug("'%s' doesn't exist.", path_info)
+            return True
+
+        if not hash_info:
+            logger.debug("hash value for '%s' is missing.", path_info)
+            return True
+
+        if self.changed_cache(hash_info):
+            logger.debug(
+                "cache for '%s'('%s') has changed.", path_info, hash_info
+            )
+            return True
+
+        actual = self.tree.get_hash(path_info)
+        if hash_info != actual:
+            logger.debug(
+                "hash value '%s' for '%s' has changed (actual '%s').",
+                hash_info,
+                actual,
+                path_info,
+            )
+            return True
+
+        logger.debug("'%s' hasn't changed.", path_info)
+        return False
+
+    def link(self, from_info, to_info):
+        self._link(from_info, to_info, self.cache_types)
+
+    def _link(self, from_info, to_info, link_types):
+        assert self.tree.isfile(from_info)
+
+        self.tree.makedirs(to_info.parent)
+
+        self._try_links(from_info, to_info, link_types)
+
+    def _verify_link(self, path_info, link_type):
+        if self.cache_type_confirmed:
+            return
+
+        is_link = getattr(self.tree, f"is_{link_type}", None)
+        if is_link and not is_link(path_info):
+            self.tree.remove(path_info)
+            raise DvcException(f"failed to verify {link_type}")
+
+        self.cache_type_confirmed = True
+
+    @slow_link_guard
+    def _try_links(self, from_info, to_info, link_types):
+        while link_types:
+            link_method = getattr(self.tree, link_types[0])
+            try:
+                self._do_link(from_info, to_info, link_method)
+                self._verify_link(to_info, link_types[0])
+                return
+
+            except DvcException as exc:
+                logger.debug(
+                    "Cache type '%s' is not supported: %s", link_types[0], exc
+                )
+                del link_types[0]
+
+        raise CacheLinkError([to_info])
+
+    def _do_link(self, from_info, to_info, link_method):
+        if self.tree.exists(to_info):
+            raise DvcException(f"Link '{to_info}' already exists!")
+
+        link_method(from_info, to_info)
+
+        logger.debug(
+            "Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
+        )
+
+    def _save_file(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        assert hash_info
+
+        cache_info = self.tree.hash_to_path_info(hash_info.value)
+        if tree == self.tree:
+            if self.changed_cache(hash_info):
+                self.tree.move(path_info, cache_info, mode=self.CACHE_MODE)
+                self.link(cache_info, path_info)
+            elif self.tree.iscopy(path_info) and self._cache_is_copy(
+                path_info
+            ):
+                # Default relink procedure involves unneeded copy
+                self.tree.unprotect(path_info)
+            else:
+                self.tree.remove(path_info)
+                self.link(cache_info, path_info)
+
+            if save_link:
+                self.tree.state.save_link(path_info)
+            # we need to update path and cache, since in case of reflink,
+            # or copy cache type moving original file results in updates on
+            # next executed command, which causes md5 recalculation
+            self.tree.state.save(path_info, hash_info)
+        else:
+            if self.changed_cache(hash_info):
+                with tree.open(path_info, mode="rb") as fobj:
+                    # if tree has fetch enabled, DVC out will be fetched on
+                    # open and we do not need to read/copy any data
+                    if not (
+                        tree.isdvc(path_info, strict=False) and tree.fetch
+                    ):
+                        self.tree.copy_fobj(fobj, cache_info)
+                callback = kwargs.get("download_callback")
+                if callback:
+                    callback(1)
+
+        self.tree.state.save(cache_info, hash_info)
+
+    def _cache_is_copy(self, path_info):
+        """Checks whether cache uses copies."""
+        if self.cache_type_confirmed:
+            return self.cache_types[0] == "copy"
+
+        if set(self.cache_types) <= {"copy"}:
+            return True
+
+        workspace_file = path_info.with_name("." + uuid())
+        test_cache_file = self.tree.path_info / ".cache_type_test_file"
+        if not self.tree.exists(test_cache_file):
+            with self.tree.open(test_cache_file, "wb") as fobj:
+                fobj.write(bytes(1))
+        try:
+            self.link(test_cache_file, workspace_file)
+        finally:
+            self.tree.remove(workspace_file)
+            self.tree.remove(test_cache_file)
+
+        self.cache_type_confirmed = True
+        return self.cache_types[0] == "copy"
+
+    def _get_dir_info_hash(self, dir_info):
+        import tempfile
+
+        from dvc.path_info import PathInfo
+        from dvc.utils import tmp_fname
+
+        tmp = tempfile.NamedTemporaryFile(delete=False).name
+        with open(tmp, "w+") as fobj:
+            json.dump(dir_info.to_list(), fobj, sort_keys=True)
+
+        from_info = PathInfo(tmp)
+        to_info = self.tree.path_info / tmp_fname("")
+        self.tree.upload(from_info, to_info, no_progress_bar=True)
+
+        hash_info = self.tree.get_file_hash(to_info)
+        hash_info.value += self.tree.CHECKSUM_DIR_SUFFIX
+        hash_info.dir_info = dir_info
+        hash_info.nfiles = dir_info.nfiles
+
+        return hash_info, to_info
+
+    @use_state
+    def save_dir_info(self, dir_info, hash_info=None):
+        if (
+            hash_info
+            and hash_info.name == self.tree.PARAM_CHECKSUM
+            and not self.changed_cache_file(hash_info)
+        ):
+            return hash_info
+
+        hi, tmp_info = self._get_dir_info_hash(dir_info)
+        new_info = self.tree.hash_to_path_info(hi.value)
+        if self.changed_cache_file(hi):
+            self.tree.makedirs(new_info.parent)
+            self.tree.move(tmp_info, new_info, mode=self.CACHE_MODE)
+
+        self.tree.state.save(new_info, hi)
+
+        return hi
+
+    def _save_dir(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        if not hash_info.dir_info:
+            hash_info.dir_info = tree.cache.get_dir_cache(hash_info)
+        hi = self.save_dir_info(hash_info.dir_info, hash_info)
+        for entry_info, entry_hash in Tqdm(
+            hi.dir_info.items(path_info),
+            desc="Saving " + path_info.name,
+            unit="file",
+        ):
+            self._save_file(
+                entry_info, tree, entry_hash, save_link=False, **kwargs
+            )
+
+        if save_link:
+            self.tree.state.save_link(path_info)
+        if self.tree.exists(path_info):
+            self.tree.state.save(path_info, hi)
+
+        cache_info = self.tree.hash_to_path_info(hi.value)
+        self.tree.state.save(cache_info, hi)
+
+    @use_state
+    def save(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        if path_info.scheme != self.tree.scheme:
+            raise RemoteActionNotImplemented(
+                f"save {path_info.scheme} -> {self.tree.scheme}",
+                self.tree.scheme,
+            )
+
+        self._save(path_info, tree, hash_info, save_link, **kwargs)
+
+    def _save(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        to_info = self.tree.hash_to_path_info(hash_info.value)
+        logger.debug("Saving '%s' to '%s'.", path_info, to_info)
+
+        if tree.isdir(path_info):
+            self._save_dir(path_info, tree, hash_info, save_link, **kwargs)
+        else:
+            self._save_file(path_info, tree, hash_info, save_link, **kwargs)
+
+    # Override to return path as a string instead of PathInfo for clouds
+    # which support string paths (see local)
+    def hash_to_path(self, hash_):
+        return self.tree.hash_to_path_info(hash_)
+
+    def changed_cache_file(self, hash_info):
+        """Compare the given hash with the (corresponding) actual one.
+
+        - Use `State` as a cache for computed hashes
+            + The entries are invalidated by taking into account the following:
+                * mtime
+                * inode
+                * size
+                * hash
+
+        - Remove the file from cache if it doesn't match the actual hash
+        """
+        # Prefer string path over PathInfo when possible due to performance
+        cache_info = self.hash_to_path(hash_info.value)
+        if self.tree.is_protected(cache_info):
+            logger.debug(
+                "Assuming '%s' is unchanged since it is read-only", cache_info
+            )
+            return False
+
+        actual = self.tree.get_hash(cache_info)
+
+        logger.debug(
+            "cache '%s' expected '%s' actual '%s'",
+            cache_info,
+            hash_info,
+            actual,
+        )
+
+        if not hash_info or not actual:
+            return True
+
+        if actual.value.split(".")[0] == hash_info.value.split(".")[0]:
+            # making cache file read-only so we don't need to check it
+            # next time
+            self.tree.protect(cache_info)
+            return False
+
+        if self.tree.exists(cache_info):
+            logger.warning("corrupted cache file '%s'.", cache_info)
+            self.tree.remove(cache_info)
+
+        return True
+
+    def _changed_dir_cache(self, hash_info, path_info=None, filter_info=None):
+        if self.changed_cache_file(hash_info):
+            return True
+
+        dir_info = self.get_dir_cache(hash_info)
+        for entry_info, entry_hash in dir_info.items(path_info):
+            if path_info and filter_info:
+                if not entry_info.isin_or_eq(filter_info):
+                    continue
+
+            if self.changed_cache_file(entry_hash):
+                return True
+
+        return False
+
+    def changed_cache(self, hash_info, path_info=None, filter_info=None):
+        if hash_info.isdir:
+            return self._changed_dir_cache(
+                hash_info, path_info=path_info, filter_info=filter_info
+            )
+        return self.changed_cache_file(hash_info)
+
+    def already_cached(self, path_info):
+        current = self.tree.get_hash(path_info)
+
+        if not current:
+            return False
+
+        return not self.changed_cache(current)
+
+    def safe_remove(self, path_info, force=False):
+        if not self.tree.exists(path_info):
+            return
+
+        if not force and not self.already_cached(path_info):
+            msg = (
+                "file '{}' is going to be removed."
+                " Are you sure you want to proceed?".format(str(path_info))
+            )
+
+            if not prompt.confirm(msg):
+                raise ConfirmRemoveError(str(path_info))
+
+        self.tree.remove(path_info)
+
+    def _checkout_file(
+        self, path_info, hash_info, force, progress_callback=None, relink=False
+    ):
+        """The file is changed we need to checkout a new copy"""
+        added, modified = True, False
+        cache_info = self.tree.hash_to_path_info(hash_info.value)
+        if self.tree.exists(path_info):
+            logger.debug("data '%s' will be replaced.", path_info)
+            self.safe_remove(path_info, force=force)
+            added, modified = False, True
+
+        self.link(cache_info, path_info)
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_info)
+        if progress_callback:
+            progress_callback(str(path_info))
+
+        return added, modified and not relink
+
+    def _checkout_dir(
+        self,
+        path_info,
+        hash_info,
+        force,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        added, modified = False, False
+        # Create dir separately so that dir is created
+        # even if there are no files in it
+        if not self.tree.exists(path_info):
+            added = True
+            self.tree.makedirs(path_info)
+
+        dir_info = self.get_dir_cache(hash_info)
+
+        logger.debug("Linking directory '%s'.", path_info)
+
+        for entry_info, entry_hash_info in dir_info.items(path_info):
+            entry_cache_info = self.tree.hash_to_path_info(
+                entry_hash_info.value
+            )
+
+            if filter_info and not entry_info.isin_or_eq(filter_info):
+                continue
+
+            if relink or self.changed(entry_info, entry_hash_info):
+                modified = True
+                self.safe_remove(entry_info, force=force)
+                self.link(entry_cache_info, entry_info)
+                self.tree.state.save(entry_info, entry_hash_info)
+            if progress_callback:
+                progress_callback(str(entry_info))
+
+        modified = (
+            self._remove_redundant_files(path_info, dir_info, force)
+            or modified
+        )
+
+        self.tree.state.save_link(path_info)
+        self.tree.state.save(path_info, hash_info)
+
+        # relink is not modified, assume it as nochange
+        return added, not added and modified and not relink
+
+    def _remove_redundant_files(self, path_info, dir_info, force):
+        existing_files = set(self.tree.walk_files(path_info))
+
+        needed_files = {info for info, _ in dir_info.items(path_info)}
+        redundant_files = existing_files - needed_files
+        for path in redundant_files:
+            self.safe_remove(path, force)
+
+        return bool(redundant_files)
+
+    @use_state
+    def checkout(
+        self,
+        path_info,
+        hash_info,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+        quiet=False,
+    ):
+        if path_info.scheme not in ["local", self.tree.scheme]:
+            raise NotImplementedError
+
+        failed = None
+        skip = False
+        if not hash_info:
+            if not quiet:
+                logger.warning(
+                    "No file hash info found for '%s'. It won't be created.",
+                    path_info,
+                )
+            self.safe_remove(path_info, force=force)
+            failed = path_info
+
+        elif not relink and not self.changed(path_info, hash_info):
+            logger.debug("Data '%s' didn't change.", path_info)
+            skip = True
+
+        elif self.changed_cache(
+            hash_info, path_info=path_info, filter_info=filter_info
+        ):
+            if not quiet:
+                logger.warning(
+                    "Cache '%s' not found. File '%s' won't be created.",
+                    hash_info,
+                    path_info,
+                )
+            self.safe_remove(path_info, force=force)
+            failed = path_info
+
+        if failed or skip:
+            if progress_callback:
+                progress_callback(
+                    str(path_info),
+                    self.get_files_number(
+                        self.tree.path_info, hash_info, filter_info
+                    ),
+                )
+            if failed:
+                raise CheckoutError([failed])
+            return
+
+        logger.debug(
+            "Checking out '%s' with cache '%s'.", path_info, hash_info
+        )
+
+        return self._checkout(
+            path_info,
+            hash_info,
+            force,
+            progress_callback,
+            relink,
+            filter_info,
+        )
+
+    def _checkout(
+        self,
+        path_info,
+        hash_info,
+        force=False,
+        progress_callback=None,
+        relink=False,
+        filter_info=None,
+    ):
+        if not hash_info.isdir:
+            return self._checkout_file(
+                path_info, hash_info, force, progress_callback, relink
+            )
+
+        return self._checkout_dir(
+            path_info, hash_info, force, progress_callback, relink, filter_info
+        )
+
+    def get_files_number(self, path_info, hash_info, filter_info):
+        from funcy.py3 import ilen
+
+        if not hash_info:
+            return 0
+
+        if not hash_info.isdir:
+            return 1
+
+        if not filter_info:
+            return self.get_dir_cache(hash_info).nfiles
+
+        return ilen(
+            filter_info.isin_or_eq(path_info / relpath)
+            for relpath, _ in self.get_dir_cache(hash_info).items()
+        )
+
+    def _get_dir_size(self, dir_info):
+        try:
+            return sum(
+                self.tree.getsize(self.tree.hash_to_path_info(hi.value))
+                for _, hi in dir_info.items()
+            )
+        except FileNotFoundError:
+            return None
+
+    def merge(self, ancestor_info, our_info, their_info):
+        assert our_info
+        assert their_info
+
+        if ancestor_info:
+            ancestor = self.get_dir_cache(ancestor_info)
+        else:
+            ancestor = DirInfo()
+
+        our = self.get_dir_cache(our_info)
+        their = self.get_dir_cache(their_info)
+
+        merged = our.merge(ancestor, their)
+        hash_info = self.save_dir_info(merged)
+        hash_info.size = self._get_dir_size(merged)
+        return hash_info
+
+    @use_state
+    def get_hash(self, tree, path_info):
+        hash_info = tree.get_hash(path_info)
+        if not hash_info.isdir:
+            assert hash_info.name == self.tree.PARAM_CHECKSUM
+            return hash_info
+
+        hi = self.save_dir_info(hash_info.dir_info, hash_info)
+        hi.size = hash_info.size
+        return hi
+
+    def set_dir_info(self, hash_info):
+        assert hash_info.isdir
+
+        hash_info.dir_info = self.get_dir_cache(hash_info)
+        hash_info.nfiles = hash_info.dir_info.nfiles
+
+    def _list_paths(self, prefix=None, progress_callback=None):
+        if prefix:
+            if len(prefix) > 2:
+                path_info = self.tree.path_info / prefix[:2] / prefix[2:]
+            else:
+                path_info = self.tree.path_info / prefix[:2]
+            prefix = True
+        else:
+            path_info = self.tree.path_info
+            prefix = False
+        if progress_callback:
+            for file_info in self.tree.walk_files(path_info, prefix=prefix):
+                progress_callback()
+                yield file_info.path
+        else:
+            yield from self.tree.walk_files(path_info, prefix=prefix)
+
+    def _path_to_hash(self, path):
+        parts = self.tree.PATH_CLS(path).parts[-2:]
+
+        if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
+            raise ValueError(f"Bad cache file path '{path}'")
+
+        return "".join(parts)
+
+    def list_hashes(self, prefix=None, progress_callback=None):
+        """Iterate over hashes in this tree.
+
+        If `prefix` is specified, only hashes which begin with `prefix`
+        will be returned.
+        """
+        for path in self._list_paths(prefix, progress_callback):
+            try:
+                yield self._path_to_hash(path)
+            except ValueError:
+                logger.debug(
+                    "'%s' doesn't look like a cache file, skipping", path
+                )
+
+    def _hashes_with_limit(self, limit, prefix=None, progress_callback=None):
+        count = 0
+        for hash_ in self.list_hashes(prefix, progress_callback):
+            yield hash_
+            count += 1
+            if count > limit:
+                logger.debug(
+                    "`list_hashes()` returned max '{}' hashes, "
+                    "skipping remaining results".format(limit)
+                )
+                return
+
+    def _max_estimation_size(self, hashes):
+        # Max remote size allowed for us to use traverse method
+        return max(
+            self.tree.TRAVERSE_THRESHOLD_SIZE,
+            len(hashes)
+            / self.tree.TRAVERSE_WEIGHT_MULTIPLIER
+            * self.tree.LIST_OBJECT_PAGE_SIZE,
+        )
+
+    def _estimate_remote_size(self, hashes=None, name=None):
+        """Estimate tree size based on number of entries beginning with
+        "00..." prefix.
+        """
+        prefix = "0" * self.tree.TRAVERSE_PREFIX_LEN
+        total_prefixes = pow(16, self.tree.TRAVERSE_PREFIX_LEN)
+        if hashes:
+            max_hashes = self._max_estimation_size(hashes)
+        else:
+            max_hashes = None
+
+        with Tqdm(
+            desc="Estimating size of "
+            + (f"cache in '{name}'" if name else "remote cache"),
+            unit="file",
+        ) as pbar:
+
+            def update(n=1):
+                pbar.update(n * total_prefixes)
+
+            if max_hashes:
+                hashes = self._hashes_with_limit(
+                    max_hashes / total_prefixes, prefix, update
+                )
+            else:
+                hashes = self.list_hashes(prefix, update)
+
+            remote_hashes = set(hashes)
+            if remote_hashes:
+                remote_size = total_prefixes * len(remote_hashes)
+            else:
+                remote_size = total_prefixes
+            logger.debug(f"Estimated remote size: {remote_size} files")
+        return remote_size, remote_hashes
+
+    def list_hashes_traverse(
+        self, remote_size, remote_hashes, jobs=None, name=None
+    ):
+        """Iterate over all hashes found in this tree.
+        Hashes are fetched in parallel according to prefix, except in
+        cases where the remote size is very small.
+
+        All hashes from the remote (including any from the size
+        estimation step passed via the `remote_hashes` argument) will be
+        returned.
+
+        NOTE: For large remotes the list of hashes will be very
+        big(e.g. 100M entries, md5 for each is 32 bytes, so ~3200Mb list)
+        and we don't really need all of it at the same time, so it makes
+        sense to use a generator to gradually iterate over it, without
+        keeping all of it in memory.
+        """
+        num_pages = remote_size / self.tree.LIST_OBJECT_PAGE_SIZE
+        if num_pages < 256 / self.tree.JOBS:
+            # Fetching prefixes in parallel requires at least 255 more
+            # requests, for small enough remotes it will be faster to fetch
+            # entire cache without splitting it into prefixes.
+            #
+            # NOTE: this ends up re-fetching hashes that were already
+            # fetched during remote size estimation
+            traverse_prefixes = [None]
+            initial = 0
+        else:
+            yield from remote_hashes
+            initial = len(remote_hashes)
+            traverse_prefixes = [f"{i:02x}" for i in range(1, 256)]
+            if self.tree.TRAVERSE_PREFIX_LEN > 2:
+                traverse_prefixes += [
+                    "{0:0{1}x}".format(i, self.tree.TRAVERSE_PREFIX_LEN)
+                    for i in range(
+                        1, pow(16, self.tree.TRAVERSE_PREFIX_LEN - 2)
+                    )
+                ]
+        with Tqdm(
+            desc="Querying "
+            + (f"cache in '{name}'" if name else "remote cache"),
+            total=remote_size,
+            initial=initial,
+            unit="file",
+        ) as pbar:
+
+            def list_with_update(prefix):
+                return list(
+                    self.list_hashes(
+                        prefix=prefix, progress_callback=pbar.update
+                    )
+                )
+
+            with ThreadPoolExecutor(
+                max_workers=jobs or self.tree.JOBS
+            ) as executor:
+                in_remote = executor.map(list_with_update, traverse_prefixes,)
+                yield from itertools.chain.from_iterable(in_remote)
+
+    def all(self, jobs=None, name=None):
+        """Iterate over all hashes in this tree.
+
+        Hashes will be fetched in parallel threads according to prefix
+        (except for small remotes) and a progress bar will be displayed.
+        """
+        logger.debug(
+            "Fetching all hashes from '{}'".format(
+                name if name else "remote cache"
+            )
+        )
+
+        if not self.tree.CAN_TRAVERSE:
+            return self.list_hashes()
+
+        remote_size, remote_hashes = self._estimate_remote_size(name=name)
+        return self.list_hashes_traverse(
+            remote_size, remote_hashes, jobs, name
+        )
+
+    def _remove_unpacked_dir(self, hash_):
+        pass
+
+    def gc(self, used, jobs=None):
+        removed = False
+        # hashes must be sorted to ensure we always remove .dir files first
+        for hash_ in sorted(
+            self.all(jobs, str(self.tree.path_info)),
+            key=self.tree.is_dir_hash,
+            reverse=True,
+        ):
+            if hash_ in used:
+                continue
+            path_info = self.tree.hash_to_path_info(hash_)
+            if self.tree.is_dir_hash(hash_):
+                # backward compatibility
+                # pylint: disable=protected-access
+                self._remove_unpacked_dir(hash_)
+            self.tree.remove(path_info)
+            removed = True
+
+        return removed
+
+    def list_hashes_exists(self, hashes, jobs=None, name=None):
+        """Return list of the specified hashes which exist in this tree.
+        Hashes will be queried individually.
+        """
+        logger.debug(
+            "Querying {} hashes via object_exists".format(len(hashes))
+        )
+        with Tqdm(
+            desc="Querying "
+            + ("cache in " + name if name else "remote cache"),
+            total=len(hashes),
+            unit="file",
+        ) as pbar:
+
+            def exists_with_progress(path_info):
+                ret = self.tree.exists(path_info)
+                pbar.update_msg(str(path_info))
+                return ret
+
+            with ThreadPoolExecutor(
+                max_workers=jobs or self.tree.JOBS
+            ) as executor:
+                path_infos = map(self.tree.hash_to_path_info, hashes)
+                in_remote = executor.map(exists_with_progress, path_infos)
+                ret = list(itertools.compress(hashes, in_remote))
+                return ret
+
+    def hashes_exist(self, hashes, jobs=None, name=None):
+        """Check if the given hashes are stored in the remote.
+
+        There are two ways of performing this check:
+
+        - Traverse method: Get a list of all the files in the remote
+            (traversing the cache directory) and compare it with
+            the given hashes. Cache entries will be retrieved in parallel
+            threads according to prefix (i.e. entries starting with, "00...",
+            "01...", and so on) and a progress bar will be displayed.
+
+        - Exists method: For each given hash, run the `exists`
+            method and filter the hashes that aren't on the remote.
+            This is done in parallel threads.
+            It also shows a progress bar when performing the check.
+
+        The reason for such an odd logic is that most of the remotes
+        take much shorter time to just retrieve everything they have under
+        a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can
+        check if particular file exists much quicker, use their own
+        implementation of hashes_exist (see ssh, local).
+
+        Which method to use will be automatically determined after estimating
+        the size of the remote cache, and comparing the estimated size with
+        len(hashes). To estimate the size of the remote cache, we fetch
+        a small subset of cache entries (i.e. entries starting with "00...").
+        Based on the number of entries in that subset, the size of the full
+        cache can be estimated, since the cache is evenly distributed according
+        to hash.
+
+        Returns:
+            A list with hashes that were found in the remote
+        """
+        # Remotes which do not use traverse prefix should override
+        # hashes_exist() (see ssh, local)
+        assert self.tree.TRAVERSE_PREFIX_LEN >= 2
+
+        hashes = set(hashes)
+        if len(hashes) == 1 or not self.tree.CAN_TRAVERSE:
+            remote_hashes = self.list_hashes_exists(hashes, jobs, name)
+            return remote_hashes
+
+        # Max remote size allowed for us to use traverse method
+        remote_size, remote_hashes = self._estimate_remote_size(hashes, name)
+
+        traverse_pages = remote_size / self.tree.LIST_OBJECT_PAGE_SIZE
+        # For sufficiently large remotes, traverse must be weighted to account
+        # for performance overhead from large lists/sets.
+        # From testing with S3, for remotes with 1M+ files, object_exists is
+        # faster until len(hashes) is at least 10k~100k
+        if remote_size > self.tree.TRAVERSE_THRESHOLD_SIZE:
+            traverse_weight = (
+                traverse_pages * self.tree.TRAVERSE_WEIGHT_MULTIPLIER
+            )
+        else:
+            traverse_weight = traverse_pages
+        if len(hashes) < traverse_weight:
+            logger.debug(
+                "Large remote ('{}' hashes < '{}' traverse weight), "
+                "using object_exists for remaining hashes".format(
+                    len(hashes), traverse_weight
+                )
+            )
+            return list(hashes & remote_hashes) + self.list_hashes_exists(
+                hashes - remote_hashes, jobs, name
+            )
+
+        logger.debug("Querying '{}' hashes via traverse".format(len(hashes)))
+        remote_hashes = set(
+            self.list_hashes_traverse(remote_size, remote_hashes, jobs, name)
+        )
+        return list(hashes & set(remote_hashes))

dvc/cache/local.py

@@ -0,0 +1,91 @@
+import logging
+import os
+
+from funcy import cached_property
+
+from dvc.hash_info import HashInfo
+from dvc.path_info import PathInfo
+from dvc.progress import Tqdm
+
+from ..tree.local import LocalTree
+from ..utils.fs import walk_files
+from .base import CloudCache
+
+logger = logging.getLogger(__name__)
+
+
+class LocalCache(CloudCache):
+    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
+    CACHE_MODE = LocalTree.CACHE_MODE
+    UNPACKED_DIR_SUFFIX = ".unpacked"
+
+    def __init__(self, tree):
+        super().__init__(tree)
+        self.cache_dir = tree.config.get("url")
+
+    @property
+    def cache_dir(self):
+        return self.tree.path_info.fspath if self.tree.path_info else None
+
+    @cache_dir.setter
+    def cache_dir(self, value):
+        self.tree.path_info = PathInfo(value) if value else None
+
+    @cached_property
+    def cache_path(self):
+        return os.path.abspath(self.cache_dir)
+
+    def hash_to_path(self, hash_):
+        # NOTE: `self.cache_path` is already normalized so we can simply use
+        # `os.sep` instead of `os.path.join`. This results in this helper
+        # being ~5.5 times faster.
+        return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
+
+    def hashes_exist(
+        self, hashes, jobs=None, name=None
+    ):  # pylint: disable=unused-argument
+        return [
+            hash_
+            for hash_ in Tqdm(
+                hashes,
+                unit="file",
+                desc="Querying "
+                + ("cache in " + name if name else "local cache"),
+            )
+            if not self.changed_cache_file(
+                HashInfo(self.tree.PARAM_CHECKSUM, hash_)
+            )
+        ]
+
+    def already_cached(self, path_info):
+        assert path_info.scheme in ["", "local"]
+
+        return super().already_cached(path_info)
+
+    def _verify_link(self, path_info, link_type):
+        if link_type == "hardlink" and self.tree.getsize(path_info) == 0:
+            return
+
+        super()._verify_link(path_info, link_type)
+
+    def _list_paths(self, prefix=None, progress_callback=None):
+        assert self.tree.path_info is not None
+        if prefix:
+            path_info = self.tree.path_info / prefix[:2]
+            if not self.tree.exists(path_info):
+                return
+        else:
+            path_info = self.tree.path_info
+        # NOTE: use utils.fs walk_files since tree.walk_files will not follow
+        # symlinks
+        if progress_callback:
+            for path in walk_files(path_info):
+                progress_callback()
+                yield path
+        else:
+            yield from walk_files(path_info)
+
+    def _remove_unpacked_dir(self, hash_):
+        info = self.tree.hash_to_path_info(hash_)
+        path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
+        self.tree.remove(path_info)

dvc/cache/ssh.py

@@ -0,0 +1,90 @@
+import errno
+import itertools
+import logging
+import posixpath
+from concurrent.futures import ThreadPoolExecutor
+
+from dvc.progress import Tqdm
+from dvc.utils import to_chunks
+
+from .base import CloudCache
+
+logger = logging.getLogger(__name__)
+
+
+class SSHCache(CloudCache):
+    def batch_exists(self, path_infos, callback):
+        def _exists(chunk_and_channel):
+            chunk, channel = chunk_and_channel
+            ret = []
+            for path in chunk:
+                try:
+                    channel.stat(path)
+                    ret.append(True)
+                except OSError as exc:
+                    if exc.errno != errno.ENOENT:
+                        raise
+                    ret.append(False)
+                callback(path)
+            return ret
+
+        with self.tree.ssh(path_infos[0]) as ssh:
+            channels = ssh.open_max_sftp_channels()
+            max_workers = len(channels)
+
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                paths = [path_info.path for path_info in path_infos]
+                chunks = to_chunks(paths, num_chunks=max_workers)
+                chunks_and_channels = zip(chunks, channels)
+                outcome = executor.map(_exists, chunks_and_channels)
+                results = list(itertools.chain.from_iterable(outcome))
+
+            return results
+
+    def hashes_exist(self, hashes, jobs=None, name=None):
+        """This is older implementation used in remote/base.py
+        We are reusing it in RemoteSSH, because SSH's batch_exists proved to be
+        faster than current approach (relying on exists(path_info)) applied in
+        remote/base.
+        """
+        if not self.tree.CAN_TRAVERSE:
+            return list(set(hashes) & set(self.all()))
+
+        # possibly prompt for credentials before "Querying" progress output
+        self.tree.ensure_credentials()
+
+        with Tqdm(
+            desc="Querying "
+            + ("cache in " + name if name else "remote cache"),
+            total=len(hashes),
+            unit="file",
+        ) as pbar:
+
+            def exists_with_progress(chunks):
+                return self.batch_exists(chunks, callback=pbar.update_msg)
+
+            with ThreadPoolExecutor(
+                max_workers=jobs or self.tree.JOBS
+            ) as executor:
+                path_infos = [self.tree.hash_to_path_info(x) for x in hashes]
+                chunks = to_chunks(path_infos, num_chunks=self.tree.JOBS)
+                results = executor.map(exists_with_progress, chunks)
+                in_remote = itertools.chain.from_iterable(results)
+                ret = list(itertools.compress(hashes, in_remote))
+                return ret
+
+    def _list_paths(self, prefix=None, progress_callback=None):
+        if prefix:
+            root = posixpath.join(self.tree.path_info.path, prefix[:2])
+        else:
+            root = self.tree.path_info.path
+        with self.tree.ssh(self.tree.path_info) as ssh:
+            if prefix and not ssh.exists(root):
+                return
+            # If we simply return an iterator then with above closes instantly
+            if progress_callback:
+                for path in ssh.walk_files(root):
+                    progress_callback()
+                    yield path
+            else:
+                yield from ssh.walk_files(root)

dvc/cli.py

@@ -1,10 +1,10 @@
-"""Main parser for the dvc cli."""
+"""DVC command line interface"""
 import argparse
 import logging
 import os
 import sys
 
-from dvc.commands import (
+from .command import (
     add,
     cache,
     check_ignore,
@@ -28,7 +28,6 @@ from dvc.commands import (
     init,
     install,
     ls,
-    machine,
     metrics,
     move,
     params,
@@ -38,13 +37,12 @@ from dvc.commands import (
     repro,
     root,
     run,
-    stage,
     unprotect,
     update,
     version,
 )
-
-from . import DvcParserError
+from .command.base import fix_subparsers
+from .exceptions import DvcParserError
 
 logger = logging.getLogger(__name__)
 
@@ -82,10 +80,8 @@ COMMANDS = [
     update,
     git_hook,
     plots,
-    stage,
     experiments,
     check_ignore,
-    machine,
 ]
 
 
@@ -93,7 +89,7 @@ def _find_parser(parser, cmd_cls):
     defaults = parser._defaults  # pylint: disable=protected-access
     if not cmd_cls or cmd_cls == defaults.get("func"):
         parser.print_help()
-        raise DvcParserError
+        raise DvcParserError()
 
     actions = parser._actions  # pylint: disable=protected-access
     for action in actions:
@@ -140,10 +136,20 @@ def get_parent_parser():
     When overwriting `-q` or `-v`, you need to instantiate a new object
     in order to prevent some weird behavior.
     """
-    from dvc._debug import add_debugging_flags
-
     parent_parser = argparse.ArgumentParser(add_help=False)
-    add_debugging_flags(parent_parser)
+
+    parent_parser.add_argument(
+        "--cprofile",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parent_parser.add_argument("--cprofile-dump", help=argparse.SUPPRESS)
+
+    parent_parser.add_argument(
+        "--pdb", action="store_true", default=False, help=argparse.SUPPRESS,
+    )
+
     log_level_group = parent_parser.add_mutually_exclusive_group()
     log_level_group.add_argument(
         "-q", "--quiet", action="count", default=0, help="Be quiet."
@@ -206,11 +212,23 @@ def get_main_parser():
         help="Use `dvc COMMAND --help` for command-specific help.",
     )
 
-    from .utils import fix_subparsers
-
     fix_subparsers(subparsers)
 
     for cmd in COMMANDS:
         cmd.add_parser(subparsers, parent_parser)
 
     return parser
+
+
+def parse_args(argv=None):
+    """Parses CLI arguments.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Raises:
+        dvc.exceptions.DvcParserError: raised for argument parsing errors.
+    """
+    parser = get_main_parser()
+    args = parser.parse_args(argv)
+    return args

dvc/cli/__init__.py

@@ -1,218 +0,0 @@
-"""This module provides an entrypoint to the dvc cli and parsing utils."""
-
-import logging
-import sys
-
-# Workaround for CPython bug. See [1] and [2] for more info.
-# [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
-# [2] https://bugs.python.org/issue29288
-from typing import Optional
-
-"".encode("idna")
-
-logger = logging.getLogger("dvc")
-
-
-class DvcParserError(Exception):
-    """Base class for CLI parser errors."""
-
-    def __init__(self):
-        super().__init__("parser error")
-
-
-def parse_args(argv=None):
-    """Parses CLI arguments.
-
-    Args:
-        argv: optional list of arguments to parse. sys.argv is used by default.
-
-    Raises:
-        DvcParserError: raised for argument parsing errors.
-    """
-    from .parser import get_main_parser
-
-    parser = get_main_parser()
-    args = parser.parse_args(argv)
-    args.parser = parser
-    return args
-
-
-def _log_unknown_exceptions() -> None:
-    from dvc.info import get_dvc_info
-    from dvc.logger import FOOTER
-
-    logger.exception("unexpected error")
-    if logger.isEnabledFor(logging.DEBUG):
-        logger.debug("Version info for developers:\n%s", get_dvc_info())
-    logger.info(FOOTER)
-    return None
-
-
-def _log_exceptions(exc: Exception) -> Optional[int]:
-    """Try to log some known exceptions, that are not DVCExceptions."""
-    from dvc.utils import error_link, format_link
-
-    if isinstance(exc, OSError):
-        import errno
-
-        if exc.errno == errno.EMFILE:
-            logger.exception(
-                "too many open files, please visit "
-                "{} to see how to handle this "
-                "problem".format(error_link("many-files")),
-                extra={"tb_only": True},
-            )
-        else:
-            _log_unknown_exceptions()
-        return None
-
-    from dvc.fs import AuthError, ConfigError, RemoteMissingDepsError
-
-    if isinstance(exc, RemoteMissingDepsError):
-        from dvc.utils.pkg import PKG
-
-        proto = exc.protocol
-        by_pkg = {
-            "pip": f"pip install 'dvc[{proto}]'",
-            "conda": f"conda install -c conda-forge dvc-{proto}",
-        }
-
-        cmd = by_pkg.get(PKG)
-        if cmd:
-            link = format_link("https://dvc.org/doc/install")
-            hint = (
-                f"To install dvc with those dependencies, run:\n"
-                "\n"
-                f"\t{cmd}\n"
-                "\n"
-                f"See {link} for more info."
-            )
-        else:
-            link = format_link("https://github.com/iterative/dvc/issues")
-            hint = f"\nPlease report this bug to {link}. Thank you!"
-
-        logger.exception(
-            f"URL '{exc.url}' is supported but requires these missing "
-            f"dependencies: {exc.missing_deps}. {hint}",
-            extra={"tb_only": True},
-        )
-        return None
-
-    if isinstance(exc, (AuthError, ConfigError)):
-        link = format_link("https://man.dvc.org/remote/modify")
-        logger.exception("configuration error")
-        logger.exception(
-            f"{exc!s}\nLearn more about configuration settings at {link}.",
-            extra={"tb_only": True},
-        )
-        return 251
-
-    from dvc_objects.cache import DiskError
-
-    if isinstance(exc, DiskError):
-        from dvc.utils import relpath
-
-        directory = relpath(exc.directory)
-        logger.exception(
-            f"Could not open pickled '{exc.type}' cache.\n"
-            f"Remove the '{directory}' directory and then retry this command."
-            f"\nSee {error_link('pickle')} for more information.",
-            extra={"tb_only": True},
-        )
-        return None
-
-    from dvc_data.stage import IgnoreInCollectedDirError
-
-    if isinstance(exc, IgnoreInCollectedDirError):
-        logger.exception("")
-        return None
-
-    _log_unknown_exceptions()
-    return None
-
-
-def main(argv=None):  # noqa: C901
-    """Main entry point for dvc CLI.
-
-    Args:
-        argv: optional list of arguments to parse. sys.argv is used by default.
-
-    Returns:
-        int: command's return code.
-    """
-    from dvc._debug import debugtools
-    from dvc.config import ConfigError
-    from dvc.exceptions import DvcException, NotDvcRepoError
-    from dvc.logger import disable_other_loggers, set_loggers_level
-
-    # NOTE: stderr/stdout may be closed if we are running from dvc.daemon.
-    # On Linux we directly call cli.main after double forking and closing
-    # the copied parent's standard file descriptors. If we make any logging
-    # calls in this state it will cause an exception due to writing to a closed
-    # file descriptor.
-    if sys.stderr.closed:  # pylint: disable=using-constant-test
-        logging.disable()
-    elif sys.stdout.closed:  # pylint: disable=using-constant-test
-        logging.disable(logging.INFO)
-
-    args = None
-    disable_other_loggers()
-
-    outerLogLevel = logger.level
-    try:
-        args = parse_args(argv)
-
-        level = None
-        if args.quiet:
-            level = logging.CRITICAL
-        elif args.verbose == 1:
-            level = logging.DEBUG
-        elif args.verbose > 1:
-            level = logging.TRACE
-
-        if level is not None:
-            set_loggers_level(level)
-
-        logger.trace(args)
-
-        if not sys.stdout.closed and not args.quiet:
-            from dvc.ui import ui
-
-            ui.enable()
-
-        with debugtools(args):
-            cmd = args.func(args)
-            ret = cmd.do_run()
-    except ConfigError:
-        logger.exception("configuration error")
-        ret = 251
-    except KeyboardInterrupt:
-        logger.exception("interrupted by the user")
-        ret = 252
-    except NotDvcRepoError:
-        logger.exception("")
-        ret = 253
-    except DvcException:
-        ret = 255
-        logger.exception("")
-    except DvcParserError:
-        ret = 254
-    except Exception as exc:  # noqa, pylint: disable=broad-except
-        # pylint: disable=no-member
-        ret = _log_exceptions(exc) or 255
-
-    try:
-        from dvc import analytics
-
-        if analytics.is_enabled():
-            analytics.collect_and_send_report(args, ret)
-
-        return ret
-    finally:
-        logger.setLevel(outerLogLevel)
-
-        from dvc.external_repo import clean_repos
-
-        # Remove cached repos in the end of the call, these are anonymous
-        # so won't be reused by any other subsequent run anyway.
-        clean_repos()

dvc/cli/command.py

@@ -1,36 +0,0 @@
-import logging
-import os
-from abc import ABC, abstractmethod
-
-logger = logging.getLogger(__name__)
-
-
-class CmdBase(ABC):
-    UNINITIALIZED = False
-
-    def __init__(self, args):
-        from dvc.repo import Repo
-
-        os.chdir(args.cd)
-
-        self.repo = Repo(uninitialized=self.UNINITIALIZED)
-        self.config = self.repo.config
-        self.args = args
-
-    def do_run(self):
-        with self.repo:
-            return self.run()
-
-    @abstractmethod
-    def run(self):
-        pass
-
-
-class CmdBaseNoRepo(CmdBase):
-    def __init__(self, args):  # pylint: disable=super-init-not-called
-        self.args = args
-
-        os.chdir(args.cd)
-
-    def do_run(self):
-        return self.run()

dvc/cli/completion.py

@@ -1,73 +0,0 @@
-import shtab
-
-BASH_PREAMBLE = """
-# $1=COMP_WORDS[1]
-_dvc_compgen_DVCFiles() {
-  compgen -d -S '/' -- $1  # recurse into subdirs
-  compgen -f -X '!*?.dvc' -- $1
-  compgen -f -X '!*Dvcfile' -- $1
-  compgen -f -X '!*dvc.yaml' -- $1
-}
-
-_dvc_compgen_stages() {
-    local _dvc_stages=($(dvc stage list -q --names-only))
-    compgen -W "${_dvc_stages[*]}" -- $1
-}
-_dvc_compgen_stages_and_files() {
-    _dvc_compgen_DVCFiles $1
-    _dvc_compgen_stages $1
-}
-
-_dvc_compgen_exps() {
-    local _dvc_exps=($(dvc exp list -q --all-commits --names-only))
-    compgen -W "${_dvc_exps[*]}" -- $1
-}
-"""
-
-ZSH_PREAMBLE = """
-_dvc_compadd_DVCFiles() {
-    _files -g '(*?.dvc|Dvcfile|dvc.yaml)'
-}
-_dvc_compadd_stages() {
-    # this will also show up the description of the stages
-    _describe 'stages' "($(_dvc_stages_output))"
-}
-
-_dvc_stages_output() {
-  dvc stage list -q | awk '{
-    # escape possible `:` on the stage name
-    sub(/:/, "\\\\\\\\:", $1);
-    # read all of the columns except the first
-    # reading `out` from $2, so as not to have a leading whitespace
-    out=$2; for(i=3;i<=NF;i++){out=out" "$i};
-    # print key, ":" and then single-quote the description
-    # colon is a delimiter used by `_describe` to separate field/description
-    print $1":""\\047"out"\\047"
-    # single quote -> \\047
-    }'
-}
-
-_dvc_compadd_stages_and_files() {
-    _dvc_compadd_DVCFiles
-    _dvc_compadd_stages
-}
-
-_dvc_compadd_exps() {
-    _describe 'experiments' "($(dvc exp list -q --all-commits --names-only))"
-}
-"""
-
-PREAMBLE = {
-    "bash": BASH_PREAMBLE,
-    "zsh": ZSH_PREAMBLE,
-}
-
-FILE = shtab.FILE
-DIR = shtab.DIRECTORY
-DVC_FILE = {"bash": "_dvc_compgen_DVCFiles", "zsh": "_dvc_compadd_DVCFiles"}
-STAGE = {"bash": "_dvc_compgen_stages", "zsh": "_dvc_compadd_stages"}
-DVCFILES_AND_STAGE = {
-    "bash": "_dvc_compgen_stages_and_files",
-    "zsh": "_dvc_compadd_stages_and_files",
-}
-EXPERIMENT = {"bash": "_dvc_compgen_exps", "zsh": "_dvc_compadd_exps"}

dvc/cli/utils.py

@@ -1,31 +0,0 @@
-def fix_subparsers(subparsers):
-    """Workaround for bug in Python 3. See more info at:
-    https://bugs.python.org/issue16308
-    https://github.com/iterative/dvc/issues/769
-
-    Args:
-        subparsers: subparsers to fix.
-    """
-    subparsers.required = True
-    subparsers.dest = "cmd"
-
-
-def append_doc_link(help_message, path):
-    from dvc.utils import format_link
-
-    if not path:
-        return help_message
-    doc_base = "https://man.dvc.org/"
-    return "{message}\nDocumentation: {link}".format(
-        message=help_message, link=format_link(doc_base + path)
-    )
-
-
-def fix_plumbing_subparsers(subparsers):
-    # metavar needs to be explicitly set in order to hide plumbing subcommands
-    # from the 'positional arguments' choices list
-    # see: https://bugs.python.org/issue22848
-    cmds = [
-        cmd for cmd, parser in subparsers.choices.items() if parser.add_help
-    ]
-    subparsers.metavar = "{{{}}}".format(",".join(cmds))

dvc/command/add.py

@@ -1,20 +1,15 @@
 import argparse
 import logging
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 
 logger = logging.getLogger(__name__)
 
 
 class CmdAdd(CmdBase):
     def run(self):
-        from dvc.exceptions import (
-            DvcException,
-            RecursiveAddingWhileUsingFilename,
-        )
-
         try:
             if len(self.args.targets) > 1 and self.args.file:
                 raise RecursiveAddingWhileUsingFilename()
@@ -27,10 +22,6 @@ class CmdAdd(CmdBase):
                 external=self.args.external,
                 glob=self.args.glob,
                 desc=self.args.desc,
-                out=self.args.out,
-                remote=self.args.remote,
-                to_remote=self.args.to_remote,
-                jobs=self.args.jobs,
             )
 
         except DvcException:
@@ -76,39 +67,9 @@ def add_parser(subparsers, parent_parser):
     )
     parser.add_argument(
         "--file",
-        help="Specify name of the .dvc file this command will generate.",
+        help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
     )
-    parser.add_argument(
-        "-o",
-        "--out",
-        help="Destination path to put files to.",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "--to-remote",
-        action="store_true",
-        default=False,
-        help="Download it directly to the remote",
-    )
-    parser.add_argument(
-        "-r",
-        "--remote",
-        help="Remote storage to download to",
-        metavar="<name>",
-    )
-    parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Only used along with '--to-remote'. "
-            "Number of jobs to run simultaneously "
-            "when pushing data to remote."
-            "The default value is 4 * cpu_count(). "
-        ),
-        metavar="<number>",
-    )
     parser.add_argument(
         "--desc",
         type=str,
@@ -119,6 +80,6 @@ def add_parser(subparsers, parent_parser):
         ),
     )
     parser.add_argument(
-        "targets", nargs="+", help="Input files/directories to add."
+        "targets", nargs="+", help="Input files/directories to add.",
     ).complete = completion.FILE
     parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -0,0 +1,65 @@
+import logging
+import os
+from abc import ABC, abstractmethod
+
+logger = logging.getLogger(__name__)
+
+
+def fix_subparsers(subparsers):
+    """Workaround for bug in Python 3. See more info at:
+        https://bugs.python.org/issue16308
+        https://github.com/iterative/dvc/issues/769
+
+        Args:
+            subparsers: subparsers to fix.
+    """
+    subparsers.required = True
+    subparsers.dest = "cmd"
+
+
+def append_doc_link(help_message, path):
+    from dvc.utils import format_link
+
+    if not path:
+        return help_message
+    doc_base = "https://man.dvc.org/"
+    return "{message}\nDocumentation: {link}".format(
+        message=help_message, link=format_link(doc_base + path)
+    )
+
+
+class CmdBase(ABC):
+    UNINITIALIZED = False
+
+    def __init__(self, args):
+        from dvc.repo import Repo
+        from dvc.updater import Updater
+
+        os.chdir(args.cd)
+
+        self.repo = Repo(uninitialized=self.UNINITIALIZED)
+        self.config = self.repo.config
+        self.args = args
+        if self.repo.tmp_dir:
+            hardlink_lock = self.config["core"].get("hardlink_lock", False)
+            updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
+            updater.check()
+
+    @property
+    def default_targets(self):
+        """Default targets for `dvc repro`."""
+        from dvc.dvcfile import PIPELINE_FILE
+
+        logger.debug(f"assuming default target '{PIPELINE_FILE}'.")
+        return [PIPELINE_FILE]
+
+    @abstractmethod
+    def run(self):
+        pass
+
+
+class CmdBaseNoRepo(CmdBase):
+    def __init__(self, args):  # pylint: disable=super-init-not-called
+        self.args = args
+
+        os.chdir(args.cd)

dvc/command/cache.py

@@ -1,34 +1,25 @@
 import argparse
+import logging
 
-from dvc.cli import completion
-from dvc.cli.utils import append_doc_link, fix_subparsers
-from dvc.commands.config import CmdConfig
-from dvc.ui import ui
+from dvc.command import completion
+from dvc.command.base import append_doc_link, fix_subparsers
+from dvc.command.config import CmdConfig
+
+logger = logging.getLogger(__name__)
 
 
 class CmdCacheDir(CmdConfig):
     def run(self):
         if self.args.value is None and not self.args.unset:
-            if self.args.level:
-                conf = self.config.read(level=self.args.level)
-            else:
-                # Use merged config with default values
-                conf = self.config
-            self._check(conf, False, "cache", "dir")
-            ui.write(conf["cache"]["dir"])
+            logger.info(self.config["cache"]["dir"])
             return 0
-        with self.config.edit(level=self.args.level) as conf:
-            if self.args.unset:
-                self._check(conf, False, "cache", "dir")
-                del conf["cache"]["dir"]
-            else:
-                self._check(conf, False, "cache")
-                conf["cache"]["dir"] = self.args.value
+        with self.config.edit(level=self.args.level) as edit:
+            edit["cache"]["dir"] = self.args.value
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    from dvc.commands.config import parent_config_parser
+    from dvc.command.config import parent_config_parser
 
     CACHE_HELP = "Manage cache settings."
 

dvc/command/check_ignore.py

@@ -1,15 +1,17 @@
 import argparse
+import logging
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.ui import ui
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
 
 
 class CmdCheckIgnore(CmdBase):
     def __init__(self, args):
         super().__init__(args)
-        self.ignore_filter = self.repo.dvcignore
+        self.ignore_filter = self.repo.tree.dvcignore
 
     def _show_results(self, result):
         if not result.match and not self.args.non_matching:
@@ -21,9 +23,9 @@ class CmdCheckIgnore(CmdBase):
                 patterns = patterns[-1:]
 
             for pattern in patterns:
-                ui.write(pattern, result.file, sep="\t")
+                logger.info("{}\t{}".format(pattern, result.file))
         else:
-            ui.write(result.file)
+            logger.info(result.file)
 
     def _check_one_file(self, target):
         result = self.ignore_filter.check_ignore(target)
@@ -53,8 +55,6 @@ class CmdCheckIgnore(CmdBase):
         return ret
 
     def _check_args(self):
-        from dvc.exceptions import DvcException
-
         if not self.args.stdin and not self.args.targets:
             raise DvcException("`targets` or `--stdin` needed")
 
@@ -121,6 +121,6 @@ def add_parser(subparsers, parent_parser):
         help="Read paths from standard input instead of providing `targets`.",
     )
     parser.add_argument(
-        "targets", nargs="*", help="File or directory paths to check"
+        "targets", nargs="*", help="File or directory paths to check",
     ).complete = completion.FILE
     parser.set_defaults(func=CmdCheckIgnore)

dvc/command/checkout.py

@@ -1,36 +1,44 @@
 import argparse
+import logging
 import operator
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+import colorama
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import CheckoutError
-from dvc.ui import ui
+from dvc.utils.humanize import get_summary
+
+logger = logging.getLogger(__name__)
 
 
 def log_changes(stats):
-    colors = {
-        "modified": "yellow",
-        "added": "green",
-        "deleted": "red",
-    }
+    colors = [
+        ("modified", colorama.Fore.YELLOW,),
+        ("added", colorama.Fore.GREEN),
+        ("deleted", colorama.Fore.RED,),
+    ]
 
-    for state, color in colors.items():
+    for state, color in colors:
         entries = stats.get(state)
 
         if not entries:
             continue
 
         for entry in entries:
-            ui.write(
-                f"[{color}]{state[0].upper()}", entry, styled=True, sep="\t"
+            logger.info(
+                "{color}{state}{nc}{spacing}{entry}".format(
+                    color=color,
+                    state=state[0].upper(),
+                    nc=colorama.Fore.RESET,
+                    spacing="\t",
+                    entry=entry,
+                )
             )
 
 
 class CmdCheckout(CmdBase):
     def run(self):
-        from dvc.utils.humanize import get_summary
-
         stats, exc = None, None
         try:
             stats = self.repo.checkout(
@@ -49,16 +57,12 @@ class CmdCheckout(CmdBase):
             msg = get_summary(
                 sorted(stats.items(), key=operator.itemgetter(0))
             )
-            ui.write(msg or default_message)
+            logger.info(msg or default_message)
         else:
             log_changes(stats)
 
         if exc:
             raise exc
-
-        if self.args.relink:
-            msg = "Relinked successfully"
-            ui.write(msg)
         return 0
 
 

dvc/command/commit.py

@@ -1,17 +1,15 @@
 import argparse
 import logging
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
 
 class CmdCommit(CmdBase):
     def run(self):
-        from dvc.exceptions import DvcException
-
         if not self.args.targets:
             self.args.targets = [None]
 
@@ -34,10 +32,7 @@ class CmdCommit(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    COMMIT_HELP = (
-        "Record changes to files or directories tracked by DVC"
-        " by storing the current versions in the cache."
-    )
+    COMMIT_HELP = "Save changed data to cache and update DVC-files."
 
     commit_parser = subparsers.add_parser(
         "commit",
@@ -70,7 +65,7 @@ def add_parser(subparsers, parent_parser):
     commit_parser.add_argument(
         "targets",
         nargs="*",
-        help="stages or .dvc files to commit. Optional. "
-        "(Finds all DVC files in the workspace by default.)",
-    ).complete = completion.DVCFILES_AND_STAGE
+        help="DVC-files to commit. Optional. "
+        "(Finds all DVC-files in the workspace by default.)",
+    ).complete = completion.DVC_FILE
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -1,22 +1,39 @@
 import argparse
 import logging
 
-from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.completion import PREAMBLE
-from dvc.cli.utils import append_doc_link
-from dvc.ui import ui
+import shtab
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
+FILE = shtab.FILE
+DIR = shtab.DIRECTORY
+DVC_FILE = {
+    "bash": "_dvc_compgen_DVCFiles",
+    "zsh": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'",
+}
+PREAMBLE = {
+    "bash": """
+# $1=COMP_WORDS[1]
+_dvc_compgen_DVCFiles() {
+  compgen -d -S '/' -- $1  # recurse into subdirs
+  compgen -f -X '!*?.dvc' -- $1
+  compgen -f -X '!*Dvcfile' -- $1
+  compgen -f -X '!*dvc.yaml' -- $1
+}
+""",
+    "zsh": "",
+}
 
 
 class CmdCompletion(CmdBaseNoRepo):
     def run(self):
-        import shtab
+        from dvc.cli import get_main_parser
 
+        parser = get_main_parser()
         shell = self.args.shell
-        parser = self.args.parser
         script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
-        ui.write(script, force=True)
+        print(script)
         return 0
 
 

dvc/command/config.py

@@ -0,0 +1,97 @@
+import argparse
+import logging
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.config import Config, ConfigError
+
+logger = logging.getLogger(__name__)
+
+
+class CmdConfig(CmdBaseNoRepo):
+    def __init__(self, args):
+        super().__init__(args)
+
+        self.config = Config(validate=False)
+
+    def run(self):
+        section, opt = self.args.name.lower().strip().split(".", 1)
+
+        if self.args.value is None and not self.args.unset:
+            conf = self.config.load_one(self.args.level)
+            self._check(conf, section, opt)
+            logger.info(conf[section][opt])
+            return 0
+
+        with self.config.edit(self.args.level) as conf:
+            if self.args.unset:
+                self._check(conf, section, opt)
+                del conf[section][opt]
+            else:
+                self._check(conf, section)
+                conf[section][opt] = self.args.value
+
+        if self.args.name == "cache.type":
+            logger.warning(
+                "You have changed the 'cache.type' option. This doesn't update"
+                " any existing workspace file links, but it can be done with:"
+                "\n             dvc checkout --relink"
+            )
+
+        return 0
+
+    def _check(self, conf, section, opt=None):
+        if section not in conf:
+            msg = "section {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
+
+        if opt and opt not in conf[section]:
+            msg = "option {} doesn't exist"
+            raise ConfigError(msg.format(self.args.name))
+
+
+parent_config_parser = argparse.ArgumentParser(add_help=False)
+level_group = parent_config_parser.add_mutually_exclusive_group()
+level_group.add_argument(
+    "--global",
+    dest="level",
+    action="store_const",
+    const="global",
+    help="Use global config.",
+)
+level_group.add_argument(
+    "--system",
+    dest="level",
+    action="store_const",
+    const="system",
+    help="Use system config.",
+)
+level_group.add_argument(
+    "--local",
+    dest="level",
+    action="store_const",
+    const="local",
+    help="Use local config.",
+)
+parent_config_parser.set_defaults(level="repo")
+
+
+def add_parser(subparsers, parent_parser):
+    CONFIG_HELP = "Get or set config options."
+
+    config_parser = subparsers.add_parser(
+        "config",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(CONFIG_HELP, "config"),
+        help=CONFIG_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    config_parser.add_argument(
+        "-u",
+        "--unset",
+        default=False,
+        action="store_true",
+        help="Unset option.",
+    )
+    config_parser.add_argument("name", help="Option name.")
+    config_parser.add_argument("value", nargs="?", help="Option value.")
+    config_parser.set_defaults(func=CmdConfig)

dvc/command/daemon.py

@@ -1,6 +1,5 @@
-from dvc.cli import completion
-from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import fix_subparsers
+from dvc.command import completion
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers
 
 
 class CmdDaemonBase(CmdBaseNoRepo):
@@ -68,6 +67,6 @@ def add_parser(subparsers, parent_parser):
         help=DAEMON_ANALYTICS_HELP,
     )
     daemon_analytics_parser.add_argument(
-        "target", help="Analytics file."
+        "target", help="Analytics file.",
     ).complete = completion.FILE
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/dag.py

@@ -1,8 +1,10 @@
 import argparse
+import logging
 
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.ui import ui
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.stage import Stage
+
+logger = logging.getLogger(__name__)
 
 
 def _show_ascii(G):
@@ -24,36 +26,10 @@ def _show_dot(G):
     from networkx.drawing.nx_pydot import write_dot
 
     dot_file = io.StringIO()
-    write_dot(G.reverse(), dot_file)
+    write_dot(G, dot_file)
     return dot_file.getvalue()
 
 
-def _show_mermaid(G, markdown: bool = False):
-    from dvc.repo.graph import get_pipelines
-
-    pipelines = get_pipelines(G)
-
-    graph = "flowchart TD"
-
-    total_nodes = 0
-    for pipeline in pipelines:
-        node_ids = {}
-        nodes = sorted(str(x) for x in pipeline.nodes)
-        for node in nodes:
-            total_nodes += 1
-            node_id = f"node{total_nodes}"
-            graph += f'\n\t{node_id}["{node}"]'
-            node_ids[node] = node_id
-        edges = sorted((str(a), str(b)) for b, a in pipeline.edges)
-        for a, b in edges:
-            graph += f"\n\t{node_ids[str(a)]}-->{node_ids[str(b)]}"
-
-    if markdown:
-        return f"```mermaid\n{graph}\n```"
-
-    return graph
-
-
 def _collect_targets(repo, target, outs):
     if not target:
         return []
@@ -63,30 +39,24 @@ def _collect_targets(repo, target, outs):
         return [stage.addressing for stage, _ in pairs]
 
     targets = []
-
-    outs_trie = repo.index.outs_trie
-    for stage, path in pairs:
-        if not path:
+    for stage, info in pairs:
+        if not info:
             targets.extend([str(out) for out in stage.outs])
             continue
 
-        for out in outs_trie.itervalues(
-            prefix=repo.fs.path.parts(path)
-        ):  # noqa: B301
+        for out in repo.outs_trie.itervalues(prefix=info.parts):  # noqa: B301
             targets.extend(str(out))
 
     return targets
 
 
-def _transform(index, outs):
+def _transform(repo, outs):
     import networkx as nx
 
-    from dvc.stage import Stage
-
     def _relabel(node) -> str:
         return node.addressing if isinstance(node, Stage) else str(node)
 
-    G = index.outs_graph if outs else index.graph
+    G = repo.outs_graph if outs else repo.graph
     return nx.relabel_nodes(G, _relabel, copy=True)
 
 
@@ -116,7 +86,7 @@ def _filter(G, targets, full):
 
 def _build(repo, target=None, full=False, outs=False):
     targets = _collect_targets(repo, target, outs)
-    G = _transform(repo.index, outs)
+    G = _transform(repo, outs)
     return _filter(G, targets, full)
 
 
@@ -130,12 +100,11 @@ class CmdDAG(CmdBase):
         )
 
         if self.args.dot:
-            ui.write(_show_dot(G))
-        elif self.args.mermaid or self.args.markdown:
-            ui.write(_show_mermaid(G, self.args.markdown))
+            logger.info(_show_dot(G))
         else:
-            with ui.pager():
-                ui.write(_show_ascii(G))
+            from dvc.utils.pager import pager
+
+            pager(_show_ascii(G))
 
         return 0
 
@@ -155,20 +124,6 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Print DAG with .dot format.",
     )
-    dag_parser.add_argument(
-        "--mermaid",
-        action="store_true",
-        default=False,
-        help="Print DAG with mermaid format.",
-    )
-    dag_parser.add_argument(
-        "--md",
-        "--show-md",
-        action="store_true",
-        default=False,
-        dest="markdown",
-        help="Print DAG with mermaid format wrapped in Markdown block.",
-    )
     dag_parser.add_argument(
         "--full",
         action="store_true",

dvc/command/data_sync.py

@@ -1,32 +1,27 @@
 import argparse
 import logging
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.checkout import log_changes
+from dvc.exceptions import CheckoutError, DvcException
+from dvc.utils.humanize import get_summary
 
 logger = logging.getLogger(__name__)
 
 
 class CmdDataBase(CmdBase):
     def log_summary(self, stats):
-        from dvc.ui import ui
-        from dvc.utils.humanize import get_summary
-
         default_msg = "Everything is up to date."
-        ui.write(get_summary(stats.items()) or default_msg)
+        logger.info(get_summary(stats.items()) or default_msg)
 
 
 class CmdDataPull(CmdDataBase):
     def log_summary(self, stats):
-        from dvc.commands.checkout import log_changes
-
         log_changes(stats)
         super().log_summary(stats)
 
     def run(self):
-        from dvc.exceptions import CheckoutError, DvcException
-
         try:
             stats = self.repo.pull(
                 targets=self.args.targets,
@@ -39,7 +34,6 @@ class CmdDataPull(CmdDataBase):
                 force=self.args.force,
                 recursive=self.args.recursive,
                 run_cache=self.args.run_cache,
-                glob=self.args.glob,
             )
             self.log_summary(stats)
         except (CheckoutError, DvcException) as exc:
@@ -52,8 +46,6 @@ class CmdDataPull(CmdDataBase):
 
 class CmdDataPush(CmdDataBase):
     def run(self):
-        from dvc.exceptions import DvcException
-
         try:
             processed_files_count = self.repo.push(
                 targets=self.args.targets,
@@ -65,7 +57,6 @@ class CmdDataPush(CmdDataBase):
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
                 run_cache=self.args.run_cache,
-                glob=self.args.glob,
             )
             self.log_summary({"pushed": processed_files_count})
         except DvcException:
@@ -76,8 +67,6 @@ class CmdDataPush(CmdDataBase):
 
 class CmdDataFetch(CmdDataBase):
     def run(self):
-        from dvc.exceptions import DvcException
-
         try:
             processed_files_count = self.repo.fetch(
                 targets=self.args.targets,
@@ -98,7 +87,7 @@ class CmdDataFetch(CmdDataBase):
 
 
 def shared_parent_parser():
-    from dvc.cli.parser import get_parent_parser
+    from dvc.cli import get_parent_parser
 
     # Parent parser used in pull/push/status
     parent_parser = argparse.ArgumentParser(
@@ -111,6 +100,7 @@ def shared_parent_parser():
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )
@@ -127,7 +117,7 @@ def shared_parent_parser():
 
 
 def add_parser(subparsers, _parent_parser):
-    from dvc.commands.status import CmdDataStatus
+    from dvc.command.status import CmdDataStatus
 
     # Pull
     PULL_HELP = "Download tracked files or directories from remote storage."
@@ -140,7 +130,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pull_parser.add_argument(
-        "-r", "--remote", help="Remote storage to pull from", metavar="<name>"
+        "-r", "--remote", help="Remote storage to pull from", metavar="<name>",
     )
     pull_parser.add_argument(
         "-a",
@@ -157,7 +147,6 @@ def add_parser(subparsers, _parent_parser):
         help="Fetch cache for all tags.",
     )
     pull_parser.add_argument(
-        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -190,12 +179,6 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch run history for all stages.",
     )
-    pull_parser.add_argument(
-        "--glob",
-        action="store_true",
-        default=False,
-        help="Pull cache for targets matching shell-style wildcards.",
-    )
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
@@ -209,7 +192,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     push_parser.add_argument(
-        "-r", "--remote", help="Remote storage to push to", metavar="<name>"
+        "-r", "--remote", help="Remote storage to push to", metavar="<name>",
     )
     push_parser.add_argument(
         "-a",
@@ -226,7 +209,6 @@ def add_parser(subparsers, _parent_parser):
         help="Push cache for all tags.",
     )
     push_parser.add_argument(
-        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -252,18 +234,10 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Push run history for all stages.",
     )
-    push_parser.add_argument(
-        "--glob",
-        action="store_true",
-        default=False,
-        help="Allows targets containing shell-style wildcards.",
-    )
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
-    FETCH_HELP = (
-        "Download files or directories from remote storage to the cache."
-    )
+    FETCH_HELP = "Download cached files or directories from remote storage."
 
     fetch_parser = subparsers.add_parser(
         "fetch",
@@ -273,7 +247,10 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     fetch_parser.add_argument(
-        "-r", "--remote", help="Remote storage to fetch from", metavar="<name>"
+        "-r",
+        "--remote",
+        help="Remote storage to fetch from",
+        metavar="<name>",
     )
     fetch_parser.add_argument(
         "-a",
@@ -290,7 +267,6 @@ def add_parser(subparsers, _parent_parser):
         help="Fetch cache for all tags.",
     )
     fetch_parser.add_argument(
-        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -371,7 +347,6 @@ def add_parser(subparsers, _parent_parser):
         "for all tags.",
     )
     status_parser.add_argument(
-        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -393,7 +368,6 @@ def add_parser(subparsers, _parent_parser):
         help="Show status of all stages in the specified directory.",
     )
     status_parser.add_argument(
-        "--json",
         "--show-json",
         action="store_true",
         default=False,

dvc/command/destroy.py

@@ -1,17 +1,15 @@
 import argparse
 import logging
 
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+import dvc.prompt as prompt
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
 
 class CmdDestroy(CmdBase):
     def run(self):
-        from dvc.exceptions import DvcException
-        from dvc.ui import ui
-
         try:
             statement = (
                 "This will destroy all information about your pipelines,"
@@ -20,7 +18,7 @@ class CmdDestroy(CmdBase):
                 "Are you sure you want to continue?"
             )
 
-            if not self.args.force and not ui.confirm(statement):
+            if not self.args.force and not prompt.confirm(statement):
                 raise DvcException(
                     "cannot destroy without a confirmation from the user."
                     " Use `-f` to force."
@@ -34,7 +32,7 @@ class CmdDestroy(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    DESTROY_HELP = "Remove DVC files, local DVC config and data cache."
+    DESTROY_HELP = "Remove DVC-files, local DVC config and data cache."
 
     destroy_parser = subparsers.add_parser(
         "destroy",

dvc/command/diff.py

@@ -1,11 +1,12 @@
 import argparse
+import json
 import logging
 import os
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.ui import ui
+import colorama
+
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
@@ -16,33 +17,32 @@ def _digest(checksum):
     return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
 
 
-def _show_markdown(diff, show_hash=False, hide_missing=False):
-    headers = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
+def _show_md(diff, show_hash=False, hide_missing=False):
+    from dvc.utils.diff import table
+
+    header = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
     rows = []
-    statuses = ["added", "deleted", "renamed", "modified"]
+    statuses = ["added", "deleted", "modified"]
     if not hide_missing:
         statuses.append("not in cache")
-
     for status in statuses:
         entries = diff.get(status, [])
         if not entries:
             continue
         for entry in entries:
             path = entry["path"]
-            if isinstance(path, dict):
-                path = f"{path['old']} -> {path['new']}"
             if show_hash:
                 check_sum = _digest(entry.get("hash", ""))
                 rows.append([status, check_sum, path])
             else:
                 rows.append([status, path])
 
-    ui.table(rows, headers=headers, markdown=True)
+    return table(header, rows, True)
 
 
 class CmdDiff(CmdBase):
     @staticmethod
-    def _show_diff(diff, hide_missing=False):
+    def _format(diff, hide_missing=False):
         """
         Given a diff structure, generate a string of paths separated
         by new lines and grouped together by their state.
@@ -69,16 +69,16 @@ class CmdDiff(CmdBase):
         """
 
         colors = {
-            "added": "green",
-            "modified": "yellow",
-            "deleted": "red",
-            "renamed": "green",
-            "not in cache": "yellow",
+            "added": colorama.Fore.GREEN,
+            "modified": colorama.Fore.YELLOW,
+            "deleted": colorama.Fore.RED,
+            "not in cache": colorama.Fore.YELLOW,
         }
 
         summary = {}
+        groups = []
 
-        states = ["added", "deleted", "renamed", "modified"]
+        states = ["added", "deleted", "modified"]
         if not hide_missing:
             states.append("not in cache")
         for state in states:
@@ -88,66 +88,66 @@ class CmdDiff(CmdBase):
             if not entries:
                 continue
 
-            header = state.capitalize()
-            ui.write(f"[{colors[state]}]{header}[/]:", styled=True)
+            content = []
 
             for entry in entries:
                 path = entry["path"]
-                if isinstance(path, dict):
-                    path = f"{path['old']} -> {path['new']}"
                 checksum = entry.get("hash")
                 summary[state] += 1 if not path.endswith(os.sep) else 0
-                ui.write(
+                content.append(
                     "{space}{checksum}{separator}{path}".format(
                         space="    ",
                         checksum=_digest(checksum) if checksum else "",
                         separator="  " if checksum else "",
-                        path=path,
+                        path=entry["path"],
                     )
                 )
 
-            ui.write()
+            groups.append(
+                "{color}{header}{nc}:\n{content}".format(
+                    color=colors[state],
+                    header=state.capitalize(),
+                    nc=colorama.Fore.RESET,
+                    content="\n".join(content),
+                )
+            )
 
         if not sum(summary.values()):
             return None
 
-        states_summary = ", ".join(
-            f"{summary[state]} {state}"
-            for state in states
-            if summary[state] > 0
+        fmt = (
+            "files summary: {added} added, {deleted} deleted,"
+            " {modified} modified"
         )
-        ui.write("files summary:", states_summary)
+        if not hide_missing:
+            fmt += ", {not in cache} not in cache"
+        groups.append(fmt.format_map(summary))
+
+        return "\n\n".join(groups)
 
     def run(self):
-        from dvc.exceptions import DvcException
-
         try:
-            diff = self.repo.diff(
-                self.args.a_rev, self.args.b_rev, self.args.targets
-            )
+            diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
             show_hash = self.args.show_hash
             hide_missing = self.args.b_rev or self.args.hide_missing
             if hide_missing:
-                diff.pop("not in cache", None)
+                del diff["not in cache"]
 
             for key, entries in diff.items():
-                entries = sorted(
-                    entries,
-                    key=lambda entry: entry["path"]["old"]
-                    if isinstance(entry["path"], dict)
-                    else entry["path"],
-                )
+                entries = sorted(entries, key=lambda entry: entry["path"])
                 if not show_hash:
                     for entry in entries:
                         del entry["hash"]
                 diff[key] = entries
 
-            if self.args.json:
-                ui.write_json(diff)
-            elif self.args.markdown:
-                _show_markdown(diff, show_hash, hide_missing)
+            if self.args.show_json:
+                logger.info(json.dumps(diff))
+            elif self.args.show_md:
+                logger.info(_show_md(diff, show_hash, hide_missing))
             elif diff:
-                self._show_diff(diff, hide_missing)
+                output = self._format(diff, hide_missing)
+                if output:
+                    logger.info(output)
 
         except DvcException:
             logger.exception("failed to get diff")
@@ -167,15 +167,6 @@ def add_parser(subparsers, parent_parser):
         help=DIFF_DESCRIPTION,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    diff_parser.add_argument(
-        "--targets",
-        nargs="*",
-        help=(
-            "Specific DVC-tracked files to compare. "
-            "Accepts one or more file paths."
-        ),
-        metavar="<paths>",
-    ).complete = completion.FILE
     diff_parser.add_argument(
         "a_rev",
         help="Old Git commit to compare (defaults to HEAD)",
@@ -188,7 +179,6 @@ def add_parser(subparsers, parent_parser):
         nargs="?",
     )
     diff_parser.add_argument(
-        "--json",
         "--show-json",
         help="Format the output into a JSON",
         action="store_true",
@@ -201,11 +191,9 @@ def add_parser(subparsers, parent_parser):
         default=False,
     )
     diff_parser.add_argument(
-        "--md",
         "--show-md",
         help="Show tabulated output in the Markdown format (GFM).",
         action="store_true",
-        dest="markdown",
         default=False,
     )
     diff_parser.add_argument(

dvc/command/experiments.py

@@ -0,0 +1,903 @@
+import argparse
+import io
+import logging
+import os
+from collections import OrderedDict
+from collections.abc import Mapping
+from datetime import date, datetime
+from itertools import groupby
+from typing import Iterable, Optional
+
+import dvc.prompt as prompt
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.command.metrics import DEFAULT_PRECISION
+from dvc.command.repro import CmdRepro
+from dvc.command.repro import add_arguments as add_repro_arguments
+from dvc.exceptions import DvcException, InvalidArgumentError
+from dvc.repo.experiments import Experiments
+from dvc.scm.git import Git
+from dvc.utils.flatten import flatten
+
+logger = logging.getLogger(__name__)
+
+
+def _filter_names(
+    names: Iterable,
+    label: str,
+    include: Optional[Iterable],
+    exclude: Optional[Iterable],
+):
+    if include and exclude:
+        intersection = set(include) & set(exclude)
+        if intersection:
+            values = ", ".join(intersection)
+            raise InvalidArgumentError(
+                f"'{values}' specified in both --include-{label} and"
+                f" --exclude-{label}"
+            )
+
+    names = [tuple(name.split(".")) for name in names]
+
+    def _filter(filters, update_func):
+        filters = [tuple(name.split(".")) for name in filters]
+        for length, groups in groupby(filters, len):
+            for group in groups:
+                matches = [name for name in names if name[:length] == group]
+                if not matches:
+                    name = ".".join(group)
+                    raise InvalidArgumentError(
+                        f"'{name}' does not match any known {label}"
+                    )
+                update_func({match: None for match in matches})
+
+    if include:
+        ret: OrderedDict = OrderedDict()
+        _filter(include, ret.update)
+    else:
+        ret = OrderedDict({name: None for name in names})
+
+    if exclude:
+        _filter(exclude, ret.difference_update)  # type: ignore[attr-defined]
+
+    return [".".join(name) for name in ret]
+
+
+def _update_names(names, items):
+    for name, item in items:
+        if isinstance(item, dict):
+            item = flatten(item)
+            names.update(item.keys())
+        else:
+            names[name] = None
+
+
+def _collect_names(all_experiments, **kwargs):
+    metric_names = set()
+    param_names = set()
+
+    for _, experiments in all_experiments.items():
+        for exp in experiments.values():
+            _update_names(metric_names, exp.get("metrics", {}).items())
+            _update_names(param_names, exp.get("params", {}).items())
+
+    metric_names = _filter_names(
+        sorted(metric_names),
+        "metrics",
+        kwargs.get("include_metrics"),
+        kwargs.get("exclude_metrics"),
+    )
+    param_names = _filter_names(
+        sorted(param_names),
+        "params",
+        kwargs.get("include_params"),
+        kwargs.get("exclude_params"),
+    )
+
+    return metric_names, param_names
+
+
+def _collect_rows(
+    base_rev,
+    experiments,
+    metric_names,
+    param_names,
+    precision=DEFAULT_PRECISION,
+    no_timestamp=False,
+    sort_by=None,
+    sort_order=None,
+):
+    if sort_by:
+        if sort_by in metric_names:
+            sort_type = "metrics"
+        elif sort_by in param_names:
+            sort_type = "params"
+        else:
+            raise InvalidArgumentError(f"Unknown sort column '{sort_by}'")
+        reverse = sort_order == "desc"
+        experiments = _sort_exp(experiments, sort_by, sort_type, reverse)
+
+    new_checkpoint = True
+    for i, (rev, exp) in enumerate(experiments.items()):
+        row = []
+        style = None
+        queued = "*" if exp.get("queued", False) else ""
+
+        tip = exp.get("checkpoint_tip")
+        parent = ""
+        if rev == "baseline":
+            if Git.is_sha(base_rev):
+                name_rev = base_rev[:7]
+            else:
+                name_rev = base_rev
+            name = exp.get("name", name_rev)
+            row.append(f"{name}")
+            style = "bold"
+        else:
+            if tip:
+                parent_rev = exp.get("checkpoint_parent", "")
+                parent_exp = experiments.get(parent_rev, {})
+                parent_tip = parent_exp.get("checkpoint_tip")
+                if tip == parent_tip:
+                    if new_checkpoint:
+                        tree = "│ ╓"
+                    else:
+                        tree = "│ ╟"
+                    new_checkpoint = False
+                else:
+                    if parent_rev == base_rev:
+                        tree = "├─╨"
+                    else:
+                        tree = "│ ╟"
+                        parent = f" ({parent_rev[:7]})"
+                    new_checkpoint = True
+            else:
+                if i < len(experiments) - 1:
+                    tree = "├──"
+                else:
+                    tree = "└──"
+                new_checkpoint = True
+            name = exp.get("name", rev[:7])
+            row.append(f"{tree} {queued}{name}{parent}")
+
+        if not no_timestamp:
+            row.append(_format_time(exp.get("timestamp")))
+
+        _extend_row(
+            row, metric_names, exp.get("metrics", {}).items(), precision
+        )
+        _extend_row(row, param_names, exp.get("params", {}).items(), precision)
+
+        yield row, style
+
+
+def _sort_exp(experiments, sort_by, typ, reverse):
+    def _sort(item):
+        rev, exp = item
+        tip = exp.get("checkpoint_tip")
+        if tip and tip != rev:
+            # Sort checkpoint experiments by tip commit
+            return _sort((tip, experiments[tip]))
+        for fname, item in exp.get(typ, {}).items():
+            if isinstance(item, dict):
+                item = flatten(item)
+            else:
+                item = {fname: item}
+            if sort_by in item:
+                val = item[sort_by]
+                return (val is None, val)
+        return (True, None)
+
+    ret = OrderedDict()
+    if "baseline" in experiments:
+        ret["baseline"] = experiments.pop("baseline")
+
+    ret.update(sorted(experiments.items(), key=_sort, reverse=reverse))
+    return ret
+
+
+def _format_time(timestamp):
+    if timestamp is None:
+        return "-"
+    if timestamp.date() == date.today():
+        fmt = "%I:%M %p"
+    else:
+        fmt = "%b %d, %Y"
+    return timestamp.strftime(fmt)
+
+
+def _format_field(val, precision=DEFAULT_PRECISION):
+    if isinstance(val, float):
+        fmt = f"{{:.{precision}g}}"
+        return fmt.format(val)
+    elif isinstance(val, Mapping):
+        return {k: _format_field(v) for k, v in val.items()}
+    elif isinstance(val, list):
+        return [_format_field(x) for x in val]
+    return str(val)
+
+
+def _extend_row(row, names, items, precision):
+    from rich.text import Text
+
+    if not items:
+        row.extend(["-"] * len(names))
+        return
+
+    for fname, item in items:
+        if isinstance(item, dict):
+            item = flatten(item)
+        else:
+            item = {fname: item}
+        for name in names:
+            if name in item:
+                value = item[name]
+                if value is None:
+                    text = "-"
+                else:
+                    # wrap field data in rich.Text, otherwise rich may
+                    # interpret unescaped braces from list/dict types as rich
+                    # markup tags
+                    text = Text(str(_format_field(value, precision)))
+                row.append(text)
+            else:
+                row.append("-")
+
+
+def _parse_list(param_list):
+    ret = []
+    for param_str in param_list:
+        # we don't care about filename prefixes for show, silently
+        # ignore it if provided to keep usage consistent with other
+        # metric/param list command options
+        _, _, param_str = param_str.rpartition(":")
+        ret.extend(param_str.split(","))
+    return ret
+
+
+def _show_experiments(all_experiments, console, **kwargs):
+    from rich.table import Table
+
+    include_metrics = _parse_list(kwargs.pop("include_metrics", []))
+    exclude_metrics = _parse_list(kwargs.pop("exclude_metrics", []))
+    include_params = _parse_list(kwargs.pop("include_params", []))
+    exclude_params = _parse_list(kwargs.pop("exclude_params", []))
+
+    metric_names, param_names = _collect_names(
+        all_experiments,
+        include_metrics=include_metrics,
+        exclude_metrics=exclude_metrics,
+        include_params=include_params,
+        exclude_params=exclude_params,
+    )
+
+    table = Table()
+    table.add_column("Experiment", no_wrap=True)
+    if not kwargs.get("no_timestamp", False):
+        table.add_column("Created")
+    for name in metric_names:
+        table.add_column(name, justify="right", no_wrap=True)
+    for name in param_names:
+        table.add_column(name, justify="left")
+
+    for base_rev, experiments in all_experiments.items():
+        for row, _, in _collect_rows(
+            base_rev, experiments, metric_names, param_names, **kwargs,
+        ):
+            table.add_row(*row)
+
+    console.print(table)
+
+
+def _format_json(item):
+    if isinstance(item, (date, datetime)):
+        return item.isoformat()
+    raise TypeError
+
+
+class CmdExperimentsShow(CmdBase):
+    def run(self):
+        from rich.console import Console
+
+        from dvc.utils.pager import pager
+
+        if not self.repo.experiments:
+            return 0
+
+        try:
+            all_experiments = self.repo.experiments.show(
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
+                sha_only=self.args.sha,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(all_experiments, default=_format_json))
+                return 0
+
+            if self.args.no_pager:
+                console = Console()
+            else:
+                # Note: rich does not currently include a native way to force
+                # infinite width for use with a pager
+                console = Console(
+                    file=io.StringIO(), force_terminal=True, width=9999
+                )
+
+            if self.args.precision is None:
+                precision = DEFAULT_PRECISION
+            else:
+                precision = self.args.precision
+
+            _show_experiments(
+                all_experiments,
+                console,
+                include_metrics=self.args.include_metrics,
+                exclude_metrics=self.args.exclude_metrics,
+                include_params=self.args.include_params,
+                exclude_params=self.args.exclude_params,
+                no_timestamp=self.args.no_timestamp,
+                sort_by=self.args.sort_by,
+                sort_order=self.args.sort_order,
+                precision=precision,
+            )
+
+            if not self.args.no_pager:
+                pager(console.file.getvalue())
+        except DvcException:
+            logger.exception("failed to show experiments")
+            return 1
+
+        return 0
+
+
+class CmdExperimentsApply(CmdBase):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        self.repo.experiments.apply(self.args.experiment)
+
+        return 0
+
+
+def _show_diff(
+    diff,
+    title="",
+    markdown=False,
+    no_path=False,
+    old=False,
+    precision=DEFAULT_PRECISION,
+):
+    from dvc.utils.diff import table
+
+    rows = []
+    for fname, diff_ in diff.items():
+        sorted_diff = OrderedDict(sorted(diff_.items()))
+        for item, change in sorted_diff.items():
+            row = [] if no_path else [fname]
+            row.append(item)
+            if old:
+                row.append(_format_field(change.get("old"), precision))
+            row.append(_format_field(change["new"], precision))
+            row.append(
+                _format_field(
+                    change.get("diff", "diff not supported"), precision
+                )
+            )
+            rows.append(row)
+
+    header = [] if no_path else ["Path"]
+    header.append(title)
+    if old:
+        header.extend(["Old", "New"])
+    else:
+        header.append("Value")
+    header.append("Change")
+
+    return table(header, rows, markdown)
+
+
+class CmdExperimentsDiff(CmdBase):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        try:
+            diff = self.repo.experiments.diff(
+                a_rev=self.args.a_rev,
+                b_rev=self.args.b_rev,
+                all=self.args.all,
+            )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                if self.args.precision is None:
+                    precision = DEFAULT_PRECISION
+                else:
+                    precision = self.args.precision
+
+                diffs = [("metrics", "Metric"), ("params", "Param")]
+                for key, title in diffs:
+                    table = _show_diff(
+                        diff[key],
+                        title=title,
+                        markdown=self.args.show_md,
+                        no_path=self.args.no_path,
+                        old=self.args.old,
+                        precision=precision,
+                    )
+                    if table:
+                        logger.info(table)
+                        logger.info("")
+
+        except DvcException:
+            logger.exception("failed to show experiments diff")
+            return 1
+
+        return 0
+
+
+class CmdExperimentsRun(CmdRepro):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        saved_dir = os.path.realpath(os.curdir)
+        os.chdir(self.args.cwd)
+
+        # Dirty hack so the for loop below can at least enter once
+        if self.args.all_pipelines:
+            self.args.targets = [None]
+        elif not self.args.targets:
+            self.args.targets = self.default_targets
+
+        ret = 0
+        for target in self.args.targets:
+            try:
+                self.repo.experiments.run(
+                    target,
+                    name=self.args.name,
+                    queue=self.args.queue,
+                    run_all=self.args.run_all,
+                    jobs=self.args.jobs,
+                    params=self.args.params,
+                    checkpoint_resume=self.args.checkpoint_resume,
+                    **self._repro_kwargs,
+                )
+            except DvcException:
+                logger.exception("")
+                ret = 1
+                break
+
+        os.chdir(saved_dir)
+        return ret
+
+
+class CmdExperimentsGC(CmdRepro):
+    def run(self):
+        from dvc.repo.gc import _raise_error_if_all_disabled
+
+        if not self.repo.experiments:
+            return 0
+
+        _raise_error_if_all_disabled(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+        )
+
+        msg = "This will remove all experiments except those derived from "
+
+        msg += "the workspace"
+        if self.args.all_commits:
+            msg += " and all git commits"
+        elif self.args.all_branches and self.args.all_tags:
+            msg += " and all git branches and tags"
+        elif self.args.all_branches:
+            msg += " and all git branches"
+        elif self.args.all_tags:
+            msg += " and all git tags"
+        msg += " of the current repo."
+        if self.args.queued:
+            msg += " Run queued experiments will be preserved."
+        if self.args.queued:
+            msg += " Run queued experiments will be removed."
+
+        logger.warning(msg)
+
+        msg = "Are you sure you want to proceed?"
+        if not self.args.force and not prompt.confirm(msg):
+            return 1
+
+        removed = self.repo.experiments.gc(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+            queued=self.args.queued,
+        )
+
+        if removed:
+            logger.info(
+                f"Removed {removed} experiments. To remove unused cache files "
+                "use 'dvc gc'."
+            )
+        else:
+            logger.info("No experiments to remove.")
+        return 0
+
+
+class CmdExperimentsBranch(CmdBase):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        self.repo.experiments.branch(self.args.experiment, self.args.branch)
+
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    EXPERIMENTS_HELP = "Commands to display and compare experiments."
+
+    experiments_parser = subparsers.add_parser(
+        "experiments",
+        parents=[parent_parser],
+        aliases=["exp"],
+        description=append_doc_link(EXPERIMENTS_HELP, "experiments"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    experiments_subparsers = experiments_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc experiments CMD --help` to display "
+        "command-specific help.",
+    )
+
+    fix_subparsers(experiments_subparsers)
+
+    EXPERIMENTS_SHOW_HELP = "Print experiments."
+    experiments_show_parser = experiments_subparsers.add_parser(
+        "show",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_SHOW_HELP, "experiments/show"),
+        help=EXPERIMENTS_SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_show_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Show metrics for all branches.",
+    )
+    experiments_show_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Show metrics for all tags.",
+    )
+    experiments_show_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Show metrics for all commits.",
+    )
+    experiments_show_parser.add_argument(
+        "--no-pager",
+        action="store_true",
+        default=False,
+        help="Do not pipe output into a pager.",
+    )
+    experiments_show_parser.add_argument(
+        "--include-metrics",
+        action="append",
+        default=[],
+        help="Include the specified metrics in output table.",
+        metavar="<metrics_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--exclude-metrics",
+        action="append",
+        default=[],
+        help="Exclude the specified metrics from output table.",
+        metavar="<metrics_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--include-params",
+        action="append",
+        default=[],
+        help="Include the specified params in output table.",
+        metavar="<params_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--exclude-params",
+        action="append",
+        default=[],
+        help="Exclude the specified params from output table.",
+        metavar="<params_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--sort-by",
+        help="Sort related experiments by the specified metric or param.",
+        metavar="<metric/param>",
+    )
+    experiments_show_parser.add_argument(
+        "--sort-order",
+        help="Sort order to use with --sort-by.",
+        choices=("asc", "desc"),
+        default="asc",
+    )
+    experiments_show_parser.add_argument(
+        "--no-timestamp",
+        action="store_true",
+        default=False,
+        help="Do not show experiment timestamps.",
+    )
+    experiments_show_parser.add_argument(
+        "--sha",
+        action="store_true",
+        default=False,
+        help="Always show git commit SHAs instead of branch/tag names.",
+    )
+    experiments_show_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Print output in JSON format instead of a human-readable table.",
+    )
+    experiments_show_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics/params to `n` digits precision after the decimal "
+            f"point. Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
+    experiments_show_parser.set_defaults(func=CmdExperimentsShow)
+
+    EXPERIMENTS_APPLY_HELP = (
+        "Apply the changes from an experiment to your workspace."
+    )
+    experiments_apply_parser = experiments_subparsers.add_parser(
+        "apply",
+        parents=[parent_parser],
+        description=append_doc_link(
+            EXPERIMENTS_APPLY_HELP, "experiments/apply"
+        ),
+        help=EXPERIMENTS_APPLY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_apply_parser.add_argument(
+        "experiment", help="Experiment to be applied.",
+    )
+    experiments_apply_parser.set_defaults(func=CmdExperimentsApply)
+
+    EXPERIMENTS_DIFF_HELP = (
+        "Show changes between experiments in the DVC repository."
+    )
+    experiments_diff_parser = experiments_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_DIFF_HELP, "experiments/diff"),
+        help=EXPERIMENTS_DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_diff_parser.add_argument(
+        "a_rev", nargs="?", help="Old experiment to compare (defaults to HEAD)"
+    )
+    experiments_diff_parser.add_argument(
+        "b_rev",
+        nargs="?",
+        help="New experiment to compare (defaults to the current workspace)",
+    )
+    experiments_diff_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="Show unchanged metrics/params as well.",
+    )
+    experiments_diff_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Show output in JSON format.",
+    )
+    experiments_diff_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    experiments_diff_parser.add_argument(
+        "--old",
+        action="store_true",
+        default=False,
+        help="Show old metric/param value.",
+    )
+    experiments_diff_parser.add_argument(
+        "--no-path",
+        action="store_true",
+        default=False,
+        help="Don't show metric/param path.",
+    )
+    experiments_diff_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics/params to `n` digits precision after the decimal "
+            f"point. Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
+    experiments_diff_parser.set_defaults(func=CmdExperimentsDiff)
+
+    EXPERIMENTS_RUN_HELP = (
+        "Reproduce complete or partial experiment pipelines."
+    )
+    experiments_run_parser = experiments_subparsers.add_parser(
+        "run",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_RUN_HELP, "experiments/run"),
+        help=EXPERIMENTS_RUN_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    _add_run_common(experiments_run_parser)
+    experiments_run_parser.add_argument(
+        "--checkpoint-resume", type=str, default=None, help=argparse.SUPPRESS,
+    )
+    experiments_run_parser.set_defaults(func=CmdExperimentsRun)
+
+    EXPERIMENTS_RESUME_HELP = "Resume checkpoint experiments."
+    experiments_resume_parser = experiments_subparsers.add_parser(
+        "resume",
+        parents=[parent_parser],
+        aliases=["res"],
+        description=append_doc_link(
+            EXPERIMENTS_RESUME_HELP, "experiments/resume"
+        ),
+        help=EXPERIMENTS_RESUME_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    _add_run_common(experiments_resume_parser)
+    experiments_resume_parser.add_argument(
+        "-r",
+        "--rev",
+        type=str,
+        default=Experiments.LAST_CHECKPOINT,
+        dest="checkpoint_resume",
+        help=(
+            "Continue the specified checkpoint experiment. "
+            "If no experiment revision is provided, "
+            "the most recently run checkpoint experiment will be used."
+        ),
+        metavar="<experiment_rev>",
+    )
+    experiments_resume_parser.set_defaults(func=CmdExperimentsRun)
+
+    EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments."
+    EXPERIMENTS_GC_DESCRIPTION = (
+        "Removes all experiments which are not derived from the specified"
+        "Git revisions."
+    )
+    experiments_gc_parser = experiments_subparsers.add_parser(
+        "gc",
+        parents=[parent_parser],
+        description=append_doc_link(
+            EXPERIMENTS_GC_DESCRIPTION, "experiments/gc"
+        ),
+        help=EXPERIMENTS_GC_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_gc_parser.add_argument(
+        "-w",
+        "--workspace",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from the current workspace.",
+    )
+    experiments_gc_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from the tips of all Git branches.",
+    )
+    experiments_gc_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from all Git tags.",
+    )
+    experiments_gc_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from all Git commits.",
+    )
+    experiments_gc_parser.add_argument(
+        "--queued",
+        action="store_true",
+        default=False,
+        help=(
+            "Keep queued experiments (experiments run queue will be cleared "
+            "by default)."
+        ),
+    )
+    experiments_gc_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force garbage collection - automatically agree to all prompts.",
+    )
+    experiments_gc_parser.set_defaults(func=CmdExperimentsGC)
+
+    EXPERIMENTS_BRANCH_HELP = "Promote an experiment to a Git branch."
+    experiments_branch_parser = experiments_subparsers.add_parser(
+        "branch",
+        parents=[parent_parser],
+        description=append_doc_link(
+            EXPERIMENTS_BRANCH_HELP, "experiments/branch"
+        ),
+        help=EXPERIMENTS_BRANCH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_branch_parser.add_argument(
+        "experiment", help="Experiment to be promoted.",
+    )
+    experiments_branch_parser.add_argument(
+        "branch", help="Git branch name to use.",
+    )
+    experiments_branch_parser.set_defaults(func=CmdExperimentsBranch)
+
+
+def _add_run_common(parser):
+    """Add common args for 'exp run' and 'exp resume'."""
+    # inherit arguments from `dvc repro`
+    add_repro_arguments(parser)
+    parser.add_argument(
+        "-n",
+        "--name",
+        default=None,
+        help=(
+            "Human-readable experiment name. If not specified, a name will "
+            "be auto-generated."
+        ),
+    )
+    parser.add_argument(
+        "--params",
+        action="append",
+        default=[],
+        help="Use the specified param values when reproducing pipelines.",
+        metavar="[<filename>:]<params_list>",
+    )
+    parser.add_argument(
+        "--queue",
+        action="store_true",
+        default=False,
+        help="Stage this experiment in the run queue for future execution.",
+    )
+    parser.add_argument(
+        "--run-all",
+        action="store_true",
+        default=False,
+        help="Execute all experiments in the run queue.",
+    )
+    parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help="Run the specified number of experiments at a time in parallel.",
+        metavar="<number>",
+    )

dvc/command/freeze.py

@@ -1,9 +1,8 @@
 import argparse
 import logging
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -41,7 +40,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     freeze_parser.add_argument(
-        "targets", nargs="+", help="Stages or .dvc files to freeze"
+        "targets", nargs="+", help="Stages or .dvc files to freeze",
     ).complete = completion.DVC_FILE
     freeze_parser.set_defaults(func=CmdFreeze)
 
@@ -54,6 +53,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unfreeze_parser.add_argument(
-        "targets", nargs="+", help="Stages or .dvc files to unfreeze"
+        "targets", nargs="+", help="Stages or .dvc files to unfreeze",
     ).complete = completion.DVC_FILE
     unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/gc.py

@@ -2,9 +2,8 @@ import argparse
 import logging
 import os
 
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.ui import ui
+import dvc.prompt as prompt
+from dvc.command.base import CmdBase, append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -31,8 +30,6 @@ class CmdGC(CmdBase):
             msg += " and all git branches"
         elif self.args.all_tags:
             msg += " and all git tags"
-        elif self.args.all_experiments:
-            msg += " and all experiments"
 
         if self.args.repos:
             msg += " of the current and the following repos:"
@@ -45,14 +42,13 @@ class CmdGC(CmdBase):
         logger.warning(msg)
 
         msg = "Are you sure you want to proceed?"
-        if not self.args.force and not ui.confirm(msg):
+        if not self.args.force and not prompt.confirm(msg):
             return 1
 
         self.repo.gc(
             all_branches=self.args.all_branches,
             all_tags=self.args.all_tags,
             all_commits=self.args.all_commits,
-            all_experiments=self.args.all_experiments,
             cloud=self.args.cloud,
             remote=self.args.remote,
             force=self.args.force,
@@ -98,18 +94,11 @@ def add_parser(subparsers, parent_parser):
         help="Keep data files for all Git tags.",
     )
     gc_parser.add_argument(
-        "-A",
         "--all-commits",
         action="store_true",
         default=False,
         help="Keep data files for all Git commits.",
     )
-    gc_parser.add_argument(
-        "--all-experiments",
-        action="store_true",
-        default=False,
-        help="Keep data files for all experiments.",
-    )
     gc_parser.add_argument(
         "-c",
         "--cloud",
@@ -137,6 +126,7 @@ def add_parser(subparsers, parent_parser):
         help=(
             "Number of jobs to run simultaneously. "
             "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
         ),
         metavar="<number>",
     )

dvc/command/get.py

@@ -1,11 +1,10 @@
 import argparse
 import logging
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBaseNoRepo
 from dvc.exceptions import DvcException
 
-from ..cli.utils import append_doc_link
+from . import completion
+from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -13,13 +12,12 @@ logger = logging.getLogger(__name__)
 class CmdGet(CmdBaseNoRepo):
     def _show_url(self):
         from dvc.api import get_url
-        from dvc.ui import ui
 
         try:
             url = get_url(
                 self.args.path, repo=self.args.url, rev=self.args.rev
             )
-            ui.write(url, force=True)
+            logger.info(url)
         except DvcException:
             logger.exception("failed to show URL")
             return 1
@@ -41,7 +39,6 @@ class CmdGet(CmdBaseNoRepo):
                 path=self.args.path,
                 out=self.args.out,
                 rev=self.args.rev,
-                jobs=self.args.jobs,
             )
             return 0
         except DvcException:
@@ -66,7 +63,7 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path", help="Path to a file or directory within the repository"
+        "path", help="Path to a file or directory within the repository",
     ).complete = completion.FILE
     get_parser.add_argument(
         "-o",
@@ -87,14 +84,4 @@ def add_parser(subparsers, parent_parser):
         help="Print the storage location (URL) the target data would be "
         "downloaded from, and exit.",
     )
-    get_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-        ),
-        metavar="<number>",
-    )
     get_parser.set_defaults(func=CmdGet)

dvc/command/get_url.py

@@ -1,11 +1,10 @@
 import argparse
 import logging
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBaseNoRepo
 from dvc.exceptions import DvcException
 
-from ..cli.utils import append_doc_link
+from . import completion
+from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
 
@@ -15,7 +14,7 @@ class CmdGetUrl(CmdBaseNoRepo):
         from dvc.repo import Repo
 
         try:
-            Repo.get_url(self.args.url, out=self.args.out, jobs=self.args.jobs)
+            Repo.get_url(self.args.url, out=self.args.out)
             return 0
         except DvcException:
             logger.exception(f"failed to get '{self.args.url}'")
@@ -35,16 +34,6 @@ def add_parser(subparsers, parent_parser):
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
     get_parser.add_argument(
-        "out", nargs="?", help="Destination path to put data to."
+        "out", nargs="?", help="Destination path to put data to.",
     ).complete = completion.DIR
-    get_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-        ),
-        metavar="<number>",
-    )
     get_parser.set_defaults(func=CmdGetUrl)

dvc/command/git_hook.py

@@ -1,8 +1,7 @@
 import logging
 import os
 
-from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import fix_subparsers
+from dvc.command.base import CmdBaseNoRepo, fix_subparsers
 from dvc.exceptions import NotDvcRepoError
 
 logger = logging.getLogger(__name__)
@@ -26,7 +25,7 @@ class CmdHookBase(CmdBaseNoRepo):
 
 class CmdPreCommit(CmdHookBase):
     def _run(self):
-        from dvc.cli import main
+        from dvc.main import main
 
         return main(["status"])
 
@@ -50,14 +49,14 @@ class CmdPostCheckout(CmdHookBase):
         if os.path.isdir(os.path.join(".git", "rebase-merge")):
             return 0
 
-        from dvc.cli import main
+        from dvc.main import main
 
         return main(["checkout"])
 
 
 class CmdPrePush(CmdHookBase):
     def _run(self):
-        from dvc.cli import main
+        from dvc.main import main
 
         return main(["push"])
 
@@ -70,13 +69,14 @@ class CmdMergeDriver(CmdHookBase):
         dvc = Repo()
 
         try:
-            ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
-            our = Dvcfile(dvc, self.args.our, verify=False)
-            their = Dvcfile(dvc, self.args.their, verify=False)
+            with dvc.state:
+                ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
+                our = Dvcfile(dvc, self.args.our, verify=False)
+                their = Dvcfile(dvc, self.args.their, verify=False)
 
-            our.merge(ancestor, their)
+                our.merge(ancestor, their)
 
-            return 0
+                return 0
         finally:
             dvc.close()
 
@@ -106,7 +106,7 @@ def add_parser(subparsers, parent_parser):
         help=PRE_COMMIT_HELP,
     )
     pre_commit_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
     )
     pre_commit_parser.set_defaults(func=CmdPreCommit)
 
@@ -118,7 +118,7 @@ def add_parser(subparsers, parent_parser):
         help=POST_CHECKOUT_HELP,
     )
     post_checkout_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
     )
     post_checkout_parser.set_defaults(func=CmdPostCheckout)
 
@@ -130,7 +130,7 @@ def add_parser(subparsers, parent_parser):
         help=PRE_PUSH_HELP,
     )
     pre_push_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
     )
     pre_push_parser.set_defaults(func=CmdPrePush)
 
@@ -147,7 +147,9 @@ def add_parser(subparsers, parent_parser):
         help="Ancestor's version of the conflicting file.",
     )
     merge_driver_parser.add_argument(
-        "--our", required=True, help="Current version of the conflicting file."
+        "--our",
+        required=True,
+        help="Current version of the conflicting file.",
     )
     merge_driver_parser.add_argument(
         "--their",

dvc/command/imp.py

@@ -1,9 +1,8 @@
 import argparse
 import logging
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -20,7 +19,6 @@ class CmdImport(CmdBase):
                 rev=self.args.rev,
                 no_exec=self.args.no_exec,
                 desc=self.args.desc,
-                jobs=self.args.jobs,
             )
         except DvcException:
             logger.exception(
@@ -49,7 +47,7 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path", help="Path to a file or directory within the repository"
+        "path", help="Path to a file or directory within the repository",
     ).complete = completion.FILE
     import_parser.add_argument(
         "-o",
@@ -66,14 +64,14 @@ def add_parser(subparsers, parent_parser):
     )
     import_parser.add_argument(
         "--file",
-        help="Specify name of the .dvc file this command will generate.",
+        help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
     )
     import_parser.add_argument(
         "--no-exec",
         action="store_true",
         default=False,
-        help="Only create .dvc file without actually downloading it.",
+        help="Only create DVC-file without actually downloading it.",
     )
     import_parser.add_argument(
         "--desc",
@@ -84,14 +82,4 @@ def add_parser(subparsers, parent_parser):
             "This doesn't affect any DVC operations."
         ),
     )
-    import_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-        ),
-        metavar="<number>",
-    )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -1,9 +1,8 @@
 import argparse
 import logging
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -17,10 +16,7 @@ class CmdImportUrl(CmdBase):
                 out=self.args.out,
                 fname=self.args.file,
                 no_exec=self.args.no_exec,
-                remote=self.args.remote,
-                to_remote=self.args.to_remote,
                 desc=self.args.desc,
-                jobs=self.args.jobs,
             )
         except DvcException:
             logger.exception(
@@ -59,40 +55,18 @@ def add_parser(subparsers, parent_parser):
         "remote://remote_name/path/to/file/or/dir (see `dvc remote`)",
     )
     import_parser.add_argument(
-        "out", nargs="?", help="Destination path to put files to."
+        "out", nargs="?", help="Destination path to put files to.",
     ).complete = completion.DIR
     import_parser.add_argument(
         "--file",
-        help="Specify name of the .dvc file this command will generate.",
+        help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
     ).complete = completion.DIR
     import_parser.add_argument(
         "--no-exec",
         action="store_true",
         default=False,
-        help="Only create .dvc file without actually downloading it.",
-    )
-    import_parser.add_argument(
-        "--to-remote",
-        action="store_true",
-        default=False,
-        help="Download it directly to the remote",
-    )
-    import_parser.add_argument(
-        "-r",
-        "--remote",
-        help="Remote storage to download to",
-        metavar="<name>",
-    )
-    import_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-        ),
-        metavar="<number>",
+        help="Only create DVC-file without actually downloading it.",
     )
     import_parser.add_argument(
         "--desc",

dvc/command/init.py

@@ -4,8 +4,7 @@ import logging
 import colorama
 
 from dvc import analytics
-from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.utils import boxify
 from dvc.utils import format_link as fmt_link
 
@@ -13,10 +12,8 @@ logger = logging.getLogger(__name__)
 
 
 def _welcome_message():
-    from dvc.ui import ui
-
     if analytics.is_enabled():
-        ui.write(
+        logger.info(
             boxify(
                 "DVC has enabled anonymous aggregate usage analytics.\n"
                 "Read the analytics documentation (and how to opt-out) here:\n"
@@ -33,7 +30,7 @@ def _welcome_message():
         f"- Star us on GitHub: {fmt_link('https://github.com/iterative/dvc')}"
     ).format(yellow=colorama.Fore.YELLOW, nc=colorama.Fore.RESET)
 
-    ui.write(msg)
+    logger.info(msg)
 
 
 class CmdInit(CmdBaseNoRepo):
@@ -42,14 +39,14 @@ class CmdInit(CmdBaseNoRepo):
         from dvc.repo import Repo
 
         try:
-            with Repo.init(
+            self.repo = Repo.init(
                 ".",
                 no_scm=self.args.no_scm,
                 force=self.args.force,
                 subdir=self.args.subdir,
-            ) as repo:
-                self.config = repo.config
-                _welcome_message()
+            )
+            self.config = self.repo.config
+            _welcome_message()
         except InitError:
             logger.exception("failed to initiate DVC")
             return 1

dvc/command/install.py

@@ -1,8 +1,7 @@
 import argparse
 import logging
 
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)

dvc/command/ls/__init__.py

@@ -1,12 +1,11 @@
 import argparse
 import logging
+import sys
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
-from dvc.commands.ls.ls_colors import LsColors
+from dvc.command import completion
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.command.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
-from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -35,11 +34,9 @@ class CmdList(CmdBaseNoRepo):
                 recursive=self.args.recursive,
                 dvc_only=self.args.dvc_only,
             )
-            if self.args.json:
-                ui.write_json(entries)
-            elif entries:
-                entries = _prettify(entries, with_color=True)
-                ui.write("\n".join(entries))
+            if entries:
+                entries = _prettify(entries, sys.stdout.isatty())
+                logger.info("\n".join(entries))
             return 0
         except DvcException:
             logger.exception(f"failed to list '{self.args.url}'")
@@ -53,7 +50,6 @@ def add_parser(subparsers, parent_parser):
     )
     list_parser = subparsers.add_parser(
         "list",
-        aliases=["ls"],
         parents=[parent_parser],
         description=append_doc_link(LIST_HELP, "list"),
         help=LIST_HELP,
@@ -69,12 +65,6 @@ def add_parser(subparsers, parent_parser):
     list_parser.add_argument(
         "--dvc-only", action="store_true", help="Show only DVC outputs."
     )
-    list_parser.add_argument(
-        "--json",
-        "--show-json",
-        action="store_true",
-        help="Show output in JSON format.",
-    )
     list_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/metrics.py

@@ -1,12 +1,9 @@
 import argparse
 import logging
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link, fix_subparsers
-from dvc.exceptions import DvcException
-from dvc.ui import ui
-from dvc.utils.serialize import encode_exception
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.exceptions import BadMetricError, DvcException
 
 logger = logging.getLogger(__name__)
 
@@ -14,6 +11,36 @@ logger = logging.getLogger(__name__)
 DEFAULT_PRECISION = 5
 
 
+def _show_metrics(
+    metrics, all_branches=False, all_tags=False, all_commits=False
+):
+    from dvc.utils.diff import format_dict
+    from dvc.utils.flatten import flatten
+
+    # When `metrics` contains a `None` key, it means that some files
+    # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
+    missing = metrics.pop(None, None)
+
+    lines = []
+    for branch, val in metrics.items():
+        if all_branches or all_tags or all_commits:
+            lines.append(f"{branch}:")
+
+        for fname, metric in val.items():
+            if not isinstance(metric, dict):
+                lines.append("\t{}: {}".format(fname, str(metric)))
+                continue
+
+            lines.append(f"\t{fname}:")
+            for key, value in flatten(format_dict(metric)).items():
+                lines.append(f"\t\t{key}: {value}")
+
+    if missing:
+        raise BadMetricError(missing)
+
+    return "\n".join(lines)
+
+
 class CmdMetricsBase(CmdBase):
     UNINITIALIZED = True
 
@@ -28,28 +55,60 @@ class CmdMetricsShow(CmdMetricsBase):
                 all_commits=self.args.all_commits,
                 recursive=self.args.recursive,
             )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(metrics))
+            else:
+                table = _show_metrics(
+                    metrics,
+                    self.args.all_branches,
+                    self.args.all_tags,
+                    self.args.all_commits,
+                )
+                if table:
+                    logger.info(table)
         except DvcException:
             logger.exception("")
             return 1
 
-        if self.args.json:
-            ui.write_json(metrics, default=encode_exception)
-        else:
-            from dvc.compare import show_metrics
-
-            show_metrics(
-                metrics,
-                markdown=self.args.markdown,
-                all_branches=self.args.all_branches,
-                all_tags=self.args.all_tags,
-                all_commits=self.args.all_commits,
-                precision=self.args.precision or DEFAULT_PRECISION,
-                round_digits=True,
-            )
-
         return 0
 
 
+def _show_diff(diff, markdown=False, no_path=False, precision=None):
+    from collections import OrderedDict
+
+    from dvc.utils.diff import table
+
+    if precision is None:
+        precision = DEFAULT_PRECISION
+
+    def _round(val):
+        if isinstance(val, float):
+            return round(val, precision)
+
+        return val
+
+    rows = []
+    for fname, mdiff in diff.items():
+        sorted_mdiff = OrderedDict(sorted(mdiff.items()))
+        for metric, change in sorted_mdiff.items():
+            row = [] if no_path else [fname]
+            row.append(metric)
+            row.append(_round(change.get("old")))
+            row.append(_round(change["new"]))
+            row.append(_round(change.get("diff")))
+            rows.append(row)
+
+    header = [] if no_path else ["Path"]
+    header.append("Metric")
+    header.extend(["Old", "New"])
+    header.append("Change")
+
+    return table(header, rows, markdown)
+
+
 class CmdMetricsDiff(CmdMetricsBase):
     def run(self):
         try:
@@ -60,26 +119,25 @@ class CmdMetricsDiff(CmdMetricsBase):
                 recursive=self.args.recursive,
                 all=self.args.all,
             )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                table = _show_diff(
+                    diff,
+                    self.args.show_md,
+                    self.args.no_path,
+                    precision=self.args.precision,
+                )
+                if table:
+                    logger.info(table)
+
         except DvcException:
             logger.exception("failed to show metrics diff")
             return 1
 
-        if self.args.json:
-            ui.write_json(diff)
-        else:
-            from dvc.compare import show_diff
-
-            show_diff(
-                diff,
-                title="Metric",
-                markdown=self.args.markdown,
-                no_path=self.args.no_path,
-                precision=self.args.precision or DEFAULT_PRECISION,
-                round_digits=True,
-                a_rev=self.args.a_rev,
-                b_rev=self.args.b_rev,
-            )
-
         return 0
 
 
@@ -132,27 +190,17 @@ def add_parser(subparsers, parent_parser):
         help="Show metrics for all tags.",
     )
     metrics_show_parser.add_argument(
-        "-A",
         "--all-commits",
         action="store_true",
         default=False,
         help="Show metrics for all commits.",
     )
     metrics_show_parser.add_argument(
-        "--json",
         "--show-json",
         action="store_true",
         default=False,
         help="Show output in JSON format.",
     )
-    metrics_show_parser.add_argument(
-        "--md",
-        "--show-md",
-        action="store_true",
-        default=False,
-        dest="markdown",
-        help="Show tabulated output in the Markdown format (GFM).",
-    )
     metrics_show_parser.add_argument(
         "-R",
         "--recursive",
@@ -163,15 +211,6 @@ def add_parser(subparsers, parent_parser):
             "metrics files."
         ),
     )
-    metrics_show_parser.add_argument(
-        "--precision",
-        type=int,
-        help=(
-            "Round metrics to `n` digits precision after the decimal point. "
-            f"Rounds to {DEFAULT_PRECISION} digits by default."
-        ),
-        metavar="<n>",
-    )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
     METRICS_DIFF_HELP = (
@@ -197,11 +236,8 @@ def add_parser(subparsers, parent_parser):
         "--targets",
         nargs="*",
         help=(
-            "Specific metrics file(s) to compare "
-            "(even if not found as `metrics` in `dvc.yaml`). "
-            "Using -R, directories to search metrics files in "
-            "can also be given."
-            "Shows all tracked metrics by default."
+            "Limit command scope to these metrics files. Using -R, "
+            "directories to search metrics files in can also be given."
         ),
         metavar="<paths>",
     ).complete = completion.FILE
@@ -222,18 +258,15 @@ def add_parser(subparsers, parent_parser):
         help="Show unchanged metrics as well.",
     )
     metrics_diff_parser.add_argument(
-        "--json",
         "--show-json",
         action="store_true",
         default=False,
         help="Show output in JSON format.",
     )
     metrics_diff_parser.add_argument(
-        "--md",
         "--show-md",
         action="store_true",
         default=False,
-        dest="markdown",
         help="Show tabulated output in the Markdown format (GFM).",
     )
     metrics_diff_parser.add_argument(

dvc/command/move.py

@@ -1,9 +1,8 @@
 import argparse
 import logging
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -26,7 +25,7 @@ def add_parser(subparsers, parent_parser):
     MOVE_HELP = "Rename or move a DVC controlled data file or a directory."
     MOVE_DESCRIPTION = (
         "Rename or move a DVC controlled data file or a directory.\n"
-        "It renames and modifies the corresponding .dvc file to reflect the"
+        "It renames and modifies the corresponding DVC-file to reflect the"
         " changes."
     )
 
@@ -38,9 +37,9 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
-        "src", help="Source path to a data file or directory."
+        "src", help="Source path to a data file or directory.",
     ).complete = completion.FILE
     move_parser.add_argument(
-        "dst", help="Destination path."
+        "dst", help="Destination path.",
     ).complete = completion.FILE
     move_parser.set_defaults(func=CmdMove)

dvc/command/params.py

@@ -1,46 +1,56 @@
 import argparse
 import logging
+from collections import OrderedDict
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link, fix_subparsers
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
 
-class CmdParamsDiff(CmdBase):
-    UNINITIALIZED = True
+def _show_diff(diff, markdown=False, no_path=False):
+    from dvc.utils.diff import table
 
+    rows = []
+    for fname, pdiff in diff.items():
+        sorted_pdiff = OrderedDict(sorted(pdiff.items()))
+        for param, change in sorted_pdiff.items():
+            row = [] if no_path else [fname]
+            row.append(param)
+            row.append(change["old"])
+            row.append(change["new"])
+            rows.append(row)
+
+    header = [] if no_path else ["Path"]
+    header.append("Param")
+    header.append("Old")
+    header.append("New")
+
+    return table(header, rows, markdown)
+
+
+class CmdParamsDiff(CmdBase):
     def run(self):
         try:
             diff = self.repo.params.diff(
                 a_rev=self.args.a_rev,
                 b_rev=self.args.b_rev,
-                targets=self.args.targets,
                 all=self.args.all,
-                deps=self.args.deps,
             )
+
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(diff))
+            else:
+                table = _show_diff(diff, self.args.show_md, self.args.no_path)
+                if table:
+                    logger.info(table)
+
         except DvcException:
             logger.exception("failed to show params diff")
             return 1
 
-        if self.args.json:
-            ui.write_json(diff)
-        else:
-            from dvc.compare import show_diff
-
-            show_diff(
-                diff,
-                title="Param",
-                markdown=self.args.markdown,
-                no_path=self.args.no_path,
-                show_changes=False,
-                a_rev=self.args.a_rev,
-                b_rev=self.args.b_rev,
-            )
-
         return 0
 
 
@@ -81,16 +91,6 @@ def add_parser(subparsers, parent_parser):
         nargs="?",
         help=("New Git commit to compare (defaults to the current workspace)"),
     )
-    params_diff_parser.add_argument(
-        "--targets",
-        nargs="*",
-        help=(
-            "Specific params file(s) to compare "
-            "(even if not found as `params` in `dvc.yaml`). "
-            "Shows all tracked params by default."
-        ),
-        metavar="<paths>",
-    ).complete = completion.FILE
     params_diff_parser.add_argument(
         "--all",
         action="store_true",
@@ -98,24 +98,15 @@ def add_parser(subparsers, parent_parser):
         help="Show unchanged params as well.",
     )
     params_diff_parser.add_argument(
-        "--deps",
-        action="store_true",
-        default=False,
-        help="Show only params that are stage dependencies.",
-    )
-    params_diff_parser.add_argument(
-        "--json",
         "--show-json",
         action="store_true",
         default=False,
         help="Show output in JSON format.",
     )
     params_diff_parser.add_argument(
-        "--md",
         "--show-md",
         action="store_true",
         default=False,
-        dest="markdown",
         help="Show tabulated output in the Markdown format (GFM).",
     )
     params_diff_parser.add_argument(

dvc/command/plots.py

@@ -1,27 +1,33 @@
 import argparse
-import json
 import logging
 import os
 
-from funcy import first
-
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link, fix_subparsers
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.ui import ui
+from dvc.schema import PLOT_PROPS
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
+PAGE_HTML = """<!DOCTYPE html>
+<html>
+<head>
+    <title>DVC Plot</title>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+</head>
+<body>
+    {divs}
+</body>
+</html>"""
 
-def _show_json(renderers, split=False):
-    from dvc.render.convert import to_json
-
-    result = {
-        renderer.name: to_json(renderer, split) for renderer in renderers
-    }
-    ui.write_json(result)
+DIV_HTML = """<div id = "{id}"></div>
+<script type = "text/javascript">
+    var spec = {vega_json};
+    vegaEmbed('#{id}', spec);
+</script>"""
 
 
 class CmdPlots(CmdBase):
@@ -29,18 +35,11 @@ class CmdPlots(CmdBase):
         raise NotImplementedError
 
     def _props(self):
-        from dvc.schema import PLOT_PROPS
-
         # Pass only props specified by user, to not shadow ones from plot def
         props = {p: getattr(self.args, p) for p in PLOT_PROPS}
         return {k: v for k, v in props.items() if v is not None}
 
     def run(self):
-        from pathlib import Path
-
-        from dvc.render.match import match_renderers
-        from dvc_render import render_html
-
         if self.args.show_vega:
             if not self.args.targets:
                 logger.error("please specify a target for `--show-vega`")
@@ -50,82 +49,34 @@ class CmdPlots(CmdBase):
                     "you can only specify one target for `--show-vega`"
                 )
                 return 1
-            if self.args.json:
-                logger.error(
-                    "'--show-vega' and '--json' are mutually exclusive "
-                    "options."
-                )
-                return 1
 
         try:
-
-            plots_data = self._func(
-                targets=self.args.targets, props=self._props()
-            )
-
-            if not plots_data:
-                ui.error_write(
-                    "No plots were loaded, "
-                    "visualization file will not be created."
-                )
-
-            out: str = self.args.out or "dvc_plots"
-
-            renderers_out = (
-                out if self.args.json else os.path.join(out, "static")
-            )
-            renderers = match_renderers(
-                plots_data=plots_data,
-                out=renderers_out,
-                templates_dir=self.repo.plots.templates_dir,
-            )
+            plots = self._func(targets=self.args.targets, props=self._props())
 
             if self.args.show_vega:
-                renderer = first(filter(lambda r: r.TYPE == "vega", renderers))
-                if renderer:
-                    ui.write_json(json.loads(renderer.get_filled_template()))
-                return 0
-            if self.args.json:
-                _show_json(renderers, self.args.split)
+                target = self.args.targets[0]
+                logger.info(plots[target])
                 return 0
 
-            html_template_path = self.args.html_template
-            if not html_template_path:
-                html_template_path = self.repo.config.get("plots", {}).get(
-                    "html_template", None
-                )
-                if html_template_path and not os.path.isabs(
-                    html_template_path
-                ):
-                    html_template_path = os.path.join(
-                        self.repo.dvc_dir, html_template_path
-                    )
+            divs = [
+                DIV_HTML.format(id=f"plot{i}", vega_json=plot)
+                for i, plot in enumerate(plots.values())
+            ]
+            html = PAGE_HTML.format(divs="\n".join(divs))
+            path = self.args.out or "plots.html"
 
-            output_file: Path = (Path.cwd() / out).resolve() / "index.html"
+            path = os.path.join(os.getcwd(), path)
+            with open(path, "w") as fobj:
+                fobj.write(html)
 
-            render_html(
-                renderers=renderers,
-                output_file=output_file,
-                template_path=html_template_path,
-            )
-
-            ui.write(output_file.as_uri())
-            auto_open = self.repo.config["plots"].get("auto_open", False)
-            if self.args.open or auto_open:
-                if not auto_open:
-                    ui.write(
-                        "To enable auto opening, you can run:\n"
-                        "\n"
-                        "\tdvc config plots.auto_open true"
-                    )
-                return ui.open_browser(output_file)
-
-            return 0
+            logger.info(f"file://{path}")
 
         except DvcException:
             logger.exception("")
             return 1
 
+        return 0
+
 
 class CmdPlotsShow(CmdPlots):
     UNINITIALIZED = True
@@ -149,46 +100,15 @@ class CmdPlotsDiff(CmdPlots):
 class CmdPlotsModify(CmdPlots):
     def run(self):
         self.repo.plots.modify(
-            self.args.target, props=self._props(), unset=self.args.unset
+            self.args.target, props=self._props(), unset=self.args.unset,
         )
         return 0
 
 
-class CmdPlotsTemplates(CmdBase):
-    TEMPLATES_CHOICES = [
-        "simple",
-        "linear",
-        "confusion",
-        "confusion_normalized",
-        "scatter",
-        "smooth",
-    ]
-
-    def run(self):
-        from dvc_render.vega_templates import dump_templates
-
-        try:
-            out = (
-                os.path.join(os.getcwd(), self.args.out)
-                if self.args.out
-                else self.repo.plots.templates_dir
-            )
-
-            targets = [self.args.target] if self.args.target else None
-            dump_templates(output=out, targets=targets)
-            templates_path = os.path.relpath(out, os.getcwd())
-            ui.write(f"Templates have been written into '{templates_path}'.")
-
-            return 0
-        except DvcException:
-            logger.exception("")
-            return 1
-
-
 def add_parser(subparsers, parent_parser):
     PLOTS_HELP = (
         "Commands to visualize and compare plot metrics in structured files "
-        "(JSON, YAML, CSV, TSV)."
+        "(JSON, YAML, CSV, TSV)"
     )
 
     plots_parser = subparsers.add_parser(
@@ -221,8 +141,7 @@ def add_parser(subparsers, parent_parser):
         "Shows all plots by default.",
     ).complete = completion.FILE
     _add_props_arguments(plots_show_parser)
-    _add_output_argument(plots_show_parser)
-    _add_ui_arguments(plots_show_parser)
+    _add_output_arguments(plots_show_parser)
     plots_show_parser.set_defaults(func=CmdPlotsShow)
 
     PLOTS_DIFF_HELP = (
@@ -239,12 +158,10 @@ def add_parser(subparsers, parent_parser):
     plots_diff_parser.add_argument(
         "--targets",
         nargs="*",
-        help=(
-            "Specific plots file(s) to visualize "
-            "(even if not found as `plots` in `dvc.yaml`). "
-            "Shows all tracked plots by default."
-        ),
-        metavar="<paths>",
+        help="Files to visualize (supports any file, "
+        "even when not found as `plots` in `dvc.yaml`). "
+        "Shows all plots by default.",
+        metavar="<path>",
     ).complete = completion.FILE
     plots_diff_parser.add_argument(
         "-e",
@@ -254,17 +171,13 @@ def add_parser(subparsers, parent_parser):
         help=argparse.SUPPRESS,
     )
     plots_diff_parser.add_argument(
-        "revisions", nargs="*", default=None, help="Git commits to plot from"
+        "revisions", nargs="*", default=None, help="Git commits to plot from",
     )
     _add_props_arguments(plots_diff_parser)
-    _add_output_argument(plots_diff_parser)
-    _add_ui_arguments(plots_diff_parser)
+    _add_output_arguments(plots_diff_parser)
     plots_diff_parser.set_defaults(func=CmdPlotsDiff)
 
-    PLOTS_MODIFY_HELP = (
-        "Modify display properties of data-series plots "
-        "(has no effect on image-type plots)."
-    )
+    PLOTS_MODIFY_HELP = "Modify display properties of plot metrics files."
     plots_modify_parser = plots_subparsers.add_parser(
         "modify",
         parents=[parent_parser],
@@ -273,7 +186,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_modify_parser.add_argument(
-        "target", help="Metric file to set properties to"
+        "target", help="Metric file to set properties to",
     ).complete = completion.FILE
     _add_props_arguments(plots_modify_parser)
     plots_modify_parser.add_argument(
@@ -284,27 +197,6 @@ def add_parser(subparsers, parent_parser):
     )
     plots_modify_parser.set_defaults(func=CmdPlotsModify)
 
-    TEMPLATES_HELP = (
-        "Write built-in plots templates to a directory "
-        "(.dvc/plots by default)."
-    )
-    plots_templates_parser = plots_subparsers.add_parser(
-        "templates",
-        parents=[parent_parser],
-        description=append_doc_link(TEMPLATES_HELP, "plots/templates"),
-        help=TEMPLATES_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    plots_templates_parser.add_argument(
-        "target",
-        default=None,
-        nargs="?",
-        choices=CmdPlotsTemplates.TEMPLATES_CHOICES,
-        help="Template to write. Writes all templates by default.",
-    )
-    _add_output_argument(plots_templates_parser, typ="templates")
-    plots_templates_parser.set_defaults(func=CmdPlotsTemplates)
-
 
 def _add_props_arguments(parser):
     parser.add_argument(
@@ -344,42 +236,17 @@ def _add_props_arguments(parser):
     )
 
 
-def _add_output_argument(parser, typ="plots"):
+def _add_output_arguments(parser):
     parser.add_argument(
         "-o",
         "--out",
         default=None,
-        help=f"Directory to save {typ} to.",
+        help="Destination path to save plots to",
         metavar="<path>",
     ).complete = completion.DIR
-
-
-def _add_ui_arguments(parser):
     parser.add_argument(
         "--show-vega",
         action="store_true",
         default=False,
         help="Show output in Vega format.",
     )
-    parser.add_argument(
-        "--json",
-        "--show-json",
-        action="store_true",
-        default=False,
-        help=argparse.SUPPRESS,
-    )
-    parser.add_argument(
-        "--split", action="store_true", default=False, help=argparse.SUPPRESS
-    )
-    parser.add_argument(
-        "--open",
-        action="store_true",
-        default=False,
-        help="Open plot file directly in the browser.",
-    )
-    parser.add_argument(
-        "--html-template",
-        default=None,
-        help="Custom HTML template for VEGA visualization.",
-        metavar="<path>",
-    )

dvc/command/remote.py

@@ -1,10 +1,13 @@
 import argparse
+import logging
 
-from dvc.cli.utils import append_doc_link, fix_subparsers
-from dvc.commands.config import CmdConfig
-from dvc.ui import ui
+from dvc.command.base import append_doc_link, fix_subparsers
+from dvc.command.config import CmdConfig
+from dvc.config import ConfigError, merge
 from dvc.utils import format_link
 
+logger = logging.getLogger(__name__)
+
 
 class CmdRemote(CmdConfig):
     def __init__(self, args):
@@ -14,18 +17,14 @@ class CmdRemote(CmdConfig):
             self.args.name = self.args.name.lower()
 
     def _check_exists(self, conf):
-        from dvc.config import ConfigError
-
         if self.args.name not in conf["remote"]:
             raise ConfigError(f"remote '{self.args.name}' doesn't exist.")
 
 
 class CmdRemoteAdd(CmdRemote):
     def run(self):
-        from dvc.config import ConfigError
-
         if self.args.default:
-            ui.write(f"Setting '{self.args.name}' as a default remote.")
+            logger.info(f"Setting '{self.args.name}' as a default remote.")
 
         with self.config.edit(self.args.level) as conf:
             if self.args.name in conf["remote"] and not self.args.force:
@@ -47,14 +46,13 @@ class CmdRemoteRemove(CmdRemote):
             self._check_exists(conf)
             del conf["remote"][self.args.name]
 
-        up_to_level = self.args.level or "repo"
         # Remove core.remote refs to this remote in any shadowing configs
         for level in reversed(self.config.LEVELS):
             with self.config.edit(level) as conf:
                 if conf["core"].get("remote") == self.args.name:
                     del conf["core"]["remote"]
 
-            if level == up_to_level:
+            if level == self.args.level:
                 break
 
         return 0
@@ -62,8 +60,6 @@ class CmdRemoteRemove(CmdRemote):
 
 class CmdRemoteModify(CmdRemote):
     def run(self):
-        from dvc.config import merge
-
         with self.config.edit(self.args.level) as conf:
             merged = self.config.load_config_to_level(self.args.level)
             merge(merged, conf)
@@ -81,14 +77,13 @@ class CmdRemoteModify(CmdRemote):
 
 class CmdRemoteDefault(CmdRemote):
     def run(self):
-        from dvc.config import ConfigError
 
         if self.args.name is None and not self.args.unset:
-            conf = self.config.read(self.args.level)
+            conf = self.config.load_one(self.args.level)
             try:
-                ui.write(conf["core"]["remote"])
+                print(conf["core"]["remote"])
             except KeyError:
-                ui.write("No default remote set")
+                logger.info("No default remote set")
                 return 1
         else:
             with self.config.edit(self.args.level) as conf:
@@ -112,9 +107,9 @@ class CmdRemoteDefault(CmdRemote):
 
 class CmdRemoteList(CmdRemote):
     def run(self):
-        conf = self.config.read(self.args.level)
-        for name, remote_conf in conf["remote"].items():
-            ui.write(name, remote_conf["url"], sep="\t")
+        conf = self.config.load_one(self.args.level)
+        for name, conf in conf["remote"].items():
+            logger.info("{}\t{}".format(name, conf["url"]))
         return 0
 
 
@@ -124,8 +119,6 @@ class CmdRemoteRename(CmdRemote):
             conf["core"]["remote"] = self.args.new
 
     def run(self):
-        from dvc.config import ConfigError
-
         all_config = self.config.load_config_to_level(None)
         if self.args.new in all_config.get("remote", {}):
             raise ConfigError(
@@ -140,9 +133,8 @@ class CmdRemoteRename(CmdRemote):
             del conf["remote"][self.args.name]
             self._rename_default(conf)
 
-        up_to_level = self.args.level or "repo"
         for level in reversed(self.config.LEVELS):
-            if level == up_to_level:
+            if level == self.args.level:
                 break
             with self.config.edit(level) as level_conf:
                 self._rename_default(level_conf)
@@ -151,7 +143,7 @@ class CmdRemoteRename(CmdRemote):
 
 
 def add_parser(subparsers, parent_parser):
-    from dvc.commands.config import parent_config_parser
+    from dvc.command.config import parent_config_parser
 
     REMOTE_HELP = "Set up and manage data remotes."
     remote_parser = subparsers.add_parser(

dvc/command/remove.py

@@ -1,9 +1,8 @@
 import argparse
 import logging
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -15,15 +14,15 @@ class CmdRemove(CmdBase):
             try:
                 self.repo.remove(target, outs=self.args.outs)
             except DvcException:
-                logger.exception("")
+                logger.exception(f"failed to remove '{target}'")
                 return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     REMOVE_HELP = (
-        "Remove stages from dvc.yaml and/or"
-        " stop tracking files or directories."
+        "Remove stages or .dvc files, unprotect their outputs, and"
+        " erase .gitignore entries."
     )
     remove_parser = subparsers.add_parser(
         "remove",
@@ -39,8 +38,6 @@ def add_parser(subparsers, parent_parser):
         help="Remove outputs as well.",
     )
     remove_parser.add_argument(
-        "targets",
-        nargs="+",
-        help=".dvc files or stages from dvc.yaml to remove.",
+        "targets", nargs="+", help="DVC-files to remove.",
     ).complete = completion.DVC_FILE
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -1,54 +1,68 @@
 import argparse
+import logging
+import os
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.commands.status import CmdDataStatus
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.metrics import _show_metrics
+from dvc.command.status import CmdDataStatus
+from dvc.dvcfile import PIPELINE_FILE
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
 
 
 class CmdRepro(CmdBase):
     def run(self):
-        from dvc.ui import ui
+        saved_dir = os.path.realpath(os.curdir)
+        os.chdir(self.args.cwd)
 
-        stages = self.repo.reproduce(
-            **self._common_kwargs, **self._repro_kwargs
-        )
-        if len(stages) == 0:
-            ui.write(CmdDataStatus.UP_TO_DATE_MSG)
-        else:
-            ui.write(
-                "Use `dvc push` to send your updates to " "remote storage."
-            )
+        # Dirty hack so the for loop below can at least enter once
+        if self.args.all_pipelines:
+            self.args.targets = [None]
+        elif not self.args.targets:
+            self.args.targets = self.default_targets
 
-        if self.args.metrics:
-            from dvc.compare import show_metrics
+        ret = 0
+        for target in self.args.targets:
+            try:
+                stages = self.repo.reproduce(target, **self._repro_kwargs)
 
-            metrics = self.repo.metrics.show()
-            show_metrics(metrics)
+                if len(stages) == 0:
+                    logger.info(CmdDataStatus.UP_TO_DATE_MSG)
+                else:
+                    logger.info(
+                        "Use `dvc push` to send your updates to "
+                        "remote storage."
+                    )
 
-        return 0
+                if self.args.metrics:
+                    metrics = self.repo.metrics.show()
+                    logger.info(_show_metrics(metrics))
+
+            except DvcException:
+                logger.exception("")
+                ret = 1
+                break
+
+        os.chdir(saved_dir)
+        return ret
 
     @property
-    def _common_kwargs(self):
+    def _repro_kwargs(self):
         return {
-            "targets": self.args.targets,
             "single_item": self.args.single_item,
             "force": self.args.force,
             "dry": self.args.dry,
             "interactive": self.args.interactive,
             "pipeline": self.args.pipeline,
             "all_pipelines": self.args.all_pipelines,
+            "run_cache": not self.args.no_run_cache,
+            "no_commit": self.args.no_commit,
             "downstream": self.args.downstream,
             "recursive": self.args.recursive,
             "force_downstream": self.args.force_downstream,
             "pull": self.args.pull,
-        }
-
-    @property
-    def _repro_kwargs(self):
-        return {
-            "run_cache": not self.args.no_run_cache,
-            "no_commit": self.args.no_commit,
             "glob": self.args.glob,
         }
 
@@ -57,8 +71,8 @@ def add_arguments(repro_parser):
     repro_parser.add_argument(
         "targets",
         nargs="*",
-        help="Stages to reproduce. 'dvc.yaml' by default.",
-    ).complete = completion.DVCFILES_AND_STAGE
+        help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
+    ).complete = completion.DVC_FILE
     repro_parser.add_argument(
         "-f",
         "--force",
@@ -66,13 +80,6 @@ def add_arguments(repro_parser):
         default=False,
         help="Reproduce even if dependencies were not changed.",
     )
-    repro_parser.add_argument(
-        "-i",
-        "--interactive",
-        action="store_true",
-        default=False,
-        help="Ask for confirmation before reproducing each stage.",
-    )
     repro_parser.add_argument(
         "-s",
         "--single-item",
@@ -81,13 +88,42 @@ def add_arguments(repro_parser):
         help="Reproduce only single data item without recursive dependencies "
         "check.",
     )
+    repro_parser.add_argument(
+        "-c",
+        "--cwd",
+        default=os.path.curdir,
+        help="Directory within your repo to reproduce from. Note: deprecated "
+        "by `dvc --cd <path>`.",
+        metavar="<path>",
+    )
+    repro_parser.add_argument(
+        "-m",
+        "--metrics",
+        action="store_true",
+        default=False,
+        help="Show metrics after reproduction.",
+    )
+    repro_parser.add_argument(
+        "--dry",
+        action="store_true",
+        default=False,
+        help="Only print the commands that would be executed without "
+        "actually executing.",
+    )
+    repro_parser.add_argument(
+        "-i",
+        "--interactive",
+        action="store_true",
+        default=False,
+        help="Ask for confirmation before reproducing each stage.",
+    )
     repro_parser.add_argument(
         "-p",
         "--pipeline",
         action="store_true",
         default=False,
-        help="Reproduce the whole pipeline that the specified targets "
-        "belong to.",
+        help="Reproduce the whole pipeline that the specified stage file "
+        "belongs to.",
     )
     repro_parser.add_argument(
         "-P",
@@ -104,17 +140,13 @@ def add_arguments(repro_parser):
         help="Reproduce all stages in the specified directory.",
     )
     repro_parser.add_argument(
-        "-m",
-        "--metrics",
+        "--no-run-cache",
         action="store_true",
         default=False,
-        help="Show metrics after reproduction.",
-    )
-    repro_parser.add_argument(
-        "--downstream",
-        action="store_true",
-        default=False,
-        help="Start from the specified stages when reproducing pipelines.",
+        help=(
+            "Execute stage commands even if they have already been run with "
+            "the same command/dependencies/outputs/etc before."
+        ),
     )
     repro_parser.add_argument(
         "--force-downstream",
@@ -123,6 +155,18 @@ def add_arguments(repro_parser):
         help="Reproduce all descendants of a changed stage even if their "
         "direct dependencies didn't change.",
     )
+    repro_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
+    repro_parser.add_argument(
+        "--downstream",
+        action="store_true",
+        default=False,
+        help="Start from the specified stages when reproducing pipelines.",
+    )
     repro_parser.add_argument(
         "--pull",
         action="store_true",
@@ -133,11 +177,10 @@ def add_arguments(repro_parser):
         ),
     )
     repro_parser.add_argument(
-        "--dry",
+        "--glob",
         action="store_true",
         default=False,
-        help="Only print the commands that would be executed without "
-        "actually executing.",
+        help="Allows targets containing shell-style wildcards.",
     )
 
 
@@ -152,28 +195,5 @@ def add_parser(subparsers, parent_parser):
         help=REPRO_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    # repro/exp run shared args
     add_arguments(repro_parser)
-    # repro only args
-    repro_parser.add_argument(
-        "--glob",
-        action="store_true",
-        default=False,
-        help="Allows targets containing shell-style wildcards.",
-    )
-    repro_parser.add_argument(
-        "--no-commit",
-        action="store_true",
-        default=False,
-        help="Don't put files/directories into cache.",
-    )
-    repro_parser.add_argument(
-        "--no-run-cache",
-        action="store_true",
-        default=False,
-        help=(
-            "Execute stage commands even if they have already been run with "
-            "the same command/dependencies/outputs/etc before."
-        ),
-    )
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/root.py

@@ -1,8 +1,7 @@
 import argparse
 import logging
 
-from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
@@ -11,9 +10,8 @@ logger = logging.getLogger(__name__)
 class CmdRoot(CmdBaseNoRepo):
     def run(self):
         from dvc.repo import Repo
-        from dvc.ui import ui
 
-        ui.write(relpath(Repo.find_root()))
+        logger.info(relpath(Repo.find_root()))
         return 0
 
 

dvc/command/run.py

@@ -0,0 +1,256 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+class CmdRun(CmdBase):
+    def run(self):
+        if not any(
+            [
+                self.args.deps,
+                self.args.outs,
+                self.args.outs_no_cache,
+                self.args.metrics,
+                self.args.metrics_no_cache,
+                self.args.plots,
+                self.args.plots_no_cache,
+                self.args.outs_persist,
+                self.args.outs_persist_no_cache,
+                self.args.checkpoints,
+                self.args.params,
+                self.args.command,
+            ]
+        ):  # pragma: no cover
+            logger.error(
+                "too few arguments. Specify at least one: `-d`, `-o`, `-O`, "
+                "`-m`, `-M`, `-p`, `--plots`, `--plots-no-cache`, "
+                "`--outs-persist`, `--outs-persist-no-cache`, `command`."
+            )
+            return 1
+
+        try:
+            self.repo.run(
+                cmd=self._parsed_cmd(),
+                outs=self.args.outs,
+                outs_no_cache=self.args.outs_no_cache,
+                metrics=self.args.metrics,
+                metrics_no_cache=self.args.metrics_no_cache,
+                plots=self.args.plots,
+                plots_no_cache=self.args.plots_no_cache,
+                deps=self.args.deps,
+                params=self.args.params,
+                fname=self.args.file,
+                wdir=self.args.wdir,
+                no_exec=(self.args.no_exec or bool(self.args.checkpoints)),
+                force=self.args.force,
+                run_cache=not self.args.no_run_cache,
+                no_commit=self.args.no_commit,
+                outs_persist=self.args.outs_persist,
+                outs_persist_no_cache=self.args.outs_persist_no_cache,
+                checkpoints=self.args.checkpoints,
+                always_changed=self.args.always_changed,
+                name=self.args.name,
+                single_stage=self.args.single_stage,
+                external=self.args.external,
+                desc=self.args.desc,
+            )
+        except DvcException:
+            logger.exception("")
+            return 1
+
+        return 0
+
+    def _parsed_cmd(self):
+        """
+        We need to take into account two cases:
+
+        - ['python code.py foo bar']: Used mainly with dvc as a library
+        - ['echo', 'foo bar']: List of arguments received from the CLI
+
+        The second case would need quoting, as it was passed through:
+                dvc run echo "foo bar"
+        """
+        if len(self.args.command) < 2:
+            return " ".join(self.args.command)
+
+        return " ".join(self._quote_argument(arg) for arg in self.args.command)
+
+    def _quote_argument(self, argument):
+        if " " not in argument or '"' in argument:
+            return argument
+
+        return f'"{argument}"'
+
+
+def add_parser(subparsers, parent_parser):
+    RUN_HELP = "Generate a stage file from a command and execute the command."
+    run_parser = subparsers.add_parser(
+        "run",
+        parents=[parent_parser],
+        description=append_doc_link(RUN_HELP, "run"),
+        help=RUN_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    run_parser.add_argument(
+        "-d",
+        "--deps",
+        action="append",
+        default=[],
+        help="Declare dependencies for reproducible cmd.",
+        metavar="<path>",
+    ).complete = completion.FILE
+    run_parser.add_argument(
+        "-n", "--name", help="Stage name.",
+    )
+    run_parser.add_argument(
+        "-o",
+        "--outs",
+        action="append",
+        default=[],
+        help="Declare output file or directory.",
+        metavar="<filename>",
+    ).complete = completion.FILE
+    run_parser.add_argument(
+        "-O",
+        "--outs-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory "
+        "(do not put into DVC cache).",
+        metavar="<filename>",
+    ).complete = completion.FILE
+    run_parser.add_argument(
+        "-p",
+        "--params",
+        action="append",
+        default=[],
+        help="Declare parameter to use as additional dependency.",
+        metavar="[<filename>:]<params_list>",
+    ).complete = completion.FILE
+    run_parser.add_argument(
+        "-m",
+        "--metrics",
+        action="append",
+        default=[],
+        help="Declare output metrics file.",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "-M",
+        "--metrics-no-cache",
+        action="append",
+        default=[],
+        help="Declare output metrics file (do not put into DVC cache).",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "--plots",
+        action="append",
+        default=[],
+        help="Declare output plot file.",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "--plots-no-cache",
+        action="append",
+        default=[],
+        help="Declare output plot file (do not put into DVC cache).",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "--file", metavar="<filename>", help=argparse.SUPPRESS,
+    )
+    run_parser.add_argument(
+        "-w",
+        "--wdir",
+        help="Directory within your repo to run your command in.",
+        metavar="<path>",
+    )
+    run_parser.add_argument(
+        "--no-exec",
+        action="store_true",
+        default=False,
+        help="Only create stage file without actually running it.",
+    )
+    run_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Overwrite existing stage",
+    )
+    run_parser.add_argument(
+        "--no-run-cache",
+        action="store_true",
+        default=False,
+        help=(
+            "Execute the command even if this stage has already been run "
+            "with the same command/dependencies/outputs/etc before."
+        ),
+    )
+    run_parser.add_argument(
+        "--no-commit",
+        action="store_true",
+        default=False,
+        help="Don't put files/directories into cache.",
+    )
+    run_parser.add_argument(
+        "--outs-persist",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro.",
+        metavar="<filename>",
+    )
+    run_parser.add_argument(
+        "--outs-persist-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro (do not put into DVC cache).",
+        metavar="<filename>",
+    )
+    run_parser.add_argument(
+        "-c",
+        "--checkpoints",
+        action="append",
+        default=[],
+        help=argparse.SUPPRESS,
+        metavar="<filename>",
+    ).complete = completion.FILE
+    run_parser.add_argument(
+        "--always-changed",
+        action="store_true",
+        default=False,
+        help="Always consider this DVC-file as changed.",
+    )
+    run_parser.add_argument(
+        "--single-stage",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    run_parser.add_argument(
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow outputs that are outside of the DVC repository.",
+    )
+    run_parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the stage (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
+    run_parser.add_argument(
+        "command", nargs=argparse.REMAINDER, help="Command to execute."
+    )
+    run_parser.set_defaults(func=CmdRun)

dvc/command/status.py

@@ -1,8 +1,7 @@
 import logging
 
-from dvc.commands.data_sync import CmdDataBase
+from dvc.command.data_sync import CmdDataBase
 from dvc.exceptions import DvcException
-from dvc.ui import ui
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
@@ -27,7 +26,7 @@ class CmdDataStatus(CmdDataBase):
         ind = indent * self.STATUS_INDENT
 
         if isinstance(status, str):
-            ui.write(f"{ind}{status}")
+            logger.info(f"{ind}{status}")
             return
 
         if isinstance(status, list):
@@ -39,53 +38,46 @@ class CmdDataStatus(CmdDataBase):
 
         for key, value in status.items():
             if isinstance(value, str):
-                ui.write(f"{ind}{self._normalize(value)}{key}")
+                logger.info("{}{}{}".format(ind, self._normalize(value), key))
             elif value:
-                ui.write(f"{ind}{key}:")
+                logger.info(f"{ind}{key}:")
                 self._show(value, indent + 1)
 
     def run(self):
-        from dvc.repo import lock_repo
-
         indent = 1 if self.args.cloud else 0
-
-        with lock_repo(self.repo):
-            try:
-                st = self.repo.status(
-                    targets=self.args.targets,
-                    jobs=self.args.jobs,
-                    cloud=self.args.cloud,
-                    remote=self.args.remote,
-                    all_branches=self.args.all_branches,
-                    all_tags=self.args.all_tags,
-                    all_commits=self.args.all_commits,
-                    with_deps=self.args.with_deps,
-                    recursive=self.args.recursive,
-                )
-            except DvcException:
-                logger.exception("")
-                return 1
+        try:
+            st = self.repo.status(
+                targets=self.args.targets,
+                jobs=self.args.jobs,
+                cloud=self.args.cloud,
+                remote=self.args.remote,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
+                with_deps=self.args.with_deps,
+                recursive=self.args.recursive,
+            )
 
             if self.args.quiet:
                 return bool(st)
 
-            if self.args.json:
-                ui.write_json(st)
-                return 0
+            if self.args.show_json:
+                import json
 
-            if st:
+                logger.info(json.dumps(st))
+            elif st:
                 self._show(st, indent)
-                return 0
-
-            # additional hints for the user
-            if not self.repo.index.stages:
-                ui.write(self.EMPTY_PROJECT_MSG)
+            elif not self.repo.stages:
+                logger.info(self.EMPTY_PROJECT_MSG)
             elif self.args.cloud or self.args.remote:
                 remote = self.args.remote or self.repo.config["core"].get(
                     "remote"
                 )
-                ui.write(self.IN_SYNC_MSG.format(remote=remote))
+                logger.info(self.IN_SYNC_MSG.format(remote=remote))
             else:
-                ui.write(self.UP_TO_DATE_MSG)
+                logger.info(self.UP_TO_DATE_MSG)
 
+        except DvcException:
+            logger.exception("")
+            return 1
         return 0

dvc/command/unprotect.py

@@ -1,9 +1,8 @@
 import argparse
 import logging
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -24,7 +23,7 @@ class CmdUnprotect(CmdBase):
 def add_parser(subparsers, parent_parser):
     UNPROTECT_HELP = (
         "Unprotect tracked files or directories (when hardlinks or symlinks "
-        "have been enabled with `dvc config cache.type`)."
+        "have been enabled with `dvc config cache.type`)"
     )
     unprotect_parser = subparsers.add_parser(
         "unprotect",
@@ -34,6 +33,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
-        "targets", nargs="+", help="Data files/directories to unprotect."
+        "targets", nargs="+", help="Data files/directories to unprotect.",
     ).complete = completion.FILE
     unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/command/update.py

@@ -1,9 +1,8 @@
 import argparse
 import logging
 
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -17,9 +16,6 @@ class CmdUpdate(CmdBase):
                 targets=self.args.targets,
                 rev=self.args.rev,
                 recursive=self.args.recursive,
-                to_remote=self.args.to_remote,
-                remote=self.args.remote,
-                jobs=self.args.jobs,
             )
         except DvcException:
             logger.exception("failed update data")
@@ -37,7 +33,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     update_parser.add_argument(
-        "targets", nargs="+", help=".dvc files to update."
+        "targets", nargs="+", help="DVC-files to update.",
     ).complete = completion.DVC_FILE
     update_parser.add_argument(
         "--rev",
@@ -52,26 +48,4 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Update all stages in the specified directory.",
     )
-    update_parser.add_argument(
-        "--to-remote",
-        action="store_true",
-        default=False,
-        help="Update data directly on the remote",
-    )
-    update_parser.add_argument(
-        "-r",
-        "--remote",
-        help="Remote storage to perform updates to",
-        metavar="<name>",
-    )
-    update_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        help=(
-            "Number of jobs to run simultaneously. "
-            "The default value is 4 * cpu_count(). "
-        ),
-        metavar="<number>",
-    )
     update_parser.set_defaults(func=CmdUpdate)

dvc/command/version.py

@@ -1,22 +1,16 @@
 import argparse
 import logging
 
-from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
-from dvc.ui import ui
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.info import get_dvc_info
 
 logger = logging.getLogger(__name__)
 
 
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
-        from dvc.info import get_dvc_info
-        from dvc.updater import notify_updates
-
         dvc_info = get_dvc_info()
-        ui.write(dvc_info, force=True)
-
-        notify_updates()
+        logger.info(dvc_info)
         return 0
 
 

dvc/commands/config.py

@@ -1,246 +0,0 @@
-import argparse
-import logging
-import os
-
-from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
-from dvc.ui import ui
-
-logger = logging.getLogger(__name__)
-
-NAME_REGEX = r"^(?P<remote>remote\.)?(?P<section>[^\.]*)\.(?P<option>[^\.]*)$"
-
-
-def _name_type(value):
-    import re
-
-    match = re.match(NAME_REGEX, value)
-    if not match:
-        raise argparse.ArgumentTypeError(
-            "name argument should look like "
-            "remote.name.option or "
-            "section.option"
-        )
-    return (
-        bool(match.group("remote")),
-        match.group("section").lower(),
-        match.group("option").lower(),
-    )
-
-
-class CmdConfig(CmdBaseNoRepo):
-    def __init__(self, args):
-        from dvc.config import Config
-
-        super().__init__(args)
-
-        self.config = Config(validate=False)
-
-    def run(self):
-        if self.args.show_origin:
-            if any((self.args.value, self.args.unset)):
-                logger.error(
-                    "--show-origin can't be used together with any of these "
-                    "options: -u/--unset, value"
-                )
-                return 1
-
-        if self.args.list:
-            return self._list()
-
-        if self.args.name is None:
-            logger.error("name argument is required")
-            return 1
-
-        remote, section, opt = self.args.name
-
-        if self.args.value is None and not self.args.unset:
-            return self._get(remote, section, opt)
-
-        return self._set(remote, section, opt)
-
-    def _list(self):
-        if any((self.args.name, self.args.value, self.args.unset)):
-            logger.error(
-                "-l/--list can't be used together with any of these "
-                "options: -u/--unset, name, value"
-            )
-            return 1
-
-        levels = self._get_appropriate_levels(self.args.level)
-
-        for level in levels:
-            conf = self.config.read(level)
-            prefix = self._config_file_prefix(
-                self.args.show_origin, self.config, level
-            )
-            configs = list(self._format_config(conf, prefix))
-            if configs:
-                ui.write("\n".join(configs))
-
-        return 0
-
-    def _get(self, remote, section, opt):
-        from dvc.config import ConfigError
-
-        levels = self._get_appropriate_levels(self.args.level)[::-1]
-
-        for level in levels:
-            conf = self.config.read(level)
-            if remote:
-                conf = conf["remote"]
-
-            try:
-                self._check(conf, remote, section, opt)
-            except ConfigError:
-                if self.args.level:
-                    raise
-            else:
-                prefix = self._config_file_prefix(
-                    self.args.show_origin, self.config, level
-                )
-                ui.write(prefix, conf[section][opt], sep="")
-                break
-
-        return 0
-
-    def _set(self, remote, section, opt):
-        with self.config.edit(self.args.level) as conf:
-            if remote:
-                conf = conf["remote"]
-            if self.args.unset:
-                self._check(conf, remote, section, opt)
-                del conf[section][opt]
-            else:
-                self._check(conf, remote, section)
-                conf[section][opt] = self.args.value
-
-        if self.args.name == "cache.type":
-            logger.warning(
-                "You have changed the 'cache.type' option. This doesn't update"
-                " any existing workspace file links, but it can be done with:"
-                "\n             dvc checkout --relink"
-            )
-
-        return 0
-
-    def _check(self, conf, remote, section, opt=None):
-        from dvc.config import ConfigError
-
-        name = "remote" if remote else "section"
-        if section not in conf:
-            raise ConfigError(f"{name} '{section}' doesn't exist")
-
-        if opt and opt not in conf[section]:
-            raise ConfigError(
-                f"option '{opt}' doesn't exist in {name} '{section}'"
-            )
-
-    def _get_appropriate_levels(self, levels):
-        if levels:
-            self._validate_level_for_non_repo_operation(levels)
-            return [levels]
-        if self.config.dvc_dir is None:
-            return self.config.SYSTEM_LEVELS
-        return self.config.LEVELS
-
-    def _validate_level_for_non_repo_operation(self, level):
-        from dvc.config import ConfigError
-
-        if self.config.dvc_dir is None and level in self.config.REPO_LEVELS:
-            raise ConfigError("Not inside a DVC repo")
-
-    @staticmethod
-    def _format_config(config, prefix=""):
-        from dvc.utils.flatten import flatten
-
-        for key, value in flatten(config).items():
-            yield f"{prefix}{key}={value}"
-
-    @staticmethod
-    def _config_file_prefix(show_origin, config, level):
-        from dvc.repo import Repo
-
-        if not show_origin:
-            return ""
-
-        level = level or "repo"
-        fname = config.files[level]
-
-        if level in ["local", "repo"]:
-            fname = os.path.relpath(fname, start=Repo.find_root())
-
-        return fname + "\t"
-
-
-parent_config_parser = argparse.ArgumentParser(add_help=False)
-level_group = parent_config_parser.add_mutually_exclusive_group()
-level_group.add_argument(
-    "--global",
-    dest="level",
-    action="store_const",
-    const="global",
-    help="Use global config.",
-)
-level_group.add_argument(
-    "--system",
-    dest="level",
-    action="store_const",
-    const="system",
-    help="Use system config.",
-)
-level_group.add_argument(
-    "--project",
-    dest="level",
-    action="store_const",
-    const="repo",
-    help="Use project config (.dvc/config).",
-)
-level_group.add_argument(
-    "--local",
-    dest="level",
-    action="store_const",
-    const="local",
-    help="Use local config (.dvc/config.local).",
-)
-parent_config_parser.set_defaults(level=None)
-
-
-def add_parser(subparsers, parent_parser):
-    CONFIG_HELP = "Get or set config options."
-
-    config_parser = subparsers.add_parser(
-        "config",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(CONFIG_HELP, "config"),
-        help=CONFIG_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    config_parser.add_argument(
-        "-u",
-        "--unset",
-        default=False,
-        action="store_true",
-        help="Unset option.",
-    )
-    config_parser.add_argument(
-        "name",
-        nargs="?",
-        type=_name_type,
-        help="Option name (section.option or remote.name.option).",
-    )
-    config_parser.add_argument("value", nargs="?", help="Option value.")
-    config_parser.add_argument(
-        "-l",
-        "--list",
-        default=False,
-        action="store_true",
-        help="List all defined config values.",
-    )
-    config_parser.add_argument(
-        "--show-origin",
-        default=False,
-        action="store_true",
-        help="Show the source file containing each config value.",
-    )
-    config_parser.set_defaults(func=CmdConfig)

dvc/commands/experiments/__init__.py

@@ -1,101 +0,0 @@
-import argparse
-
-from dvc.cli.utils import (
-    append_doc_link,
-    fix_plumbing_subparsers,
-    fix_subparsers,
-)
-from dvc.commands.experiments import (
-    apply,
-    branch,
-    diff,
-    exec_run,
-    gc,
-    init,
-    ls,
-    pull,
-    push,
-    remove,
-    run,
-    show,
-)
-
-SUB_COMMANDS = [
-    apply,
-    branch,
-    diff,
-    exec_run,
-    gc,
-    init,
-    ls,
-    pull,
-    push,
-    remove,
-    run,
-    show,
-]
-
-
-def add_parser(subparsers, parent_parser):
-    EXPERIMENTS_HELP = "Commands to run and compare experiments."
-
-    experiments_parser = subparsers.add_parser(
-        "experiments",
-        parents=[parent_parser],
-        aliases=["exp"],
-        description=append_doc_link(EXPERIMENTS_HELP, "exp"),
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        help=EXPERIMENTS_HELP,
-    )
-
-    experiments_subparsers = experiments_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc experiments CMD --help` to display "
-        "command-specific help.",
-    )
-
-    fix_subparsers(experiments_subparsers)
-    for cmd in SUB_COMMANDS:
-        cmd.add_parser(experiments_subparsers, parent_parser)
-    fix_plumbing_subparsers(experiments_subparsers)
-
-
-def add_rev_selection_flags(
-    experiments_subcmd_parser, command: str, default: bool = True
-):
-    experiments_subcmd_parser.add_argument(
-        "-A",
-        "--all-commits",
-        action="store_true",
-        default=False,
-        help=(
-            f"{command} all experiments in the repository "
-            "(overrides `--rev` and `--num`)."
-        ),
-    )
-    default_msg = " (HEAD by default)" if default else ""
-    msg = (
-        f"{command} experiments derived from the specified `<commit>` as "
-        f"baseline{default_msg}."
-    )
-    experiments_subcmd_parser.add_argument(
-        "--rev",
-        type=str,
-        default=None,
-        help=msg,
-        metavar="<commit>",
-    )
-    experiments_subcmd_parser.add_argument(
-        "-n",
-        "--num",
-        type=int,
-        default=1,
-        dest="num",
-        metavar="<num>",
-        help=(
-            f"{command} experiments from the last `num` commits "
-            "(first parents) starting from the `--rev` baseline. "
-            "Give a negative value to include all first-parent commits "
-            "(similar to `git log -n`)."
-        ),
-    )

dvc/commands/experiments/apply.py

@@ -1,42 +0,0 @@
-import argparse
-import logging
-
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-
-logger = logging.getLogger(__name__)
-
-
-class CmdExperimentsApply(CmdBase):
-    def run(self):
-
-        self.repo.experiments.apply(
-            self.args.experiment, force=self.args.force
-        )
-
-        return 0
-
-
-def add_parser(experiments_subparsers, parent_parser):
-
-    EXPERIMENTS_APPLY_HELP = (
-        "Apply the changes from an experiment to your workspace."
-    )
-    experiments_apply_parser = experiments_subparsers.add_parser(
-        "apply",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_APPLY_HELP, "exp/apply"),
-        help=EXPERIMENTS_APPLY_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    experiments_apply_parser.add_argument(
-        "--no-force",
-        action="store_false",
-        dest="force",
-        help="Fail if this command would overwrite conflicting changes.",
-    )
-    experiments_apply_parser.add_argument(
-        "experiment", help="Experiment to be applied."
-    ).complete = completion.EXPERIMENT
-    experiments_apply_parser.set_defaults(func=CmdExperimentsApply)

dvc/commands/experiments/branch.py

@@ -1,34 +0,0 @@
-import argparse
-import logging
-
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-
-logger = logging.getLogger(__name__)
-
-
-class CmdExperimentsBranch(CmdBase):
-    def run(self):
-
-        self.repo.experiments.branch(self.args.experiment, self.args.branch)
-
-        return 0
-
-
-def add_parser(experiments_subparsers, parent_parser):
-
-    EXPERIMENTS_BRANCH_HELP = "Promote an experiment to a Git branch."
-    experiments_branch_parser = experiments_subparsers.add_parser(
-        "branch",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_BRANCH_HELP, "exp/branch"),
-        help=EXPERIMENTS_BRANCH_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    experiments_branch_parser.add_argument(
-        "experiment", help="Experiment to be promoted."
-    )
-    experiments_branch_parser.add_argument(
-        "branch", help="Git branch name to use."
-    )
-    experiments_branch_parser.set_defaults(func=CmdExperimentsBranch)

dvc/commands/experiments/diff.py

@@ -1,118 +0,0 @@
-import argparse
-import logging
-
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.commands.metrics import DEFAULT_PRECISION
-from dvc.exceptions import DvcException
-from dvc.ui import ui
-
-logger = logging.getLogger(__name__)
-
-
-class CmdExperimentsDiff(CmdBase):
-    def run(self):
-
-        try:
-            diff = self.repo.experiments.diff(
-                a_rev=self.args.a_rev,
-                b_rev=self.args.b_rev,
-                all=self.args.all,
-                param_deps=self.args.param_deps,
-            )
-        except DvcException:
-            logger.exception("failed to show experiments diff")
-            return 1
-
-        if self.args.json:
-            ui.write_json(diff)
-        else:
-            from dvc.compare import show_diff
-
-            precision = self.args.precision or DEFAULT_PRECISION
-            diffs = [("metrics", "Metric"), ("params", "Param")]
-            for idx, (key, title) in enumerate(diffs):
-                if idx:
-                    # we are printing tables even in `--quiet` mode
-                    # so we should also be printing the "table" separator
-                    ui.write(force=True)
-
-                show_diff(
-                    diff[key],
-                    title=title,
-                    markdown=self.args.markdown,
-                    no_path=self.args.no_path,
-                    on_empty_diff="diff not supported",
-                    precision=precision if key == "metrics" else None,
-                    a_rev=self.args.a_rev,
-                    b_rev=self.args.b_rev,
-                )
-
-        return 0
-
-
-def add_parser(experiments_subparsers, parent_parser):
-
-    EXPERIMENTS_DIFF_HELP = (
-        "Show changes between experiments in the DVC repository."
-    )
-
-    experiments_diff_parser = experiments_subparsers.add_parser(
-        "diff",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_DIFF_HELP, "exp/diff"),
-        help=EXPERIMENTS_DIFF_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    experiments_diff_parser.add_argument(
-        "a_rev", nargs="?", help="Old experiment to compare (defaults to HEAD)"
-    ).complete = completion.EXPERIMENT
-    experiments_diff_parser.add_argument(
-        "b_rev",
-        nargs="?",
-        help="New experiment to compare (defaults to the current workspace)",
-    ).complete = completion.EXPERIMENT
-    experiments_diff_parser.add_argument(
-        "--all",
-        action="store_true",
-        default=False,
-        help="Show unchanged metrics/params as well.",
-    )
-    experiments_diff_parser.add_argument(
-        "--param-deps",
-        action="store_true",
-        default=False,
-        help="Show only params that are stage dependencies.",
-    )
-    experiments_diff_parser.add_argument(
-        "--json",
-        "--show-json",
-        action="store_true",
-        default=False,
-        help="Show output in JSON format.",
-    )
-    experiments_diff_parser.add_argument(
-        "--md",
-        "--show-md",
-        action="store_true",
-        default=False,
-        dest="markdown",
-        help="Show tabulated output in the Markdown format (GFM).",
-    )
-    experiments_diff_parser.add_argument(
-        "--no-path",
-        action="store_true",
-        default=False,
-        help="Don't show metric/param path.",
-    )
-    experiments_diff_parser.add_argument(
-        "--precision",
-        type=int,
-        help=(
-            "Round metrics/params to `n` digits precision after the decimal "
-            f"point. Rounds to {DEFAULT_PRECISION} digits by default."
-        ),
-        metavar="<n>",
-    )
-    experiments_diff_parser.set_defaults(func=CmdExperimentsDiff)

dvc/commands/experiments/exec_run.py

@@ -1,42 +0,0 @@
-import logging
-
-from dvc.cli.command import CmdBaseNoRepo
-
-logger = logging.getLogger(__name__)
-
-
-class CmdExecutorRun(CmdBaseNoRepo):
-    """Run an experiment executor."""
-
-    def run(self):
-        from dvc.repo.experiments.executor.base import (
-            BaseExecutor,
-            ExecutorInfo,
-        )
-        from dvc.utils.serialize import load_json
-
-        info = ExecutorInfo.from_dict(load_json(self.args.infofile))
-        BaseExecutor.reproduce(
-            info=info,
-            rev="",
-            queue=None,
-            log_level=logger.getEffectiveLevel(),
-            infofile=self.args.infofile,
-        )
-        return 0
-
-
-def add_parser(experiments_subparsers, parent_parser):
-    EXEC_RUN_HELP = "Run an experiment executor."
-    exec_run_parser = experiments_subparsers.add_parser(
-        "exec-run",
-        parents=[parent_parser],
-        description=EXEC_RUN_HELP,
-        add_help=False,
-    )
-    exec_run_parser.add_argument(
-        "--infofile",
-        help="Path to executor info file",
-        default=None,
-    )
-    exec_run_parser.set_defaults(func=CmdExecutorRun)

dvc/commands/experiments/gc.py

@@ -1,129 +0,0 @@
-import argparse
-import logging
-
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.exceptions import InvalidArgumentError
-from dvc.ui import ui
-
-logger = logging.getLogger(__name__)
-
-
-def _raise_error_if_all_disabled(**kwargs):
-    if not any(kwargs.values()):
-        raise InvalidArgumentError(
-            "Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags` "
-            "or `--all-commits` needs to be set."
-        )
-
-
-class CmdExperimentsGC(CmdBase):
-    def run(self):
-        _raise_error_if_all_disabled(
-            all_branches=self.args.all_branches,
-            all_tags=self.args.all_tags,
-            all_commits=self.args.all_commits,
-            workspace=self.args.workspace,
-        )
-
-        msg = "This will remove all experiments except those derived from "
-
-        msg += "the workspace"
-        if self.args.all_commits:
-            msg += " and all git commits"
-        elif self.args.all_branches and self.args.all_tags:
-            msg += " and all git branches and tags"
-        elif self.args.all_branches:
-            msg += " and all git branches"
-        elif self.args.all_tags:
-            msg += " and all git tags"
-        msg += " of the current repo."
-        if self.args.queued:
-            msg += " Run queued experiments will be preserved."
-        else:
-            msg += " Run queued experiments will be removed."
-
-        logger.warning(msg)
-
-        msg = "Are you sure you want to proceed?"
-        if not self.args.force and not ui.confirm(msg):
-            return 1
-
-        removed = self.repo.experiments.gc(
-            all_branches=self.args.all_branches,
-            all_tags=self.args.all_tags,
-            all_commits=self.args.all_commits,
-            workspace=self.args.workspace,
-            queued=self.args.queued,
-        )
-
-        if removed:
-            ui.write(
-                f"Removed {removed} experiments.",
-                "To remove unused cache files",
-                "use 'dvc gc'.",
-            )
-        else:
-            ui.write("No experiments to remove.")
-        return 0
-
-
-def add_parser(experiments_subparsers, parent_parser):
-
-    EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments."
-    EXPERIMENTS_GC_DESCRIPTION = (
-        "Removes all experiments which are not derived from the specified"
-        "Git revisions."
-    )
-    experiments_gc_parser = experiments_subparsers.add_parser(
-        "gc",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_GC_DESCRIPTION, "exp/gc"),
-        help=EXPERIMENTS_GC_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    experiments_gc_parser.add_argument(
-        "-w",
-        "--workspace",
-        action="store_true",
-        default=False,
-        help="Keep experiments derived from the current workspace.",
-    )
-    experiments_gc_parser.add_argument(
-        "-a",
-        "--all-branches",
-        action="store_true",
-        default=False,
-        help="Keep experiments derived from the tips of all Git branches.",
-    )
-    experiments_gc_parser.add_argument(
-        "-T",
-        "--all-tags",
-        action="store_true",
-        default=False,
-        help="Keep experiments derived from all Git tags.",
-    )
-    experiments_gc_parser.add_argument(
-        "-A",
-        "--all-commits",
-        action="store_true",
-        default=False,
-        help="Keep experiments derived from all Git commits.",
-    )
-    experiments_gc_parser.add_argument(
-        "--queued",
-        action="store_true",
-        default=False,
-        help=(
-            "Keep queued experiments (experiments run queue will be cleared "
-            "by default)."
-        ),
-    )
-    experiments_gc_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Force garbage collection - automatically agree to all prompts.",
-    )
-    experiments_gc_parser.set_defaults(func=CmdExperimentsGC)

dvc/commands/experiments/init.py

@@ -1,210 +0,0 @@
-import argparse
-import logging
-from typing import TYPE_CHECKING, List
-
-from funcy import compact
-
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.exceptions import InvalidArgumentError
-from dvc.ui import ui
-
-if TYPE_CHECKING:
-    from dvc.dependency import Dependency
-    from dvc.stage import PipelineStage
-
-
-logger = logging.getLogger(__name__)
-
-
-class CmdExperimentsInit(CmdBase):
-    DEFAULT_NAME = "train"
-    CODE = "src"
-    DATA = "data"
-    MODELS = "models"
-    DEFAULT_METRICS = "metrics.json"
-    DEFAULT_PARAMS = "params.yaml"
-    PLOTS = "plots"
-    DEFAULTS = {
-        "code": CODE,
-        "data": DATA,
-        "models": MODELS,
-        "metrics": DEFAULT_METRICS,
-        "params": DEFAULT_PARAMS,
-        "plots": PLOTS,
-    }
-
-    def run(self):
-        from dvc.commands.stage import parse_cmd
-
-        cmd = parse_cmd(self.args.command)
-        if not self.args.interactive and not cmd:
-            raise InvalidArgumentError("command is not specified")
-
-        from dvc.repo.experiments.init import init
-
-        defaults = {}
-        if not self.args.explicit:
-            config = self.repo.config["exp"]
-            defaults.update({**self.DEFAULTS, **config})
-
-        cli_args = compact(
-            {
-                "cmd": cmd,
-                "code": self.args.code,
-                "data": self.args.data,
-                "models": self.args.models,
-                "metrics": self.args.metrics,
-                "params": self.args.params,
-                "plots": self.args.plots,
-                "live": self.args.live,
-            }
-        )
-
-        initialized_stage, initialized_deps, initialized_out_dirs = init(
-            self.repo,
-            name=self.args.name,
-            type=self.args.type,
-            defaults=defaults,
-            overrides=cli_args,
-            interactive=self.args.interactive,
-            force=self.args.force,
-        )
-        self._post_init_display(
-            initialized_stage, initialized_deps, initialized_out_dirs
-        )
-        if self.args.run:
-            self.repo.experiments.run(targets=[initialized_stage.addressing])
-        return 0
-
-    def _post_init_display(
-        self,
-        stage: "PipelineStage",
-        new_deps: List["Dependency"],
-        new_out_dirs: List[str],
-    ) -> None:
-        from dvc.utils import humanize
-
-        path_fmt = "[green]{}[/green]".format
-        if new_deps:
-            deps_paths = humanize.join(map(path_fmt, new_deps))
-            ui.write(f"Creating dependencies: {deps_paths}", styled=True)
-
-        if new_out_dirs:
-            out_dirs_paths = humanize.join(map(path_fmt, new_out_dirs))
-            ui.write(
-                f"Creating output directories: {out_dirs_paths}", styled=True
-            )
-
-        ui.write(
-            f"Creating [b]{self.args.name}[/b] stage in [green]dvc.yaml[/]",
-            styled=True,
-        )
-        if stage.outs or not self.args.run:
-            # separate the above status-like messages with help/tips section
-            ui.write(styled=True)
-
-        if stage.outs:
-            outs_paths = humanize.join(map(path_fmt, stage.outs))
-            tips = f"Ensure your experiment command creates {outs_paths}."
-            ui.write(tips, styled=True)
-
-        if not self.args.run:
-            ui.write(
-                'You can now run your experiment using [b]"dvc exp run"[/].',
-                styled=True,
-            )
-        else:
-            # separate between `exp.run` output and `dvc exp init` output
-            ui.write(styled=True)
-
-
-def add_parser(experiments_subparsers, parent_parser):
-
-    EXPERIMENTS_INIT_HELP = "Quickly setup any project to use experiments."
-    experiments_init_parser = experiments_subparsers.add_parser(
-        "init",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_INIT_HELP, "exp/init"),
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        help=EXPERIMENTS_INIT_HELP,
-    )
-    experiments_init_parser.add_argument(
-        "command",
-        nargs=argparse.REMAINDER,
-        help="Command to execute.",
-        metavar="command",
-    )
-    experiments_init_parser.add_argument(
-        "--run",
-        action="store_true",
-        help="Run the experiment after initializing it",
-    )
-    experiments_init_parser.add_argument(
-        "--interactive",
-        "-i",
-        action="store_true",
-        help="Prompt for values that are not provided",
-    )
-    experiments_init_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Overwrite existing stage",
-    )
-    experiments_init_parser.add_argument(
-        "--explicit",
-        action="store_true",
-        default=False,
-        help="Only use the path values explicitly provided",
-    )
-    experiments_init_parser.add_argument(
-        "--name",
-        "-n",
-        help="Name of the stage to create (default: %(default)s)",
-        default=CmdExperimentsInit.DEFAULT_NAME,
-    )
-    experiments_init_parser.add_argument(
-        "--code",
-        help="Path to the source file or directory "
-        "which your experiments depend"
-        f" (default: {CmdExperimentsInit.CODE})",
-    )
-    experiments_init_parser.add_argument(
-        "--data",
-        help="Path to the data file or directory "
-        "which your experiments depend"
-        f" (default: {CmdExperimentsInit.DATA})",
-    )
-    experiments_init_parser.add_argument(
-        "--models",
-        help="Path to the model file or directory for your experiments"
-        f" (default: {CmdExperimentsInit.MODELS})",
-    )
-    experiments_init_parser.add_argument(
-        "--params",
-        help="Path to the parameters file for your experiments"
-        f" (default: {CmdExperimentsInit.DEFAULT_PARAMS})",
-    )
-    experiments_init_parser.add_argument(
-        "--metrics",
-        help="Path to the metrics file for your experiments"
-        f" (default: {CmdExperimentsInit.DEFAULT_METRICS})",
-    )
-    experiments_init_parser.add_argument(
-        "--plots",
-        help="Path to the plots file or directory for your experiments"
-        f" (default: {CmdExperimentsInit.PLOTS})",
-    )
-    experiments_init_parser.add_argument(
-        "--live",
-        help="Path to log dvclive outputs for your experiments",
-    )
-    experiments_init_parser.add_argument(
-        "--type",
-        choices=["default", "checkpoint"],
-        default="default",
-        help="Select type of stage to create (default: %(default)s)",
-    )
-    experiments_init_parser.set_defaults(func=CmdExperimentsInit)

dvc/commands/experiments/ls.py

@@ -1,64 +0,0 @@
-import argparse
-import logging
-
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-
-logger = logging.getLogger(__name__)
-
-
-class CmdExperimentsList(CmdBase):
-    def run(self):
-        name_only = self.args.name_only
-        exps = self.repo.experiments.ls(
-            all_commits=self.args.all_commits,
-            rev=self.args.rev,
-            num=self.args.num,
-            git_remote=self.args.git_remote,
-        )
-        for baseline in exps:
-            tag = self.repo.scm.describe(baseline)
-            if not tag:
-                branch = self.repo.scm.describe(baseline, base="refs/heads")
-                if branch:
-                    tag = branch.split("/")[-1]
-            name = tag if tag else baseline[:7]
-            if not name_only:
-                print(f"{name}:")
-            for exp_name in exps[baseline]:
-                indent = "" if name_only else "\t"
-                print(f"{indent}{exp_name}")
-
-        return 0
-
-
-def add_parser(experiments_subparsers, parent_parser):
-    from . import add_rev_selection_flags
-
-    EXPERIMENTS_LIST_HELP = "List local and remote experiments."
-    experiments_list_parser = experiments_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_LIST_HELP, "exp/list"),
-        help=EXPERIMENTS_LIST_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    add_rev_selection_flags(experiments_list_parser, "List")
-    experiments_list_parser.add_argument(
-        "--name-only",
-        "--names-only",
-        action="store_true",
-        help="Only output experiment names (without parent commits).",
-    )
-    experiments_list_parser.add_argument(
-        "git_remote",
-        nargs="?",
-        default=None,
-        help=(
-            "Optional Git remote name or Git URL. "
-            "If provided, experiments from the specified Git repository "
-            " will be listed instead of local ones."
-        ),
-        metavar="<git_remote>",
-    )
-    experiments_list_parser.set_defaults(func=CmdExperimentsList)

dvc/commands/experiments/pull.py

@@ -1,117 +0,0 @@
-import argparse
-import logging
-
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.exceptions import InvalidArgumentError
-from dvc.ui import ui
-
-logger = logging.getLogger(__name__)
-
-
-class CmdExperimentsPull(CmdBase):
-    def raise_error_if_all_disabled(self):
-        if not any(
-            [self.args.experiment, self.args.all_commits, self.args.rev]
-        ):
-            raise InvalidArgumentError(
-                "Either provide an `experiment` argument, or use the "
-                "`--rev` or `--all-commits` flag."
-            )
-
-    def run(self):
-        self.raise_error_if_all_disabled()
-
-        pulled_exps = self.repo.experiments.pull(
-            self.args.git_remote,
-            self.args.experiment,
-            all_commits=self.args.all_commits,
-            rev=self.args.rev,
-            num=self.args.num,
-            force=self.args.force,
-            pull_cache=self.args.pull_cache,
-            dvc_remote=self.args.dvc_remote,
-            jobs=self.args.jobs,
-            run_cache=self.args.run_cache,
-        )
-
-        if pulled_exps:
-            ui.write(
-                f"Pulled experiment '{pulled_exps}'",
-                f"from Git remote '{self.args.git_remote}'.",
-            )
-        else:
-            ui.write("No experiments to pull.")
-        if not self.args.pull_cache:
-            ui.write(
-                "To pull cached outputs for this experiment"
-                "from DVC remote storage,"
-                "re-run this command without '--no-cache'."
-            )
-
-        return 0
-
-
-def add_parser(experiments_subparsers, parent_parser):
-    from . import add_rev_selection_flags
-
-    EXPERIMENTS_PULL_HELP = "Pull an experiment from a Git remote."
-    experiments_pull_parser = experiments_subparsers.add_parser(
-        "pull",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_PULL_HELP, "exp/pull"),
-        help=EXPERIMENTS_PULL_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    add_rev_selection_flags(experiments_pull_parser, "Pull", False)
-    experiments_pull_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        help="Replace local experiment already exists.",
-    )
-    experiments_pull_parser.add_argument(
-        "--no-cache",
-        action="store_false",
-        dest="pull_cache",
-        help=(
-            "Do not pull cached outputs for this experiment from DVC remote "
-            "storage."
-        ),
-    )
-    experiments_pull_parser.add_argument(
-        "-r",
-        "--remote",
-        dest="dvc_remote",
-        metavar="<name>",
-        help="Name of the DVC remote to use when pulling cached outputs.",
-    )
-    experiments_pull_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        metavar="<number>",
-        help=(
-            "Number of jobs to run simultaneously when pulling from DVC "
-            "remote storage."
-        ),
-    )
-    experiments_pull_parser.add_argument(
-        "--run-cache",
-        action="store_true",
-        default=False,
-        help="Pull run history for all stages.",
-    )
-    experiments_pull_parser.add_argument(
-        "git_remote",
-        help="Git remote name or Git URL.",
-        metavar="<git_remote>",
-    )
-    experiments_pull_parser.add_argument(
-        "experiment",
-        nargs="*",
-        default=None,
-        help="Experiments to pull.",
-        metavar="<experiment>",
-    )
-    experiments_pull_parser.set_defaults(func=CmdExperimentsPull)

dvc/commands/experiments/push.py

@@ -1,119 +0,0 @@
-import argparse
-import logging
-
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.exceptions import InvalidArgumentError
-from dvc.ui import ui
-
-logger = logging.getLogger(__name__)
-
-
-class CmdExperimentsPush(CmdBase):
-    def raise_error_if_all_disabled(self):
-        if not any(
-            [self.args.experiment, self.args.all_commits, self.args.rev]
-        ):
-            raise InvalidArgumentError(
-                "Either provide an `experiment` argument, or use the "
-                "`--rev` or `--all-commits` flag."
-            )
-
-    def run(self):
-
-        self.raise_error_if_all_disabled()
-
-        pushed_exps = self.repo.experiments.push(
-            self.args.git_remote,
-            self.args.experiment,
-            all_commits=self.args.all_commits,
-            rev=self.args.rev,
-            num=self.args.num,
-            force=self.args.force,
-            push_cache=self.args.push_cache,
-            dvc_remote=self.args.dvc_remote,
-            jobs=self.args.jobs,
-            run_cache=self.args.run_cache,
-        )
-
-        if pushed_exps:
-            ui.write(
-                f"Pushed experiment '{pushed_exps}'"
-                f"to Git remote '{self.args.git_remote}'."
-            )
-        else:
-            ui.write("No experiments to push.")
-        if not self.args.push_cache:
-            ui.write(
-                "To push cached outputs",
-                "for this experiment to DVC remote storage,"
-                "re-run this command without '--no-cache'.",
-            )
-
-        return 0
-
-
-def add_parser(experiments_subparsers, parent_parser):
-    from . import add_rev_selection_flags
-
-    EXPERIMENTS_PUSH_HELP = "Push a local experiment to a Git remote."
-    experiments_push_parser = experiments_subparsers.add_parser(
-        "push",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_PUSH_HELP, "exp/push"),
-        help=EXPERIMENTS_PUSH_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    add_rev_selection_flags(experiments_push_parser, "Push", False)
-    experiments_push_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        help="Replace experiment in the Git remote if it already exists.",
-    )
-    experiments_push_parser.add_argument(
-        "--no-cache",
-        action="store_false",
-        dest="push_cache",
-        help=(
-            "Do not push cached outputs for this experiment to DVC remote "
-            "storage."
-        ),
-    )
-    experiments_push_parser.add_argument(
-        "-r",
-        "--remote",
-        dest="dvc_remote",
-        metavar="<name>",
-        help="Name of the DVC remote to use when pushing cached outputs.",
-    )
-    experiments_push_parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        metavar="<number>",
-        help=(
-            "Number of jobs to run simultaneously when pushing to DVC remote "
-            "storage."
-        ),
-    )
-    experiments_push_parser.add_argument(
-        "--run-cache",
-        action="store_true",
-        default=False,
-        help="Push run history for all stages.",
-    )
-    experiments_push_parser.add_argument(
-        "git_remote",
-        help="Git remote name or Git URL.",
-        metavar="<git_remote>",
-    )
-    experiments_push_parser.add_argument(
-        "experiment",
-        nargs="*",
-        default=None,
-        help="Experiments to push.",
-        metavar="<experiment>",
-    ).complete = completion.EXPERIMENT
-    experiments_push_parser.set_defaults(func=CmdExperimentsPush)

dvc/commands/experiments/remove.py

@@ -1,73 +0,0 @@
-import argparse
-import logging
-
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.exceptions import InvalidArgumentError
-from dvc.ui import ui
-
-logger = logging.getLogger(__name__)
-
-
-class CmdExperimentsRemove(CmdBase):
-    def raise_error_if_all_disabled(self):
-        if not any(
-            [
-                self.args.experiment,
-                self.args.all_commits,
-                self.args.rev,
-                self.args.queue,
-            ]
-        ):
-            raise InvalidArgumentError(
-                "Either provide an `experiment` argument, or use the "
-                "`--rev` or `--all-commits` flag."
-            )
-
-    def run(self):
-
-        self.raise_error_if_all_disabled()
-
-        removed_list = self.repo.experiments.remove(
-            exp_names=self.args.experiment,
-            all_commits=self.args.all_commits,
-            rev=self.args.rev,
-            num=self.args.num,
-            queue=self.args.queue,
-            git_remote=self.args.git_remote,
-        )
-        removed = ",".join(removed_list)
-        ui.write(f"Removed experiments: {removed}")
-
-        return 0
-
-
-def add_parser(experiments_subparsers, parent_parser):
-    from . import add_rev_selection_flags
-
-    EXPERIMENTS_REMOVE_HELP = "Remove experiments."
-    experiments_remove_parser = experiments_subparsers.add_parser(
-        "remove",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_REMOVE_HELP, "exp/remove"),
-        help=EXPERIMENTS_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    remove_group = experiments_remove_parser.add_mutually_exclusive_group()
-    add_rev_selection_flags(experiments_remove_parser, "Remove", False)
-    remove_group.add_argument(
-        "--queue", action="store_true", help="Remove all queued experiments."
-    )
-    remove_group.add_argument(
-        "-g",
-        "--git-remote",
-        metavar="<git_remote>",
-        help="Name or URL of the Git remote to remove the experiment from",
-    )
-    experiments_remove_parser.add_argument(
-        "experiment",
-        nargs="*",
-        help="Experiments to remove.",
-        metavar="<experiment>",
-    )
-    experiments_remove_parser.set_defaults(func=CmdExperimentsRemove)

dvc/commands/experiments/run.py

@@ -1,142 +0,0 @@
-import argparse
-import logging
-
-from dvc.cli import completion
-from dvc.cli.utils import append_doc_link
-from dvc.commands.repro import CmdRepro
-from dvc.commands.repro import add_arguments as add_repro_arguments
-from dvc.exceptions import InvalidArgumentError
-from dvc.ui import ui
-
-logger = logging.getLogger(__name__)
-
-
-class CmdExperimentsRun(CmdRepro):
-    def run(self):
-        from dvc.compare import show_metrics
-
-        if self.args.checkpoint_resume:
-            if self.args.reset:
-                raise InvalidArgumentError(
-                    "--reset and --rev are mutually exclusive."
-                )
-            if not (self.args.queue or self.args.tmp_dir):
-                raise InvalidArgumentError(
-                    "--rev can only be used in conjunction with "
-                    "--queue or --temp."
-                )
-
-        if self.args.reset:
-            ui.write("Any existing checkpoints will be reset and re-run.")
-
-        results = self.repo.experiments.run(
-            name=self.args.name,
-            queue=self.args.queue,
-            run_all=self.args.run_all,
-            jobs=self.args.jobs,
-            params=self.args.set_param,
-            checkpoint_resume=self.args.checkpoint_resume,
-            reset=self.args.reset,
-            tmp_dir=self.args.tmp_dir,
-            machine=self.args.machine,
-            **self._common_kwargs,
-        )
-
-        if self.args.metrics and results:
-            metrics = self.repo.metrics.show(revs=list(results))
-            metrics.pop("workspace", None)
-            show_metrics(metrics)
-
-        return 0
-
-
-def add_parser(experiments_subparsers, parent_parser):
-
-    EXPERIMENTS_RUN_HELP = "Run or resume an experiment."
-    experiments_run_parser = experiments_subparsers.add_parser(
-        "run",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_RUN_HELP, "exp/run"),
-        help=EXPERIMENTS_RUN_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    _add_run_common(experiments_run_parser)
-    experiments_run_parser.add_argument(
-        "-r",
-        "--rev",
-        type=str,
-        dest="checkpoint_resume",
-        help=(
-            "Continue the specified checkpoint experiment. Can only be used "
-            "in conjunction with --queue or --temp."
-        ),
-        metavar="<experiment_rev>",
-    ).complete = completion.EXPERIMENT
-    experiments_run_parser.add_argument(
-        "--reset",
-        action="store_true",
-        help="Reset existing checkpoints and restart the experiment.",
-    )
-    experiments_run_parser.set_defaults(func=CmdExperimentsRun)
-
-
-def _add_run_common(parser):
-    """Add common args for 'exp run'."""
-    # inherit arguments from `dvc repro`
-    add_repro_arguments(parser)
-    parser.add_argument(
-        "-n",
-        "--name",
-        default=None,
-        help=(
-            "Human-readable experiment name. If not specified, a name will "
-            "be auto-generated."
-        ),
-        metavar="<name>",
-    )
-    parser.add_argument(
-        "-S",
-        "--set-param",
-        action="append",
-        default=[],
-        help="Use the specified param value when reproducing pipelines.",
-        metavar="[<filename>:]<param_name>=<param_value>",
-    )
-    parser.add_argument(
-        "--queue",
-        action="store_true",
-        default=False,
-        help="Stage this experiment in the run queue for future execution.",
-    )
-    parser.add_argument(
-        "--run-all",
-        action="store_true",
-        default=False,
-        help="Execute all experiments in the run queue. Implies --temp.",
-    )
-    parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        default=1,
-        help="Run the specified number of experiments at a time in parallel.",
-        metavar="<number>",
-    )
-    parser.add_argument(
-        "--temp",
-        action="store_true",
-        dest="tmp_dir",
-        help=(
-            "Run this experiment in a separate temporary directory instead of "
-            "your workspace."
-        ),
-    )
-    parser.add_argument(
-        "--machine",
-        default=None,
-        help=argparse.SUPPRESS,
-        # help=(
-        #     "Run this experiment on the specified 'dvc machine' instance."
-        # )
-        # metavar="<name>",
-    )

dvc/commands/experiments/show.py

@@ -1,655 +0,0 @@
-import argparse
-import logging
-import os
-import re
-from collections import Counter, OrderedDict, defaultdict
-from datetime import date, datetime
-from typing import TYPE_CHECKING
-
-from funcy import lmap
-
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.commands.metrics import DEFAULT_PRECISION
-from dvc.exceptions import DvcException, InvalidArgumentError
-from dvc.ui import ui
-from dvc.utils.flatten import flatten
-from dvc.utils.serialize import encode_exception
-
-if TYPE_CHECKING:
-    from dvc.compare import TabularData
-    from dvc.ui import RichText
-
-FILL_VALUE = "-"
-FILL_VALUE_ERRORED = "!"
-
-
-logger = logging.getLogger(__name__)
-
-
-def _update_names(names, items):
-    for name, item in items:
-        item = item.get("data", {})
-        if isinstance(item, dict):
-            item = flatten(item)
-            names[name].update({key: None for key in item})
-
-
-def _collect_names(all_experiments, **kwargs):
-    metric_names = defaultdict(dict)
-    param_names = defaultdict(dict)
-    deps_names = set()
-
-    for _, experiments in all_experiments.items():
-        for exp_data in experiments.values():
-            exp = exp_data.get("data", {})
-            _update_names(metric_names, exp.get("metrics", {}).items())
-            _update_names(param_names, exp.get("params", {}).items())
-            for dep_name in exp.get("deps", {}):
-                deps_names.add(dep_name)
-
-    return metric_names, param_names, sorted(deps_names)
-
-
-experiment_types = {
-    "checkpoint_tip": "│ ╓",
-    "checkpoint_commit": "│ ╟",
-    "checkpoint_base": "├─╨",
-    "branch_commit": "├──",
-    "branch_base": "└──",
-    "baseline": "",
-}
-
-
-def _collect_rows(
-    base_rev,
-    experiments,
-    all_headers,
-    metric_headers,
-    param_headers,
-    metric_names,
-    param_names,
-    deps_names,
-    precision=DEFAULT_PRECISION,
-    sort_by=None,
-    sort_order=None,
-    fill_value=FILL_VALUE,
-    iso=False,
-):
-    from scmrepo.git import Git
-
-    if sort_by:
-        sort_path, sort_name, sort_type = _sort_column(
-            sort_by, metric_names, param_names
-        )
-        reverse = sort_order == "desc"
-        experiments = _sort_exp(
-            experiments, sort_path, sort_name, sort_type, reverse
-        )
-
-    new_checkpoint = True
-    for i, (rev, results) in enumerate(experiments.items()):
-        fill_value = FILL_VALUE_ERRORED if results.get("error") else fill_value
-        row_dict = {k: fill_value for k in all_headers}
-
-        exp = results.get("data", {})
-
-        if exp.get("running"):
-            state = "Running"
-        elif exp.get("queued"):
-            state = "Queued"
-        else:
-            state = fill_value
-
-        is_baseline = rev == "baseline"
-
-        if is_baseline:
-            name_rev = base_rev[:7] if Git.is_sha(base_rev) else base_rev
-        else:
-            name_rev = rev[:7]
-
-        tip = exp.get("checkpoint_tip")
-        parent_rev = exp.get("checkpoint_parent", "")
-        parent_exp = experiments.get(parent_rev, {}).get("data", {})
-        parent_tip = parent_exp.get("checkpoint_tip")
-
-        parent = ""
-        if is_baseline:
-            typ = "baseline"
-        elif tip:
-            if tip == parent_tip:
-                typ = (
-                    "checkpoint_tip" if new_checkpoint else "checkpoint_commit"
-                )
-            elif parent_rev == base_rev:
-                typ = "checkpoint_base"
-            else:
-                typ = "checkpoint_commit"
-                parent = parent_rev[:7]
-        elif i < len(experiments) - 1:
-            typ = "branch_commit"
-        else:
-            typ = "branch_base"
-
-        if not is_baseline:
-            new_checkpoint = not (tip and tip == parent_tip)
-
-        row_dict["Experiment"] = exp.get("name", "")
-        row_dict["rev"] = name_rev
-        row_dict["typ"] = typ
-        row_dict["Created"] = _format_time(
-            exp.get("timestamp"), fill_value, iso
-        )
-        row_dict["parent"] = parent
-        row_dict["State"] = state
-        row_dict["Executor"] = exp.get("executor", fill_value)
-
-        _extend_row(
-            row_dict,
-            metric_names,
-            metric_headers,
-            exp.get("metrics", {}).items(),
-            precision,
-            fill_value=fill_value,
-        )
-        _extend_row(
-            row_dict,
-            param_names,
-            param_headers,
-            exp.get("params", {}).items(),
-            precision,
-            fill_value=fill_value,
-        )
-        for dep in deps_names:
-            hash_info = exp.get("deps", {}).get(dep, {}).get("hash")
-            if hash_info is not None:
-                hash_info = hash_info[:7]
-            row_dict[dep] = hash_info
-        yield list(row_dict.values())
-
-
-def _sort_column(sort_by, metric_names, param_names):
-    path, _, sort_name = sort_by.rpartition(":")
-    matches = set()
-
-    if path:
-        if path in metric_names and sort_name in metric_names[path]:
-            matches.add((path, sort_name, "metrics"))
-        if path in param_names and sort_name in param_names[path]:
-            matches.add((path, sort_name, "params"))
-    else:
-        for path in metric_names:
-            if sort_name in metric_names[path]:
-                matches.add((path, sort_name, "metrics"))
-        for path in param_names:
-            if sort_name in param_names[path]:
-                matches.add((path, sort_name, "params"))
-
-    if len(matches) == 1:
-        return matches.pop()
-    if len(matches) > 1:
-        raise InvalidArgumentError(
-            "Ambiguous sort column '{}' matched '{}'".format(
-                sort_by,
-                ", ".join([f"{path}:{name}" for path, name, _ in matches]),
-            )
-        )
-    raise InvalidArgumentError(f"Unknown sort column '{sort_by}'")
-
-
-def _sort_exp(experiments, sort_path, sort_name, typ, reverse):
-    def _sort(item):
-        rev, exp = item
-        exp_data = exp.get("data", {})
-        tip = exp_data.get("checkpoint_tip")
-        if tip and tip != rev:
-            # Sort checkpoint experiments by tip commit
-            return _sort((tip, experiments[tip]))
-        data = exp_data.get(typ, {}).get(sort_path, {}).get("data", {})
-        val = flatten(data).get(sort_name)
-        return val is None, val
-
-    ret = OrderedDict()
-    if "baseline" in experiments:
-        ret["baseline"] = experiments.pop("baseline")
-
-    ret.update(sorted(experiments.items(), key=_sort, reverse=reverse))
-    return ret
-
-
-def _format_time(datetime_obj, fill_value=FILL_VALUE, iso=False):
-    if datetime_obj is None:
-        return fill_value
-
-    if iso:
-        return datetime_obj.isoformat()
-
-    if datetime_obj.date() == date.today():
-        fmt = "%I:%M %p"
-    else:
-        fmt = "%b %d, %Y"
-    return datetime_obj.strftime(fmt)
-
-
-def _extend_row(row, names, headers, items, precision, fill_value=FILL_VALUE):
-    from dvc.compare import _format_field, with_value
-
-    for fname, data in items:
-        item = data.get("data", {})
-        item = flatten(item) if isinstance(item, dict) else {fname: item}
-        for name in names[fname]:
-            value = with_value(
-                item.get(name),
-                FILL_VALUE_ERRORED if data.get("error", None) else fill_value,
-            )
-            # wrap field data in ui.rich_text, otherwise rich may
-            # interpret unescaped braces from list/dict types as rich
-            # markup tags
-            value = ui.rich_text(str(_format_field(value, precision)))
-            if name in headers:
-                row[name] = value
-            else:
-                row[f"{fname}:{name}"] = value
-
-
-def experiments_table(
-    all_experiments,
-    headers,
-    metric_headers,
-    metric_names,
-    param_headers,
-    param_names,
-    deps_names,
-    sort_by=None,
-    sort_order=None,
-    precision=DEFAULT_PRECISION,
-    fill_value=FILL_VALUE,
-    iso=False,
-) -> "TabularData":
-    from funcy import lconcat
-
-    from dvc.compare import TabularData
-
-    all_headers = lconcat(headers, metric_headers, param_headers, deps_names)
-    td = TabularData(all_headers, fill_value=fill_value)
-    for base_rev, experiments in all_experiments.items():
-        rows = _collect_rows(
-            base_rev,
-            experiments,
-            all_headers,
-            metric_headers,
-            param_headers,
-            metric_names,
-            param_names,
-            deps_names,
-            sort_by=sort_by,
-            sort_order=sort_order,
-            precision=precision,
-            fill_value=fill_value,
-            iso=iso,
-        )
-        td.extend(rows)
-
-    return td
-
-
-def prepare_exp_id(kwargs) -> "RichText":
-    exp_name = kwargs["Experiment"]
-    rev = kwargs["rev"]
-    typ = kwargs.get("typ", "baseline")
-
-    if typ == "baseline" or not exp_name:
-        text = ui.rich_text(exp_name or rev)
-    else:
-        text = ui.rich_text.assemble(rev, " [", (exp_name, "bold"), "]")
-
-    parent = kwargs.get("parent")
-    suff = f" ({parent})" if parent else ""
-    text.append(suff)
-
-    tree = experiment_types[typ]
-    pref = f"{tree} " if tree else ""
-    return ui.rich_text(pref) + text
-
-
-def baseline_styler(typ):
-    return {"style": "bold"} if typ == "baseline" else {}
-
-
-def show_experiments(
-    all_experiments,
-    keep=None,
-    drop=None,
-    pager=True,
-    csv=False,
-    markdown=False,
-    pcp=False,
-    **kwargs,
-):
-    from funcy.seqs import flatten as flatten_list
-
-    metric_names, param_names, deps_names = _collect_names(all_experiments)
-
-    headers = [
-        "Experiment",
-        "rev",
-        "typ",
-        "Created",
-        "parent",
-        "State",
-        "Executor",
-    ]
-
-    names = {**metric_names, **param_names}
-    counter = Counter(flatten_list([list(a.keys()) for a in names.values()]))
-    counter.update(headers)
-    metric_headers = _normalize_headers(metric_names, counter)
-    param_headers = _normalize_headers(param_names, counter)
-
-    td = experiments_table(
-        all_experiments,
-        headers,
-        metric_headers,
-        metric_names,
-        param_headers,
-        param_names,
-        deps_names,
-        kwargs.get("sort_by"),
-        kwargs.get("sort_order"),
-        kwargs.get("precision"),
-        kwargs.get("fill_value"),
-        kwargs.get("iso"),
-    )
-    if keep:
-        for col in td.keys():
-            if re.match(keep, col):
-                td.protect(col)
-
-    for col in ("State", "Executor"):
-        if td.is_empty(col):
-            td.drop(col)
-
-    row_styles = lmap(baseline_styler, td.column("typ"))
-
-    if not csv:
-        merge_headers = ["Experiment", "rev", "typ", "parent"]
-        td.column("Experiment")[:] = map(
-            prepare_exp_id, td.as_dict(merge_headers)
-        )
-        td.drop(*merge_headers[1:])
-
-    headers = {
-        "metrics": metric_headers,
-        "params": param_headers,
-        "deps": deps_names,
-    }
-    styles = {
-        "Experiment": {"no_wrap": True, "header_style": "black on grey93"},
-        "Created": {"header_style": "black on grey93"},
-        "State": {"header_style": "black on grey93"},
-        "Executor": {"header_style": "black on grey93"},
-    }
-    header_bg_colors = {
-        "metrics": "cornsilk1",
-        "params": "light_cyan1",
-        "deps": "plum2",
-    }
-    styles.update(
-        {
-            header: {
-                "justify": "right" if typ == "metrics" else "left",
-                "header_style": f"black on {header_bg_colors[typ]}",
-                "collapse": idx != 0,
-                "no_wrap": typ == "metrics",
-            }
-            for typ, hs in headers.items()
-            for idx, header in enumerate(hs)
-        }
-    )
-
-    if kwargs.get("only_changed", False) or pcp:
-        td.drop_duplicates("cols", ignore_empty=False)
-
-    cols_to_drop = set()
-    if drop is not None:
-        cols_to_drop = {col for col in td.keys() if re.match(drop, col)}
-    if pcp:
-        cols_to_drop.add("Created")
-    td.drop(*cols_to_drop)
-
-    if pcp:
-        subset = {x for x in td.keys() if x != "Experiment"}
-        td.dropna(
-            "rows",
-            how="all",
-            subset=subset,
-        )
-        td.drop_duplicates("rows", subset=subset)
-        if "Experiment" in td:
-            td.column("Experiment")[:] = [
-                # remove tree characters
-                str(x).encode("ascii", "ignore").strip().decode()
-                for x in td.column("Experiment")
-            ]
-        out = kwargs.get("out") or "dvc_plots"
-        output_file = os.path.join(out, "index.html")
-        ui.write(
-            td.to_parallel_coordinates(
-                output_path=os.path.abspath(output_file),
-                color_by=kwargs.get("sort_by") or "Experiment",
-            )
-        )
-        if kwargs.get("open"):
-            return ui.open_browser(output_file)
-
-    else:
-        td.render(
-            pager=pager,
-            borders="horizontals",
-            rich_table=True,
-            header_styles=styles,
-            row_styles=row_styles,
-            csv=csv,
-            markdown=markdown,
-        )
-
-
-def _normalize_headers(names, count):
-    return [
-        name if count[name] == 1 else f"{path}:{name}"
-        for path in names
-        for name in names[path]
-    ]
-
-
-def _format_json(item):
-    if isinstance(item, (date, datetime)):
-        return item.isoformat()
-    return encode_exception(item)
-
-
-class CmdExperimentsShow(CmdBase):
-    def run(self):
-        try:
-            all_experiments = self.repo.experiments.show(
-                all_branches=self.args.all_branches,
-                all_tags=self.args.all_tags,
-                all_commits=self.args.all_commits,
-                revs=self.args.rev,
-                num=self.args.num,
-                sha_only=self.args.sha,
-                param_deps=self.args.param_deps,
-                fetch_running=self.args.fetch_running,
-            )
-        except DvcException:
-            logger.exception("failed to show experiments")
-            return 1
-
-        if self.args.json:
-            ui.write_json(all_experiments, default=_format_json)
-        else:
-            precision = (
-                self.args.precision or None
-                if self.args.csv
-                else DEFAULT_PRECISION
-            )
-            fill_value = "" if self.args.csv else FILL_VALUE
-            iso = True if self.args.csv else False
-
-            show_experiments(
-                all_experiments,
-                keep=self.args.keep,
-                drop=self.args.drop,
-                sort_by=self.args.sort_by,
-                sort_order=self.args.sort_order,
-                precision=precision,
-                fill_value=fill_value,
-                iso=iso,
-                pager=not self.args.no_pager,
-                csv=self.args.csv,
-                markdown=self.args.markdown,
-                only_changed=self.args.only_changed,
-                pcp=self.args.pcp,
-                out=self.args.out,
-                open=self.args.open,
-            )
-        return 0
-
-
-def add_parser(experiments_subparsers, parent_parser):
-    from . import add_rev_selection_flags
-
-    EXPERIMENTS_SHOW_HELP = "Print experiments."
-    experiments_show_parser = experiments_subparsers.add_parser(
-        "show",
-        parents=[parent_parser],
-        description=append_doc_link(EXPERIMENTS_SHOW_HELP, "exp/show"),
-        help=EXPERIMENTS_SHOW_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    add_rev_selection_flags(experiments_show_parser, "Show")
-    experiments_show_parser.add_argument(
-        "-a",
-        "--all-branches",
-        action="store_true",
-        default=False,
-        help="Show experiments derived from the tip of all Git branches.",
-    )
-    experiments_show_parser.add_argument(
-        "-T",
-        "--all-tags",
-        action="store_true",
-        default=False,
-        help="Show experiments derived from all Git tags.",
-    )
-    experiments_show_parser.add_argument(
-        "--no-pager",
-        action="store_true",
-        default=False,
-        help="Do not pipe output into a pager.",
-    )
-    experiments_show_parser.add_argument(
-        "--only-changed",
-        action="store_true",
-        default=False,
-        help=(
-            "Only show metrics/params with values varying "
-            "across the selected experiments."
-        ),
-    )
-    experiments_show_parser.add_argument(
-        "--drop",
-        help="Remove the columns matching the specified regex pattern.",
-        metavar="<regex_pattern>",
-    )
-    experiments_show_parser.add_argument(
-        "--keep",
-        help="Preserve the columns matching the specified regex pattern.",
-        metavar="<regex_pattern>",
-    )
-    experiments_show_parser.add_argument(
-        "--param-deps",
-        action="store_true",
-        default=False,
-        help="Show only params that are stage dependencies.",
-    )
-    experiments_show_parser.add_argument(
-        "--sort-by",
-        help="Sort related experiments by the specified metric or param.",
-        metavar="<metric/param>",
-    )
-    experiments_show_parser.add_argument(
-        "--sort-order",
-        help=(
-            "Sort order to use with --sort-by."
-            " Defaults to ascending ('asc')."
-        ),
-        choices=("asc", "desc"),
-        default="asc",
-    )
-    experiments_show_parser.add_argument(
-        "--sha",
-        action="store_true",
-        default=False,
-        help="Always show git commit SHAs instead of branch/tag names.",
-    )
-    experiments_show_parser.add_argument(
-        "--json",
-        "--show-json",
-        action="store_true",
-        default=False,
-        help="Print output in JSON format instead of a human-readable table.",
-    )
-    experiments_show_parser.add_argument(
-        "--csv",
-        "--show-csv",
-        action="store_true",
-        default=False,
-        help="Print output in csv format instead of a human-readable table.",
-    )
-    experiments_show_parser.add_argument(
-        "--md",
-        "--show-md",
-        action="store_true",
-        default=False,
-        dest="markdown",
-        help="Show tabulated output in the Markdown format (GFM).",
-    )
-    experiments_show_parser.add_argument(
-        "--precision",
-        type=int,
-        help=(
-            "Round metrics/params to `n` digits precision after the decimal "
-            f"point. Rounds to {DEFAULT_PRECISION} digits by default."
-        ),
-        metavar="<n>",
-    )
-    experiments_show_parser.add_argument(
-        "--parallel-coordinates-plot",
-        "--pcp",
-        dest="pcp",
-        action="store_true",
-        default=False,
-        help="Generate a Parallel Coordinates Plot from the tabulated output.",
-    )
-    experiments_show_parser.add_argument(
-        "-o",
-        "--out",
-        default=None,
-        help="Destination folder to save the Parallel Coordinates Plot to",
-        metavar="<path>",
-    ).complete = completion.DIR
-    experiments_show_parser.add_argument(
-        "--open",
-        action="store_true",
-        default=False,
-        help="Open the Parallel Coordinates Plot directly in the browser.",
-    )
-    experiments_show_parser.add_argument(
-        "--no-fetch",
-        dest="fetch_running",
-        action="store_false",
-        help=argparse.SUPPRESS,
-    )
-    experiments_show_parser.set_defaults(func=CmdExperimentsShow)

dvc/commands/machine.py

@@ -1,508 +0,0 @@
-import argparse
-
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link, fix_subparsers
-from dvc.commands.config import CmdConfig
-from dvc.compare import TabularData
-from dvc.config import ConfigError
-from dvc.exceptions import DvcException
-from dvc.types import Dict, List
-from dvc.ui import ui
-from dvc.utils import format_link
-
-
-class MachineDisabledError(ConfigError):
-    def __init__(self):
-        super().__init__("Machine feature is disabled")
-
-
-class CmdMachineConfig(CmdConfig):
-    def __init__(self, args):
-
-        super().__init__(args)
-        if not self.config["feature"].get("machine", False):
-            raise MachineDisabledError
-
-        if getattr(self.args, "name", None):
-            self.args.name = self.args.name.lower()
-
-    def _check_exists(self, conf):
-        if self.args.name not in conf["machine"]:
-            raise ConfigError(f"machine '{self.args.name}' doesn't exist.")
-
-
-class CmdMachineAdd(CmdMachineConfig):
-    def run(self):
-        from dvc.machine import validate_name
-
-        validate_name(self.args.name)
-
-        if self.args.default:
-            ui.write(f"Setting '{self.args.name}' as a default machine.")
-
-        with self.config.edit(self.args.level) as conf:
-            if self.args.name in conf["machine"] and not self.args.force:
-                raise ConfigError(
-                    "machine '{}' already exists. Use `-f|--force` to "
-                    "overwrite it.".format(self.args.name)
-                )
-
-            conf["machine"][self.args.name] = {"cloud": self.args.cloud}
-            if self.args.default:
-                conf["core"]["machine"] = self.args.name
-
-        return 0
-
-
-class CmdMachineRemove(CmdMachineConfig):
-    def run(self):
-        with self.config.edit(self.args.level) as conf:
-            self._check_exists(conf)
-            del conf["machine"][self.args.name]
-
-        up_to_level = self.args.level or "repo"
-        # Remove core.machine refs to this machine in any shadowing configs
-        for level in reversed(self.config.LEVELS):
-            with self.config.edit(level) as conf:
-                if conf["core"].get("machine") == self.args.name:
-                    del conf["core"]["machine"]
-
-            if level == up_to_level:
-                break
-
-        return 0
-
-
-class CmdMachineList(CmdMachineConfig):
-    TABLE_COLUMNS = [
-        "name",
-        "cloud",
-        "region",
-        "image",
-        "spot",
-        "spot_price",
-        "instance_hdd_size",
-        "instance_type",
-        "ssh_private",
-        "startup_script",
-    ]
-
-    PRIVATE_COLUMNS = ["ssh_private", "startup_script"]
-
-    def _hide_private(self, conf):
-        for machine in conf:
-            for column in self.PRIVATE_COLUMNS:
-                if column in conf[machine]:
-                    conf[machine][column] = "***"
-
-    def _show_origin(self):
-        levels = [self.args.level] if self.args.level else self.config.LEVELS
-        for level in levels:
-            conf = self.config.read(level)["machine"]
-            if self.args.name:
-                conf = conf.get(self.args.name, {})
-            self._hide_private(conf)
-            prefix = self._config_file_prefix(True, self.config, level)
-            configs = list(self._format_config(conf, prefix))
-            if configs:
-                ui.write("\n".join(configs))
-
-    def _show_table(self):
-        td = TabularData(self.TABLE_COLUMNS, fill_value="-")
-        conf = self.config.read()["machine"]
-        if self.args.name:
-            conf = {self.args.name: conf.get(self.args.name, {})}
-        self._hide_private(conf)
-        for machine, machine_config in conf.items():
-            machine_config["name"] = machine
-            td.row_from_dict(machine_config)
-        td.dropna("cols", "all")
-        td.render()
-
-    def run(self):
-        if self.args.show_origin:
-            self._show_origin()
-        else:
-            self._show_table()
-        return 0
-
-
-class CmdMachineModify(CmdMachineConfig):
-    def run(self):
-        from dvc.config import merge
-
-        with self.config.edit(self.args.level) as conf:
-            merged = self.config.load_config_to_level(self.args.level)
-            merge(merged, conf)
-            self._check_exists(merged)
-
-            if self.args.name not in conf["machine"]:
-                conf["machine"][self.args.name] = {}
-            section = conf["machine"][self.args.name]
-            if self.args.unset:
-                section.pop(self.args.option, None)
-            else:
-                section[self.args.option] = self.args.value
-        return 0
-
-
-class CmdMachineRename(CmdBase):
-    def _check_exists(self, conf):
-        if self.args.name not in conf["machine"]:
-            raise ConfigError(f"machine '{self.args.name}' doesn't exist.")
-
-    def _rename_default(self, conf):
-        if conf["core"].get("machine") == self.args.name:
-            conf["core"]["machine"] = self.args.new
-
-    def _check_before_rename(self):
-        from dvc.machine import validate_name
-
-        validate_name(self.args.new)
-
-        all_config = self.config.load_config_to_level(None)
-        if self.args.new in all_config.get("machine", {}):
-            raise ConfigError(
-                "Rename failed. Machine '{}' already exists.".format(
-                    self.args.new
-                )
-            )
-        ui.write(f"Rename machine '{self.args.name}' to '{self.args.new}'.")
-
-    def run(self):
-
-        self._check_before_rename()
-
-        with self.config.edit(self.args.level) as conf:
-            self._check_exists(conf)
-            conf["machine"][self.args.new] = conf["machine"][self.args.name]
-            try:
-                self.repo.machine.rename(self.args.name, self.args.new)
-            except DvcException as error:
-                del conf["machine"][self.args.new]
-                raise ConfigError("terraform rename failed") from error
-            del conf["machine"][self.args.name]
-            self._rename_default(conf)
-
-        up_to_level = self.args.level or "repo"
-        for level in reversed(self.config.LEVELS):
-            if level == up_to_level:
-                break
-            with self.config.edit(level) as level_conf:
-                self._rename_default(level_conf)
-
-        return 0
-
-
-class CmdMachineDefault(CmdMachineConfig):
-    def run(self):
-        if self.args.name is None and not self.args.unset:
-            conf = self.config.read(self.args.level)
-            try:
-                print(conf["core"]["machine"])
-            except KeyError:
-                ui.write("No default machine set")
-                return 1
-        else:
-            with self.config.edit(self.args.level) as conf:
-                if self.args.unset:
-                    conf["core"].pop("machine", None)
-                else:
-                    merged_conf = self.config.load_config_to_level(
-                        self.args.level
-                    )
-                    if (
-                        self.args.name in conf["machine"]
-                        or self.args.name in merged_conf["machine"]
-                    ):
-                        conf["core"]["machine"] = self.args.name
-                    else:
-                        raise ConfigError(
-                            "default machine must be present in machine "
-                            "list."
-                        )
-        return 0
-
-
-class CmdMachineCreate(CmdBase):
-    def run(self):
-        if self.repo.machine is None:
-            raise MachineDisabledError
-
-        self.repo.machine.create(self.args.name)
-        return 0
-
-
-class CmdMachineStatus(CmdBase):
-    INSTANCE_FIELD = ["name", "instance", "status"]
-    SHOWN_FIELD = [
-        "cloud",
-        "instance_ip",
-        "instance_type",
-        "instance_hdd_size",
-        "instance_gpu",
-    ]
-    FILL_VALUE = "-"
-
-    def _add_row(
-        self,
-        name: str,
-        all_status: List[Dict],
-        td: TabularData,
-    ):
-
-        if not all_status:
-            row = [
-                name,
-                self.FILL_VALUE,
-                "offline",
-            ]  # back to `None` after #7167
-            td.append(row)
-        for i, status in enumerate(all_status, start=1):
-            row = [name, f"num_{i}", "running" if status else "offline"]
-            for field in self.SHOWN_FIELD:
-                value = str(status.get(field, ""))
-                row.append(value)
-            td.append(row)
-
-    def run(self):
-        if self.repo.machine is None:
-            raise MachineDisabledError
-
-        td = TabularData(
-            self.INSTANCE_FIELD + self.SHOWN_FIELD, fill_value=self.FILL_VALUE
-        )
-
-        if self.args.name:
-            all_status = list(self.repo.machine.status(self.args.name))
-            self._add_row(self.args.name, all_status, td)
-        else:
-            name_set = set()
-            for level in self.repo.config.LEVELS:
-                conf = self.repo.config.read(level)["machine"]
-                name_set.update(conf.keys())
-            name_list = list(name_set)
-            for name in sorted(name_list):
-                all_status = list(self.repo.machine.status(name))
-                self._add_row(name, all_status, td)
-
-        td.dropna("cols", "all")
-        td.render()
-        return 0
-
-
-class CmdMachineDestroy(CmdBase):
-    def run(self):
-        if self.repo.machine is None:
-            raise MachineDisabledError
-
-        self.repo.machine.destroy(self.args.name)
-        return 0
-
-
-class CmdMachineSsh(CmdBase):
-    def run(self):
-        if self.repo.machine is None:
-            raise MachineDisabledError
-
-        self.repo.machine.run_shell(self.args.name)
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    from dvc.commands.config import parent_config_parser
-
-    machine_HELP = "Set up and manage cloud machines."
-    machine_parser = subparsers.add_parser(
-        "machine",
-        parents=[parent_parser],
-        description=append_doc_link(machine_HELP, "machine"),
-        # NOTE: suppress help during development to hide command
-        # help=machine_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-
-    machine_subparsers = machine_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc machine CMD --help` for " "command-specific help.",
-    )
-
-    fix_subparsers(machine_subparsers)
-
-    machine_ADD_HELP = "Add a new data machine."
-    machine_add_parser = machine_subparsers.add_parser(
-        "add",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_ADD_HELP, "machine/add"),
-        help=machine_ADD_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_add_parser.add_argument("name", help="Name of the machine")
-    machine_add_parser.add_argument(
-        "cloud",
-        help="Machine cloud. See full list of supported clouds at {}".format(
-            format_link(
-                "https://github.com/iterative/"
-                "terraform-provider-iterative#machine"
-            )
-        ),
-    )
-    machine_add_parser.add_argument(
-        "-d",
-        "--default",
-        action="store_true",
-        default=False,
-        help="Set as default machine.",
-    )
-    machine_add_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Force overwriting existing configs",
-    )
-    machine_add_parser.set_defaults(func=CmdMachineAdd)
-
-    machine_DEFAULT_HELP = "Set/unset the default machine."
-    machine_default_parser = machine_subparsers.add_parser(
-        "default",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_DEFAULT_HELP, "machine/default"),
-        help=machine_DEFAULT_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_default_parser.add_argument(
-        "name", nargs="?", help="Name of the machine"
-    )
-    machine_default_parser.add_argument(
-        "-u",
-        "--unset",
-        action="store_true",
-        default=False,
-        help="Unset default machine.",
-    )
-    machine_default_parser.set_defaults(func=CmdMachineDefault)
-
-    machine_LIST_HELP = "List the configuration of one/all machines."
-    machine_list_parser = machine_subparsers.add_parser(
-        "list",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_LIST_HELP, "machine/list"),
-        help=machine_LIST_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_list_parser.add_argument(
-        "--show-origin",
-        default=False,
-        action="store_true",
-        help="Show the source file containing each config value.",
-    )
-    machine_list_parser.add_argument(
-        "name",
-        nargs="?",
-        type=str,
-        help="name of machine to specify",
-    )
-    machine_list_parser.set_defaults(func=CmdMachineList)
-    machine_MODIFY_HELP = "Modify the configuration of an machine."
-    machine_modify_parser = machine_subparsers.add_parser(
-        "modify",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_MODIFY_HELP, "machine/modify"),
-        help=machine_MODIFY_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_modify_parser.add_argument("name", help="Name of the machine")
-    machine_modify_parser.add_argument(
-        "option", help="Name of the option to modify."
-    )
-    machine_modify_parser.add_argument(
-        "value", nargs="?", help="(optional) Value of the option."
-    )
-    machine_modify_parser.add_argument(
-        "-u",
-        "--unset",
-        default=False,
-        action="store_true",
-        help="Unset option.",
-    )
-    machine_modify_parser.set_defaults(func=CmdMachineModify)
-
-    machine_RENAME_HELP = "Rename a machine "
-    machine_rename_parser = machine_subparsers.add_parser(
-        "rename",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_RENAME_HELP, "remote/rename"),
-        help=machine_RENAME_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_rename_parser.add_argument("name", help="Machine to be renamed")
-    machine_rename_parser.add_argument("new", help="New name of the machine")
-    machine_rename_parser.set_defaults(func=CmdMachineRename)
-
-    machine_REMOVE_HELP = "Remove an machine."
-    machine_remove_parser = machine_subparsers.add_parser(
-        "remove",
-        parents=[parent_config_parser, parent_parser],
-        description=append_doc_link(machine_REMOVE_HELP, "machine/remove"),
-        help=machine_REMOVE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_remove_parser.add_argument(
-        "name", help="Name of the machine to remove."
-    )
-    machine_remove_parser.set_defaults(func=CmdMachineRemove)
-
-    machine_CREATE_HELP = "Create and start a machine instance."
-    machine_create_parser = machine_subparsers.add_parser(
-        "create",
-        parents=[parent_parser],
-        description=append_doc_link(machine_CREATE_HELP, "machine/create"),
-        help=machine_CREATE_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_create_parser.add_argument(
-        "name", help="Name of the machine to create."
-    )
-    machine_create_parser.set_defaults(func=CmdMachineCreate)
-
-    machine_STATUS_HELP = (
-        "List the status of running instances for one/all machines."
-    )
-    machine_status_parser = machine_subparsers.add_parser(
-        "status",
-        parents=[parent_parser],
-        description=append_doc_link(machine_STATUS_HELP, "machine/status"),
-        help=machine_STATUS_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_status_parser.add_argument(
-        "name", nargs="?", help="(optional) Name of the machine."
-    )
-    machine_status_parser.set_defaults(func=CmdMachineStatus)
-
-    machine_DESTROY_HELP = "Destroy an machine instance."
-    machine_destroy_parser = machine_subparsers.add_parser(
-        "destroy",
-        parents=[parent_parser],
-        description=append_doc_link(machine_DESTROY_HELP, "machine/destroy"),
-        help=machine_DESTROY_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_destroy_parser.add_argument(
-        "name", help="Name of the machine instance to destroy."
-    )
-    machine_destroy_parser.set_defaults(func=CmdMachineDestroy)
-
-    machine_SSH_HELP = "Connect to a machine via SSH."
-    machine_ssh_parser = machine_subparsers.add_parser(
-        "ssh",
-        parents=[parent_parser],
-        description=append_doc_link(machine_SSH_HELP, "machine/ssh"),
-        help=machine_SSH_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    machine_ssh_parser.add_argument(
-        "name", help="Name of the machine instance to connect to."
-    )
-    machine_ssh_parser.set_defaults(func=CmdMachineSsh)

dvc/commands/run.py

@@ -1,100 +0,0 @@
-import argparse
-import logging
-
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
-from dvc.commands.stage import parse_cmd
-from dvc.exceptions import DvcException
-
-logger = logging.getLogger(__name__)
-
-
-class CmdRun(CmdBase):
-    def run(self):
-        if not any(
-            [
-                self.args.deps,
-                self.args.outs,
-                self.args.outs_no_cache,
-                self.args.metrics,
-                self.args.metrics_no_cache,
-                self.args.plots,
-                self.args.plots_no_cache,
-                self.args.outs_persist,
-                self.args.outs_persist_no_cache,
-                self.args.checkpoints,
-                self.args.params,
-                self.args.command,
-            ]
-        ):  # pragma: no cover
-            logger.error(
-                "too few arguments. Specify at least one: `-d`, `-o`, `-O`, "
-                "`-m`, `-M`, `-p`, `--plots`, `--plots-no-cache`, "
-                "`--outs-persist`, `--outs-persist-no-cache`, `command`."
-            )
-            return 1
-
-        kwargs = vars(self.args)
-        kwargs.update(
-            {
-                "cmd": parse_cmd(self.args.command),
-                "fname": kwargs.pop("file"),
-                "no_exec": (self.args.no_exec or bool(self.args.checkpoints)),
-                "run_cache": not kwargs.pop("no_run_cache"),
-            }
-        )
-        try:
-            self.repo.run(**kwargs)
-        except DvcException:
-            logger.exception("")
-            return 1
-
-        return 0
-
-
-def add_parser(subparsers, parent_parser):
-    from dvc.commands.stage import _add_common_args
-
-    RUN_HELP = (
-        "Generate a dvc.yaml file from a command and execute the command."
-    )
-    run_parser = subparsers.add_parser(
-        "run",
-        parents=[parent_parser],
-        description=append_doc_link(RUN_HELP, "run"),
-        help=RUN_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    run_parser.add_argument("-n", "--name", help="Stage name.")
-    run_parser.add_argument(
-        "--file", metavar="<filename>", help=argparse.SUPPRESS
-    )
-    run_parser.add_argument(
-        "--single-stage",
-        action="store_true",
-        default=False,
-        help=argparse.SUPPRESS,
-    )
-    _add_common_args(run_parser)
-    run_parser.add_argument(
-        "--no-exec",
-        action="store_true",
-        default=False,
-        help="Only create dvc.yaml without actually running it.",
-    )
-    run_parser.add_argument(
-        "--no-commit",
-        action="store_true",
-        default=False,
-        help="Don't put files/directories into cache.",
-    )
-    run_parser.add_argument(
-        "--no-run-cache",
-        action="store_true",
-        default=False,
-        help=(
-            "Execute the command even if this stage has already been run "
-            "with the same command/dependencies/outputs/etc before."
-        ),
-    )
-    run_parser.set_defaults(func=CmdRun)

dvc/commands/stage.py

@@ -1,365 +0,0 @@
-import argparse
-import logging
-from itertools import chain, filterfalse
-from typing import TYPE_CHECKING, Dict, Iterable, List
-
-from dvc.cli import completion
-from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link, fix_subparsers
-from dvc.utils.cli_parse import parse_params
-from dvc.utils.humanize import truncate_text
-
-if TYPE_CHECKING:
-    from dvc.output import Output
-    from dvc.stage import Stage
-
-logger = logging.getLogger(__name__)
-
-MAX_TEXT_LENGTH = 80
-
-
-def generate_description(stage: "Stage") -> str:
-    def part_desc(outs: Iterable["Output"]) -> str:
-        return ", ".join(out.def_path for out in outs)
-
-    if not stage.deps and not stage.outs:
-        return "No outputs or dependencies"
-
-    if not stage.outs and stage.deps:
-        return "Depends on " + part_desc(stage.deps)
-
-    def is_plot_or_metric(out: "Output"):
-        return bool(out.plot) or bool(out.metric)
-
-    desc: List[str] = []
-
-    outs = list(filterfalse(is_plot_or_metric, stage.outs))
-    if outs:
-        desc.append("Outputs " + part_desc(outs))
-
-    plots_and_metrics = list(filter(is_plot_or_metric, stage.outs))
-    if plots_and_metrics:
-        desc.append("Reports " + part_desc(plots_and_metrics))
-
-    return "; ".join(desc)
-
-
-def prepare_description(
-    stage: "Stage", max_length: int = MAX_TEXT_LENGTH
-) -> str:
-    desc = stage.short_description() or generate_description(stage)
-    return truncate_text(desc, max_length)
-
-
-def prepare_stages_data(
-    stages: Iterable["Stage"],
-    description: bool = True,
-    max_length: int = MAX_TEXT_LENGTH,
-) -> Dict[str, str]:
-    return {
-        stage.addressing: prepare_description(stage, max_length=max_length)
-        if description
-        else ""
-        for stage in stages
-    }
-
-
-class CmdStageList(CmdBase):
-    def _get_stages(self) -> Iterable["Stage"]:
-        if self.args.all:
-            stages: List["Stage"] = self.repo.index.stages  # type: ignore
-            logger.trace(  # type: ignore[attr-defined]
-                "%d no. of stages found", len(stages)
-            )
-            return stages
-
-        # removing duplicates while maintaining order
-        collected = chain.from_iterable(
-            self.repo.stage.collect(
-                target=target, recursive=self.args.recursive, accept_group=True
-            )
-            for target in self.args.targets
-        )
-        return dict.fromkeys(collected).keys()
-
-    def run(self):
-        from dvc.ui import ui
-
-        def log_error(relpath: str, exc: Exception):
-            if self.args.fail:
-                raise exc
-            logger.debug("Stages from %s failed to load", relpath)
-
-        # silence stage collection error by default
-        self.repo.stage_collection_error_handler = log_error
-
-        stages = self._get_stages()
-        data = prepare_stages_data(stages, description=not self.args.name_only)
-        ui.table(data.items())
-
-        return 0
-
-
-def parse_cmd(commands: List[str]) -> str:
-    """
-    We need to take into account two cases:
-
-    - ['python code.py foo bar']: Used mainly with dvc as a library
-    - ['echo', 'foo bar']: List of arguments received from the CLI
-
-    The second case would need quoting, as it was passed through:
-            dvc run echo "foo bar"
-    """
-
-    def quote_argument(arg: str):
-        if not arg:
-            return '""'
-        if " " in arg and '"' not in arg:
-            return f'"{arg}"'
-        return arg
-
-    if len(commands) < 2:
-        return " ".join(commands)
-    return " ".join(map(quote_argument, commands))
-
-
-class CmdStageAdd(CmdBase):
-    def run(self):
-        kwargs = vars(self.args)
-        kwargs.update(
-            {
-                "cmd": parse_cmd(kwargs.pop("command")),
-                "params": parse_params(self.args.params),
-            }
-        )
-        self.repo.stage.add(**kwargs)
-        return 0
-
-
-def _add_common_args(parser):
-    parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Overwrite existing stage",
-    )
-    parser.add_argument(
-        "-d",
-        "--deps",
-        action="append",
-        default=[],
-        help="Declare dependencies for reproducible cmd.",
-        metavar="<path>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "-p",
-        "--params",
-        action="append",
-        default=[],
-        help="Declare parameter to use as additional dependency.",
-        metavar="[<filename>:]<params_list>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "-o",
-        "--outs",
-        action="append",
-        default=[],
-        help="Declare output file or directory.",
-        metavar="<filename>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "-O",
-        "--outs-no-cache",
-        action="append",
-        default=[],
-        help="Declare output file or directory "
-        "(do not put into DVC cache).",
-        metavar="<filename>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "-c",
-        "--checkpoints",
-        action="append",
-        default=[],
-        help="Declare checkpoint output file or directory for 'dvc exp run'. "
-        "Not compatible with 'dvc repro'.",
-        metavar="<filename>",
-    ).complete = completion.FILE
-    parser.add_argument(
-        "--external",
-        action="store_true",
-        default=False,
-        help="Allow outputs that are outside of the DVC repository.",
-    )
-    parser.add_argument(
-        "--outs-persist",
-        action="append",
-        default=[],
-        help="Declare output file or directory that will not be "
-        "removed upon repro.",
-        metavar="<filename>",
-    )
-    parser.add_argument(
-        "--outs-persist-no-cache",
-        action="append",
-        default=[],
-        help="Declare output file or directory that will not be "
-        "removed upon repro (do not put into DVC cache).",
-        metavar="<filename>",
-    )
-    parser.add_argument(
-        "-m",
-        "--metrics",
-        action="append",
-        default=[],
-        help="Declare output metrics file.",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "-M",
-        "--metrics-no-cache",
-        action="append",
-        default=[],
-        help="Declare output metrics file (do not put into DVC cache).",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "--plots",
-        action="append",
-        default=[],
-        help="Declare output plot file.",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "--plots-no-cache",
-        action="append",
-        default=[],
-        help="Declare output plot file (do not put into DVC cache).",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "--live", help="Declare output as dvclive.", metavar="<path>"
-    )
-    parser.add_argument(
-        "--live-no-cache",
-        help="Declare output as dvclive (do not put into DVC cache).",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "--live-no-summary",
-        action="store_true",
-        default=False,
-        help=argparse.SUPPRESS,
-    )
-    parser.add_argument(
-        "--live-no-html",
-        action="store_true",
-        default=False,
-        help="Signal dvclive logger to not produce training report.",
-    )
-    parser.add_argument(
-        "-w",
-        "--wdir",
-        help="Directory within your repo to run your command in.",
-        metavar="<path>",
-    )
-    parser.add_argument(
-        "--always-changed",
-        action="store_true",
-        default=False,
-        help="Always consider this DVC-file as changed.",
-    )
-    parser.add_argument(
-        "--desc",
-        type=str,
-        metavar="<text>",
-        help=(
-            "User description of the stage (optional). "
-            "This doesn't affect any DVC operations."
-        ),
-    )
-    parser.add_argument(
-        "command",
-        nargs=argparse.REMAINDER,
-        help="Command to execute.",
-        metavar="command",
-    )
-
-
-def add_parser(subparsers, parent_parser):
-    STAGES_HELP = "Commands to list and create stages."
-
-    stage_parser = subparsers.add_parser(
-        "stage",
-        parents=[parent_parser],
-        description=append_doc_link(STAGES_HELP, "stage"),
-        help=STAGES_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-
-    stage_subparsers = stage_parser.add_subparsers(
-        dest="cmd",
-        help="Use `dvc stage CMD --help` to display command-specific help.",
-    )
-
-    fix_subparsers(stage_subparsers)
-
-    STAGE_ADD_HELP = "Create stage"
-    stage_add_parser = stage_subparsers.add_parser(
-        "add",
-        parents=[parent_parser],
-        description=append_doc_link(STAGE_ADD_HELP, "stage/add"),
-        help=STAGE_ADD_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    stage_add_parser.add_argument(
-        "-n", "--name", help="Name of the stage to add", required=True
-    )
-    _add_common_args(stage_add_parser)
-    stage_add_parser.set_defaults(func=CmdStageAdd)
-
-    STAGE_LIST_HELP = "List stages."
-    stage_list_parser = stage_subparsers.add_parser(
-        "list",
-        parents=[parent_parser],
-        description=append_doc_link(STAGE_LIST_HELP, "stage/list"),
-        help=STAGE_LIST_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    stage_list_parser.add_argument(
-        "targets",
-        nargs="*",
-        default=["dvc.yaml"],
-        help=(
-            "Show stages from a dvc.yaml/.dvc file or a directory. "
-            "'dvc.yaml' by default"
-        ),
-    )
-    stage_list_parser.add_argument(
-        "--all",
-        action="store_true",
-        default=False,
-        help="List all of the stages in the repo.",
-    )
-    stage_list_parser.add_argument(
-        "--fail",
-        action="store_true",
-        default=False,
-        help="Fail immediately, do not suppress any syntax errors.",
-    )
-    stage_list_parser.add_argument(
-        "-R",
-        "--recursive",
-        action="store_true",
-        default=False,
-        help="List all stages inside the specified directory.",
-    )
-    stage_list_parser.add_argument(
-        "--name-only",
-        "--names-only",
-        action="store_true",
-        default=False,
-        help="List only stage names.",
-    )
-    stage_list_parser.set_defaults(func=CmdStageList)

dvc/compare.py

@@ -1,474 +0,0 @@
-from collections import abc
-from itertools import chain, repeat, zip_longest
-from operator import itemgetter
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Dict,
-    ItemsView,
-    Iterable,
-    Iterator,
-    List,
-    Mapping,
-    MutableSequence,
-    Optional,
-    Sequence,
-    Set,
-    Tuple,
-    Union,
-    overload,
-)
-
-from funcy import reraise
-
-if TYPE_CHECKING:
-    from dvc.types import StrPath
-    from dvc.ui.table import CellT
-
-
-class Column(List["CellT"]):
-    pass
-
-
-def with_value(value, default):
-    return default if value is None else value
-
-
-class TabularData(MutableSequence[Sequence["CellT"]]):
-    def __init__(self, columns: Sequence[str], fill_value: str = ""):
-        self._columns: Dict[str, Column] = {name: Column() for name in columns}
-        self._keys: List[str] = list(columns)
-        self._fill_value = fill_value
-        self._protected: Set[str] = set()
-
-    @property
-    def columns(self) -> List[Column]:
-        return list(map(self.column, self.keys()))
-
-    def is_protected(self, col_name) -> bool:
-        return col_name in self._protected
-
-    def protect(self, *col_names: str):
-        self._protected.update(col_names)
-
-    def unprotect(self, *col_names: str):
-        self._protected = self._protected.difference(col_names)
-
-    def column(self, name: str) -> Column:
-        return self._columns[name]
-
-    def items(self) -> ItemsView[str, Column]:
-        projection = {k: self.column(k) for k in self.keys()}
-        return projection.items()
-
-    def keys(self) -> List[str]:
-        return self._keys
-
-    def _iter_col_row(
-        self, row: Sequence["CellT"]
-    ) -> Iterator[Tuple["CellT", Column]]:
-        for val, col in zip_longest(row, self.columns):
-            if col is None:
-                break
-            yield with_value(val, self._fill_value), col
-
-    def append(self, value: Sequence["CellT"]) -> None:
-        for val, col in self._iter_col_row(value):
-            col.append(val)
-
-    def extend(self, values: Iterable[Sequence["CellT"]]) -> None:
-        for row in values:
-            self.append(row)
-
-    def insert(self, index: int, value: Sequence["CellT"]) -> None:
-        for val, col in self._iter_col_row(value):
-            col.insert(index, val)
-
-    def __iter__(self) -> Iterator[List["CellT"]]:
-        return map(list, zip(*self.columns))
-
-    def __getattr__(self, item: str) -> Column:
-        with reraise(KeyError, AttributeError):
-            return self.column(item)
-
-    def __getitem__(self, item: Union[int, slice]):
-        func = itemgetter(item)
-        it = map(func, self.columns)
-        if isinstance(item, slice):
-            it = map(list, zip(*it))
-        return list(it)
-
-    @overload
-    def __setitem__(self, item: int, value: Sequence["CellT"]) -> None:
-        ...
-
-    @overload
-    def __setitem__(
-        self, item: slice, value: Iterable[Sequence["CellT"]]
-    ) -> None:
-        ...
-
-    def __setitem__(self, item, value) -> None:
-        it = value
-        if isinstance(item, slice):
-            n = len(self.columns)
-            normalized_rows = (
-                chain(val, repeat(self._fill_value, n - len(val)))
-                for val in value
-            )
-            # we need to transpose those rows into columnar format
-            # as we work in terms of column-based arrays
-            it = zip(*normalized_rows)
-
-        for i, col in self._iter_col_row(it):
-            col[item] = i
-
-    def __delitem__(self, item: Union[int, slice]) -> None:
-        for col in self.columns:
-            del col[item]
-
-    def __len__(self) -> int:
-        if not self._columns:
-            return 0
-        return len(self.columns[0])
-
-    @property
-    def shape(self) -> Tuple[int, int]:
-        return len(self.columns), len(self)
-
-    def drop(self, *col_names: str) -> None:
-        for col in col_names:
-            if not self.is_protected(col):
-                self._keys.remove(col)
-                self._columns.pop(col)
-
-    def rename(self, from_col_name: str, to_col_name: str) -> None:
-        self._columns[to_col_name] = self._columns.pop(from_col_name)
-        self._keys[self._keys.index(from_col_name)] = to_col_name
-
-    def project(self, *col_names: str) -> None:
-        self.drop(*(set(self._keys) - set(col_names)))
-        self._keys = list(col_names)
-
-    def is_empty(self, col_name: str) -> bool:
-        col = self.column(col_name)
-        return not any(item != self._fill_value for item in col)
-
-    def to_csv(self) -> str:
-        import csv
-        from io import StringIO
-
-        buff = StringIO()
-        writer = csv.writer(buff)
-        writer.writerow(self.keys())
-
-        for row in self:
-            writer.writerow(row)
-        return buff.getvalue()
-
-    def to_parallel_coordinates(
-        self, output_path: "StrPath", color_by: str = None
-    ) -> "StrPath":
-        from dvc_render.html import render_html
-        from dvc_render.plotly import ParallelCoordinatesRenderer
-
-        render_html(
-            renderers=[
-                ParallelCoordinatesRenderer(
-                    self.as_dict(),
-                    color_by=color_by,
-                    fill_value=self._fill_value,
-                )
-            ],
-            output_file=output_path,
-        )
-        return output_path
-
-    def add_column(self, name: str) -> None:
-        self._columns[name] = Column([self._fill_value] * len(self))
-        self._keys.append(name)
-
-    def row_from_dict(self, d: Mapping[str, "CellT"]) -> None:
-        keys = self.keys()
-        for key in d:
-            if key not in keys:
-                self.add_column(key)
-
-        row: List["CellT"] = [
-            with_value(d.get(key), self._fill_value) for key in self.keys()
-        ]
-        self.append(row)
-
-    def render(self, **kwargs: Any):
-        from dvc.ui import ui
-
-        if kwargs.pop("csv", False):
-            ui.write(self.to_csv(), end="")
-        else:
-            ui.table(self, headers=self.keys(), **kwargs)
-
-    def as_dict(
-        self, cols: Iterable[str] = None
-    ) -> Iterable[Dict[str, "CellT"]]:
-        keys = self.keys() if cols is None else set(cols)
-        return [
-            {k: self._columns[k][i] for k in keys} for i in range(len(self))
-        ]
-
-    def dropna(
-        self,
-        axis: str = "rows",
-        how="any",
-        subset: Optional[Iterable[str]] = None,
-    ):
-        if axis not in ["rows", "cols"]:
-            raise ValueError(
-                f"Invalid 'axis' value {axis}."
-                "Choose one of ['rows', 'cols']"
-            )
-        if how not in ["any", "all"]:
-            raise ValueError(
-                f"Invalid 'how' value {how}." "Choose one of ['any', 'all']"
-            )
-
-        match_line: Set = set()
-        match_any = True
-        if how == "all":
-            match_any = False
-
-        for n_row, row in enumerate(self):
-            for n_col, col in enumerate(row):
-                if subset and self.keys()[n_col] not in subset:
-                    continue
-                if (col == self._fill_value) is match_any:
-                    if axis == "rows":
-                        match_line.add(n_row)
-                        break
-                    else:
-                        match_line.add(self.keys()[n_col])
-
-        to_drop = match_line
-        if how == "all":
-            if axis == "rows":
-                to_drop = set(range(len(self)))
-            else:
-                to_drop = set(self.keys())
-            to_drop -= match_line
-
-        if axis == "rows":
-            for name in self.keys():
-                self._columns[name] = Column(
-                    [
-                        x
-                        for n, x in enumerate(self._columns[name])
-                        if n not in to_drop
-                    ]
-                )
-        else:
-            self.drop(*to_drop)
-
-    def drop_duplicates(
-        self,
-        axis: str = "rows",
-        subset: Optional[Iterable[str]] = None,
-        ignore_empty: bool = True,
-    ):
-        if axis not in ["rows", "cols"]:
-            raise ValueError(
-                f"Invalid 'axis' value {axis}."
-                "Choose one of ['rows', 'cols']"
-            )
-
-        if axis == "cols":
-            cols_to_drop: List[str] = []
-            for n_col, col in enumerate(self.columns):
-                if subset and self.keys()[n_col] not in subset:
-                    continue
-                # Cast to str because Text is not hashable error
-                unique_vals = {str(x) for x in col}
-                if ignore_empty and self._fill_value in unique_vals:
-                    unique_vals -= {self._fill_value}
-                if len(unique_vals) == 1:
-                    cols_to_drop.append(self.keys()[n_col])
-            self.drop(*cols_to_drop)
-
-        elif axis == "rows":
-            unique_rows = []
-            rows_to_drop: List[int] = []
-            for n_row, row in enumerate(self):
-                if subset:
-                    row = [
-                        col
-                        for n_col, col in enumerate(row)
-                        if self.keys()[n_col] in subset
-                    ]
-
-                tuple_row = tuple(row)
-                if tuple_row in unique_rows:
-                    rows_to_drop.append(n_row)
-                else:
-                    unique_rows.append(tuple_row)
-
-            for name in self.keys():
-                self._columns[name] = Column(
-                    [
-                        x
-                        for n, x in enumerate(self._columns[name])
-                        if n not in rows_to_drop
-                    ]
-                )
-
-
-def _normalize_float(val: float, precision: int):
-    return f"{val:.{precision}g}"
-
-
-def _format_field(
-    val: Any, precision: int = None, round_digits: bool = False
-) -> str:
-    def _format(_val):
-        if isinstance(_val, float) and precision:
-            func = round if round_digits else _normalize_float
-            return func(_val, precision)
-        if isinstance(_val, abc.Mapping):
-            return {k: _format(v) for k, v in _val.items()}
-        if isinstance(_val, list):
-            return [_format(x) for x in _val]
-        return _val
-
-    return str(_format(val))
-
-
-def diff_table(
-    diff,
-    title: str,
-    old: bool = True,
-    no_path: bool = False,
-    show_changes: bool = True,
-    precision: int = None,
-    round_digits: bool = False,
-    on_empty_diff: str = None,
-    a_rev: str = None,
-    b_rev: str = None,
-) -> TabularData:
-    a_rev = a_rev or "HEAD"
-    b_rev = b_rev or "workspace"
-    headers: List[str] = ["Path", title, a_rev, b_rev, "Change"]
-    fill_value = "-"
-    td = TabularData(headers, fill_value=fill_value)
-
-    for fname, diff_in_file in diff.items():
-        for item, change in sorted(diff_in_file.items()):
-            old_value = with_value(change.get("old"), fill_value)
-            new_value = with_value(change.get("new"), fill_value)
-            diff_value = with_value(
-                change.get("diff", on_empty_diff), fill_value
-            )
-            td.append(
-                [
-                    fname,
-                    str(item),
-                    _format_field(old_value, precision, round_digits),
-                    _format_field(new_value, precision, round_digits),
-                    _format_field(diff_value, precision, round_digits),
-                ]
-            )
-
-    if no_path:
-        td.drop("Path")
-
-    if not show_changes:
-        td.drop("Change")
-
-    if not old:
-        td.drop(a_rev)
-        td.rename(b_rev, "Value")
-
-    return td
-
-
-def show_diff(
-    diff,
-    title: str,
-    old: bool = True,
-    no_path: bool = False,
-    show_changes: bool = True,
-    precision: int = None,
-    round_digits: bool = False,
-    on_empty_diff: str = None,
-    markdown: bool = False,
-    a_rev: str = None,
-    b_rev: str = None,
-) -> None:
-    td = diff_table(
-        diff,
-        title=title,
-        old=old,
-        no_path=no_path,
-        show_changes=show_changes,
-        precision=precision,
-        round_digits=round_digits,
-        on_empty_diff=on_empty_diff,
-        a_rev=a_rev,
-        b_rev=b_rev,
-    )
-    td.render(markdown=markdown)
-
-
-def metrics_table(
-    metrics,
-    all_branches: bool = False,
-    all_tags: bool = False,
-    all_commits: bool = False,
-    precision: int = None,
-    round_digits: bool = False,
-):
-    from dvc.utils.diff import format_dict
-    from dvc.utils.flatten import flatten
-
-    td = TabularData(["Revision", "Path"], fill_value="-")
-
-    for branch, val in metrics.items():
-        for fname, metric in val.get("data", {}).items():
-            row_data: Dict[str, str] = {"Revision": branch, "Path": fname}
-            metric = metric.get("data", {})
-            flattened = (
-                flatten(format_dict(metric))
-                if isinstance(metric, dict)
-                else {"": metric}
-            )
-            row_data.update(
-                {
-                    k: _format_field(v, precision, round_digits)
-                    for k, v in flattened.items()
-                }
-            )
-            td.row_from_dict(row_data)
-
-    rev, path, *metrics_headers = td.keys()
-    td.project(rev, path, *sorted(metrics_headers))
-
-    if not any([all_branches, all_tags, all_commits]):
-        td.drop("Revision")
-
-    return td
-
-
-def show_metrics(
-    metrics,
-    markdown: bool = False,
-    all_branches: bool = False,
-    all_tags: bool = False,
-    all_commits: bool = False,
-    precision: int = None,
-    round_digits: bool = False,
-) -> None:
-    td = metrics_table(
-        metrics,
-        all_branches=all_branches,
-        all_tags=all_tags,
-        all_commits=all_commits,
-        precision=precision,
-        round_digits=round_digits,
-    )
-    td.render(markdown=markdown)

dvc/config.py

@@ -4,10 +4,25 @@ import os
 import re
 from contextlib import contextmanager
 from functools import partial
+from urllib.parse import urlparse
 
-from funcy import cached_property, compact, memoize, re_find
+import configobj
+from funcy import cached_property, compact, re_find, walk_values
+from voluptuous import (
+    ALLOW_EXTRA,
+    All,
+    Any,
+    Coerce,
+    Invalid,
+    Lower,
+    Optional,
+    Range,
+    Schema,
+)
 
 from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.path_info import PathInfo
+from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
@@ -19,43 +34,209 @@ class ConfigError(DvcException):
         super().__init__(f"config file error: {msg}")
 
 
-class RemoteConfigError(ConfigError):
+class NoRemoteError(ConfigError):
     pass
 
 
-class NoRemoteError(RemoteConfigError):
+def supported_cache_type(types):
+    """Checks if link type config option consists only of valid values.
+
+    Args:
+        types (list/string): type(s) of links that dvc should try out.
+    """
+    if types is None:
+        return None
+    if isinstance(types, str):
+        types = [typ.strip() for typ in types.split(",")]
+
+    unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
+    if unsupported:
+        raise Invalid(
+            "Unsupported cache type(s): {}".format(", ".join(unsupported))
+        )
+
+    return types
+
+
+# Checks that value is either true or false and converts it to bool
+to_bool = Bool = All(
+    Lower,
+    Any("true", "false"),
+    lambda v: v == "true",
+    msg="expected true or false",
+)
+
+
+def Choices(*choices):
+    """Checks that value belongs to the specified set of values
+
+    Args:
+        *choices: pass allowed values as arguments, or pass a list or
+            tuple as a single argument
+    """
+    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
+
+
+def ByUrl(mapping):
+    schemas = walk_values(Schema, mapping)
+
+    def validate(data):
+        if "url" not in data:
+            raise Invalid("expected 'url'")
+
+        parsed = urlparse(data["url"])
+        # Windows absolute paths should really have scheme == "" (local)
+        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
+            return schemas[""](data)
+        if parsed.scheme not in schemas:
+            raise Invalid(f"Unsupported URL type {parsed.scheme}://")
+
+        return schemas[parsed.scheme](data)
+
+    return validate
+
+
+class RelPath(str):
     pass
 
 
-class RemoteNotFoundError(RemoteConfigError):
-    pass
+REMOTE_COMMON = {
+    "url": str,
+    "checksum_jobs": All(Coerce(int), Range(1)),
+    "jobs": All(Coerce(int), Range(1)),
+    Optional("no_traverse"): Bool,  # obsoleted
+    "verify": Bool,
+}
+LOCAL_COMMON = {
+    "type": supported_cache_type,
+    Optional("protected", default=False): Bool,  # obsoleted
+    "shared": All(Lower, Choices("group")),
+    Optional("slow_link_warning", default=True): Bool,
+}
+HTTP_COMMON = {
+    "auth": All(Lower, Choices("basic", "digest", "custom")),
+    "custom_auth_header": str,
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "ssl_verify": Bool,
+    "method": str,
+}
+WEBDAV_COMMON = {
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "token": str,
+    "cert_path": str,
+    "key_path": str,
+    "timeout": Coerce(int),
+}
 
-
-class MachineConfigError(ConfigError):
-    pass
-
-
-class NoMachineError(MachineConfigError):
-    pass
-
-
-class MachineNotFoundError(MachineConfigError):
-    pass
-
-
-@memoize
-def get_compiled_schema():
-    from voluptuous import Schema
-
-    from .config_schema import SCHEMA
-
-    return Schema(SCHEMA)
-
-
-def to_bool(value):
-    from .config_schema import Bool
-
-    return Bool(value)
+SCHEMA = {
+    "core": {
+        "remote": Lower,
+        "checksum_jobs": All(Coerce(int), Range(1)),
+        Optional("interactive", default=False): Bool,
+        Optional("analytics", default=True): Bool,
+        Optional("hardlink_lock", default=False): Bool,
+        Optional("no_scm", default=False): Bool,
+        Optional("autostage", default=False): Bool,
+        Optional("experiments", default=False): Bool,
+        Optional("check_update", default=True): Bool,
+    },
+    "cache": {
+        "local": str,
+        "s3": str,
+        "gs": str,
+        "hdfs": str,
+        "webhdfs": str,
+        "ssh": str,
+        "azure": str,
+        # This is for default local cache
+        "dir": str,
+        **LOCAL_COMMON,
+    },
+    "remote": {
+        str: ByUrl(
+            {
+                "": {**LOCAL_COMMON, **REMOTE_COMMON},
+                "s3": {
+                    "region": str,
+                    "profile": str,
+                    "credentialpath": str,
+                    "endpointurl": str,
+                    "access_key_id": str,
+                    "secret_access_key": str,
+                    Optional("listobjects", default=False): Bool,  # obsoleted
+                    Optional("use_ssl", default=True): Bool,
+                    "sse": str,
+                    "sse_kms_key_id": str,
+                    "acl": str,
+                    "grant_read": str,
+                    "grant_read_acp": str,
+                    "grant_write_acp": str,
+                    "grant_full_control": str,
+                    **REMOTE_COMMON,
+                },
+                "gs": {
+                    "projectname": str,
+                    "credentialpath": str,
+                    **REMOTE_COMMON,
+                },
+                "ssh": {
+                    "type": supported_cache_type,
+                    "port": Coerce(int),
+                    "user": str,
+                    "password": str,
+                    "ask_password": Bool,
+                    "keyfile": str,
+                    "timeout": Coerce(int),
+                    "gss_auth": Bool,
+                    "allow_agent": Bool,
+                    **REMOTE_COMMON,
+                },
+                "hdfs": {"user": str, **REMOTE_COMMON},
+                "webhdfs": {
+                    "hdfscli_config": str,
+                    "webhdfs_token": str,
+                    "user": str,
+                    "webhdfs_alias": str,
+                    **REMOTE_COMMON,
+                },
+                "azure": {"connection_string": str, **REMOTE_COMMON},
+                "oss": {
+                    "oss_key_id": str,
+                    "oss_key_secret": str,
+                    "oss_endpoint": str,
+                    **REMOTE_COMMON,
+                },
+                "gdrive": {
+                    "gdrive_use_service_account": Bool,
+                    "gdrive_client_id": str,
+                    "gdrive_client_secret": str,
+                    "gdrive_user_credentials_file": str,
+                    "gdrive_service_account_email": str,
+                    "gdrive_service_account_user_email": str,
+                    "gdrive_service_account_p12_file_path": str,
+                    Optional("gdrive_trash_only", default=False): Bool,
+                    **REMOTE_COMMON,
+                },
+                "http": {**HTTP_COMMON, **REMOTE_COMMON},
+                "https": {**HTTP_COMMON, **REMOTE_COMMON},
+                "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "remote": {str: object},  # Any of the above options are valid
+            }
+        )
+    },
+    "state": {
+        "row_limit": All(Coerce(int), Range(1)),
+        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),
+    },
+    # section for experimental features
+    "feature": {Optional("parametrization", default=False): Bool},
+}
+COMPILED_SCHEMA = Schema(SCHEMA)
 
 
 class Config(dict):
@@ -75,34 +256,33 @@ class Config(dict):
     APPNAME = "dvc"
     APPAUTHOR = "iterative"
 
-    SYSTEM_LEVELS = ("system", "global")
-    REPO_LEVELS = ("repo", "local")
     # In the order they shadow each other
-    LEVELS = SYSTEM_LEVELS + REPO_LEVELS
+    LEVELS = ("system", "global", "repo", "local")
 
     CONFIG = "config"
     CONFIG_LOCAL = "config.local"
 
     def __init__(
-        self, dvc_dir=None, validate=True, fs=None, config=None
+        self, dvc_dir=None, validate=True, tree=None,
     ):  # pylint: disable=super-init-not-called
-        from dvc.fs import LocalFileSystem
+        from dvc.tree.local import LocalTree
 
         self.dvc_dir = dvc_dir
-        self.wfs = LocalFileSystem()
-        self.fs = fs or self.wfs
 
         if not dvc_dir:
             try:
                 from dvc.repo import Repo
 
-                self.dvc_dir = Repo.find_dvc_dir()
+                self.dvc_dir = os.path.join(Repo.find_dvc_dir())
             except NotDvcRepoError:
                 self.dvc_dir = None
         else:
-            self.dvc_dir = self.fs.path.abspath(self.fs.path.realpath(dvc_dir))
+            self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
-        self.load(validate=validate, config=config)
+        self.wtree = LocalTree(None, {"url": self.dvc_dir})
+        self.tree = tree or self.wtree
+
+        self.load(validate=validate)
 
     @classmethod
     def get_dir(cls, level):
@@ -123,8 +303,8 @@ class Config(dict):
         }
 
         if self.dvc_dir is not None:
-            files["repo"] = self.fs.path.join(self.dvc_dir, self.CONFIG)
-            files["local"] = self.fs.path.join(self.dvc_dir, self.CONFIG_LOCAL)
+            files["repo"] = os.path.join(self.dvc_dir, self.CONFIG)
+            files["local"] = os.path.join(self.dvc_dir, self.CONFIG_LOCAL)
 
         return files
 
@@ -139,10 +319,10 @@ class Config(dict):
             dvc.config.Config: config object.
         """
         config_file = os.path.join(dvc_dir, Config.CONFIG)
-        open(config_file, "w+", encoding="utf-8").close()
+        open(config_file, "w+").close()
         return Config(dvc_dir)
 
-    def load(self, validate=True, config=None):
+    def load(self, validate=True):
         """Loads config from all the config files.
 
         Raises:
@@ -150,9 +330,6 @@ class Config(dict):
         """
         conf = self.load_config_to_level()
 
-        if config is not None:
-            merge(conf, config)
-
         if validate:
             conf = self.validate(conf)
 
@@ -163,36 +340,32 @@ class Config(dict):
         if not self["cache"].get("dir") and self.dvc_dir:
             self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
-    def _get_fs(self, level):
-        # NOTE: this might be a Gitfs, which doesn't see things outside of
+    def _get_tree(self, level):
+        # NOTE: this might be a GitTree, which doesn't see things outside of
         # the repo.
-        return self.fs if level == "repo" else self.wfs
+        return self.tree if level == "repo" else self.wtree
 
     def _load_config(self, level):
-        from configobj import ConfigObj
-
         filename = self.files[level]
-        fs = self._get_fs(level)
+        tree = self._get_tree(level)
 
-        if fs.exists(filename):
-            with fs.open(filename) as fobj:
-                conf_obj = ConfigObj(fobj)
+        if tree.exists(filename, use_dvcignore=False):
+            with tree.open(filename) as fobj:
+                conf_obj = configobj.ConfigObj(fobj)
         else:
-            conf_obj = ConfigObj()
-        return _parse_named(_lower_keys(conf_obj.dict()))
+            conf_obj = configobj.ConfigObj()
+        return _parse_remotes(_lower_keys(conf_obj.dict()))
 
     def _save_config(self, level, conf_dict):
-        from configobj import ConfigObj
-
         filename = self.files[level]
-        fs = self._get_fs(level)
+        tree = self._get_tree(level)
 
         logger.debug(f"Writing '{filename}'.")
 
-        fs.makedirs(os.path.dirname(filename))
+        tree.makedirs(os.path.dirname(filename))
 
-        config = ConfigObj(_pack_named(conf_dict))
-        with fs.open(filename, "wb") as fobj:
+        config = configobj.ConfigObj(_pack_remotes(conf_dict))
+        with tree.open(filename, "wb") as fobj:
             config.write(fobj)
         config.filename = filename
 
@@ -201,7 +374,7 @@ class Config(dict):
         conf = self._load_paths(conf, self.files[level])
 
         # Auto-verify sections
-        for key in get_compiled_schema().schema:
+        for key in COMPILED_SCHEMA.schema:
             conf.setdefault(key, {})
 
         return conf
@@ -211,8 +384,6 @@ class Config(dict):
         abs_conf_dir = os.path.abspath(os.path.dirname(filename))
 
         def resolve(path):
-            from .config_schema import RelPath
-
             if os.path.isabs(path) or re.match(r"\w+://", path):
                 return path
 
@@ -227,19 +398,13 @@ class Config(dict):
 
     @staticmethod
     def _to_relpath(conf_dir, path):
-        from dvc.fs import localfs
-        from dvc.utils import relpath
-
-        from .config_schema import RelPath
-
         if re.match(r"\w+://", path):
             return path
 
         if isinstance(path, RelPath) or not os.path.isabs(path):
             path = relpath(path, conf_dir)
-            return localfs.path.as_posix(path)
 
-        return path
+        return PathInfo(path).as_posix()
 
     @staticmethod
     def _save_paths(conf, filename):
@@ -250,27 +415,19 @@ class Config(dict):
 
     @staticmethod
     def _map_dirs(conf, func):
-        from voluptuous import ALLOW_EXTRA, Schema
-
         dirs_schema = {
             "cache": {"dir": func},
             "remote": {
                 str: {
                     "url": func,
                     "gdrive_user_credentials_file": func,
-                    "gdrive_service_account_json_file_path": func,
+                    "gdrive_service_account_p12_file_path": func,
                     "credentialpath": func,
                     "keyfile": func,
                     "cert_path": func,
                     "key_path": func,
                 }
             },
-            "machine": {
-                str: {
-                    "startup_script": func,
-                    "setup_script": func,
-                }
-            },
         }
         return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
 
@@ -283,17 +440,9 @@ class Config(dict):
                 merge(merged_conf, self.load_one(merge_level))
         return merged_conf
 
-    def read(self, level=None):
-        # NOTE: we read from a merged config by default, same as git config
-        if level is None:
-            return self.load_config_to_level()
-        return self.load_one(level)
-
     @contextmanager
-    def edit(self, level=None, validate=True):
-        # NOTE: we write to repo config by default, same as git config
-        level = level or "repo"
-        if self.dvc_dir is None and level in self.REPO_LEVELS:
+    def edit(self, level="repo"):
+        if level in {"repo", "local"} and self.dvc_dir is None:
             raise ConfigError("Not inside a DVC repo")
 
         conf = self.load_one(level)
@@ -303,46 +452,40 @@ class Config(dict):
 
         merged_conf = self.load_config_to_level(level)
         merge(merged_conf, conf)
-
-        if validate:
-            self.validate(merged_conf)
+        self.validate(merged_conf)
 
         self._save_config(level, conf)
-        self.load(validate=validate)
+        self.load()
 
     @staticmethod
     def validate(data):
-        from voluptuous import Invalid
-
         try:
-            return get_compiled_schema()(data)
+            return COMPILED_SCHEMA(data)
         except Invalid as exc:
             raise ConfigError(str(exc)) from None
 
 
-def _parse_named(conf):
-    result = {"remote": {}, "machine": {}}
+def _parse_remotes(conf):
+    result = {"remote": {}}
 
     for section, val in conf.items():
-        match = re_find(r'^\s*(remote|machine)\s*"(.*)"\s*$', section)
-        if match:
-            key, name = match
-            result[key][name] = val
+        name = re_find(r'^\s*remote\s*"(.*)"\s*$', section)
+        if name:
+            result["remote"][name] = val
         else:
             result[section] = val
 
     return result
 
 
-def _pack_named(conf):
+def _pack_remotes(conf):
     # Drop empty sections
     result = compact(conf)
 
     # Transform remote.name -> 'remote "name"'
-    for key in ("remote", "machine"):
-        for name, val in conf[key].items():
-            result[f'{key} "{name}"'] = val
-        result.pop(key, None)
+    for name, val in conf["remote"].items():
+        result[f'remote "{name}"'] = val
+    result.pop("remote", None)
 
     return result
 

dvc/config_schema.py

@@ -1,282 +0,0 @@
-import os
-from urllib.parse import urlparse
-
-from funcy import walk_values
-from voluptuous import (
-    All,
-    Any,
-    Coerce,
-    Invalid,
-    Lower,
-    Optional,
-    Range,
-    Schema,
-)
-
-Bool = All(
-    Lower,
-    Any("true", "false"),
-    lambda v: v == "true",
-    msg="expected true or false",
-)
-
-
-def supported_cache_type(types):
-    """Checks if link type config option consists only of valid values.
-
-    Args:
-        types (list/string): type(s) of links that dvc should try out.
-    """
-    if types is None:
-        return None
-    if isinstance(types, str):
-        types = [typ.strip() for typ in types.split(",")]
-
-    unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
-    if unsupported:
-        raise Invalid(
-            "Unsupported cache type(s): {}".format(", ".join(unsupported))
-        )
-
-    return types
-
-
-def Choices(*choices):
-    """Checks that value belongs to the specified set of values
-
-    Args:
-        *choices: pass allowed values as arguments, or pass a list or
-            tuple as a single argument
-    """
-    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
-
-
-def ByUrl(mapping):
-    schemas = walk_values(Schema, mapping)
-
-    def validate(data):
-        if "url" not in data:
-            raise Invalid("expected 'url'")
-
-        parsed = urlparse(data["url"])
-        # Windows absolute paths should really have scheme == "" (local)
-        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
-            return schemas[""](data)
-        if parsed.scheme not in schemas:
-            raise Invalid(f"Unsupported URL type {parsed.scheme}://")
-
-        return schemas[parsed.scheme](data)
-
-    return validate
-
-
-class RelPath(str):
-    pass
-
-
-REMOTE_COMMON = {
-    "url": str,
-    "checksum_jobs": All(Coerce(int), Range(1)),
-    "jobs": All(Coerce(int), Range(1)),
-    Optional("no_traverse"): Bool,  # obsoleted
-}
-LOCAL_COMMON = {
-    "type": supported_cache_type,
-    Optional("protected", default=False): Bool,  # obsoleted
-    "shared": All(Lower, Choices("group")),
-    Optional("slow_link_warning", default=True): Bool,
-    Optional("verify", default=False): Bool,
-}
-HTTP_COMMON = {
-    "auth": All(Lower, Choices("basic", "digest", "custom")),
-    "custom_auth_header": str,
-    "user": str,
-    "password": str,
-    "ask_password": Bool,
-    "ssl_verify": Any(Bool, str),
-    "method": str,
-    Optional("verify", default=False): Bool,
-}
-WEBDAV_COMMON = {
-    "user": str,
-    "password": str,
-    "ask_password": Bool,
-    "token": str,
-    "cert_path": str,
-    "key_path": str,
-    "timeout": Coerce(int),
-    "ssl_verify": Any(Bool, str),
-    Optional("verify", default=False): Bool,
-}
-
-SCHEMA = {
-    "core": {
-        "remote": Lower,
-        "checksum_jobs": All(Coerce(int), Range(1)),
-        Optional("interactive", default=False): Bool,
-        Optional("analytics", default=True): Bool,
-        Optional("hardlink_lock", default=False): Bool,
-        Optional("no_scm", default=False): Bool,
-        Optional("autostage", default=False): Bool,
-        Optional("experiments"): Bool,  # obsoleted
-        Optional("check_update", default=True): Bool,
-        "machine": Lower,
-    },
-    "cache": {
-        "local": str,
-        "s3": str,
-        "gs": str,
-        "hdfs": str,
-        "webhdfs": str,
-        "ssh": str,
-        "azure": str,
-        # This is for default local cache
-        "dir": str,
-        **LOCAL_COMMON,
-    },
-    "remote": {
-        str: ByUrl(
-            {
-                "": {**LOCAL_COMMON, **REMOTE_COMMON},
-                "s3": {
-                    "region": str,
-                    "profile": str,
-                    "credentialpath": str,
-                    "configpath": str,
-                    "endpointurl": str,
-                    "access_key_id": str,
-                    "secret_access_key": str,
-                    "session_token": str,
-                    Optional("listobjects", default=False): Bool,  # obsoleted
-                    Optional("use_ssl", default=True): Bool,
-                    "ssl_verify": Any(Bool, str),
-                    "sse": str,
-                    "sse_kms_key_id": str,
-                    "acl": str,
-                    "grant_read": str,
-                    "grant_read_acp": str,
-                    "grant_write_acp": str,
-                    "grant_full_control": str,
-                    "cache_regions": bool,
-                    "read_timeout": Coerce(int),
-                    "connect_timeout": Coerce(int),
-                    Optional("verify", default=False): Bool,
-                    **REMOTE_COMMON,
-                },
-                "gs": {
-                    "projectname": str,
-                    "credentialpath": str,
-                    Optional("verify", default=False): Bool,
-                    **REMOTE_COMMON,
-                },
-                "ssh": {
-                    "type": supported_cache_type,
-                    "port": Coerce(int),
-                    "user": str,
-                    "password": str,
-                    "ask_password": Bool,
-                    "keyfile": str,
-                    "timeout": Coerce(int),
-                    "gss_auth": Bool,
-                    "allow_agent": Bool,
-                    Optional("verify", default=False): Bool,
-                    **REMOTE_COMMON,
-                },
-                "hdfs": {"user": str, "kerb_ticket": str, **REMOTE_COMMON},
-                "webhdfs": {
-                    "kerberos": Bool,
-                    "kerberos_principal": str,
-                    "proxy_to": str,
-                    "ssl_verify": Any(Bool, str),
-                    "token": str,
-                    "use_https": Bool,
-                    Optional("verify", default=False): Bool,
-                    **REMOTE_COMMON,
-                },
-                "azure": {
-                    "connection_string": str,
-                    "sas_token": str,
-                    "account_name": str,
-                    "account_key": str,
-                    "tenant_id": str,
-                    "client_id": str,
-                    "client_secret": str,
-                    "allow_anonymous_login": Bool,
-                    "exclude_environment_credential": Bool,
-                    "exclude_visual_studio_code_credential": Bool,
-                    "exclude_shared_token_cache_credential": Bool,
-                    "exclude_managed_identity_credential": Bool,
-                    Optional("verify", default=False): Bool,
-                    **REMOTE_COMMON,
-                },
-                "oss": {
-                    "oss_key_id": str,
-                    "oss_key_secret": str,
-                    "oss_endpoint": str,
-                    Optional("verify", default=True): Bool,
-                    **REMOTE_COMMON,
-                },
-                "gdrive": {
-                    "gdrive_use_service_account": Bool,
-                    "gdrive_client_id": str,
-                    "gdrive_client_secret": str,
-                    "gdrive_user_credentials_file": str,
-                    "gdrive_service_account_user_email": str,
-                    "gdrive_service_account_json_file_path": str,
-                    Optional("gdrive_trash_only", default=False): Bool,
-                    Optional("verify", default=True): Bool,
-                    **REMOTE_COMMON,
-                },
-                "http": {**HTTP_COMMON, **REMOTE_COMMON},
-                "https": {**HTTP_COMMON, **REMOTE_COMMON},
-                "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON},
-                "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON},
-                "remote": {str: object},  # Any of the above options are valid
-            }
-        )
-    },
-    "state": {
-        "dir": str,
-        "row_limit": All(Coerce(int), Range(1)),  # obsoleted
-        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),  # obsoleted
-    },
-    "index": {
-        "dir": str,
-    },
-    "machine": {
-        str: {
-            "cloud": All(Lower, Choices("aws", "azure")),
-            "region": All(
-                Lower, Choices("us-west", "us-east", "eu-west", "eu-north")
-            ),
-            "image": str,
-            "spot": Bool,
-            "spot_price": Coerce(float),
-            "instance_hdd_size": Coerce(int),
-            "instance_type": Lower,
-            "instance_gpu": Lower,
-            "ssh_private": str,
-            "startup_script": str,
-            "setup_script": str,
-        },
-    },
-    # section for experimental features
-    "feature": {
-        Optional("machine", default=False): Bool,
-        # enabled by default. It's of no use, kept for backward compatibility.
-        Optional("parametrization", default=True): Bool,
-    },
-    "plots": {
-        "html_template": str,
-        Optional("auto_open", default=False): Bool,
-    },
-    "exp": {
-        "code": str,
-        "data": str,
-        "models": str,
-        "metrics": str,
-        "params": str,
-        "plots": str,
-        "live": str,
-    },
-}

dvc/daemon.py

@@ -12,53 +12,49 @@ from dvc.utils import fix_env, is_binary
 
 logger = logging.getLogger(__name__)
 
+CREATE_NEW_PROCESS_GROUP = 0x00000200
+DETACHED_PROCESS = 0x00000008
+
 
 def _popen(cmd, **kwargs):
     prefix = [sys.executable]
     if not is_binary():
-        main_entrypoint = os.path.join(
-            os.path.abspath(os.path.dirname(__file__)), "__main__.py"
-        )
-        prefix += [main_entrypoint]
+        prefix += [sys.argv[0]]
+
     return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)
 
 
 def _spawn_windows(cmd, env):
-    from subprocess import (
-        CREATE_NEW_PROCESS_GROUP,
-        CREATE_NO_WINDOW,
-        STARTF_USESHOWWINDOW,
-        STARTUPINFO,
-    )
+    from subprocess import STARTF_USESHOWWINDOW, STARTUPINFO
 
-    # https://stackoverflow.com/a/7006424
-    # https://bugs.python.org/issue41619
-    creationflags = CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW
+    creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
 
     startupinfo = STARTUPINFO()
     startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 
-    _popen(cmd, env=env, creationflags=creationflags, startupinfo=startupinfo)
+    _popen(
+        cmd, env=env, creationflags=creationflags, startupinfo=startupinfo,
+    ).communicate()
 
 
 def _spawn_posix(cmd, env):
-    from dvc.cli import main
+    from dvc.main import main
 
     # NOTE: using os._exit instead of sys.exit, because dvc built
     # with PyInstaller has trouble with SystemExit exception and throws
     # errors such as "[26338] Failed to execute script __main__"
     try:
-        pid = os.fork()  # pylint: disable=no-member
+        pid = os.fork()
         if pid > 0:
             return
     except OSError:
         logger.exception("failed at first fork")
         os._exit(1)  # pylint: disable=protected-access
 
-    os.setsid()  # pylint: disable=no-member
+    os.setsid()
 
     try:
-        pid = os.fork()  # pylint: disable=no-member
+        pid = os.fork()
         if pid > 0:
             os._exit(0)  # pylint: disable=protected-access
     except OSError:
@@ -106,9 +102,8 @@ def daemon(args):
     cmd = ["daemon", "-q"] + args
 
     env = fix_env()
-    if not is_binary():
-        file_path = os.path.abspath(inspect.stack()[0][1])
-        env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path))
+    file_path = os.path.abspath(inspect.stack()[0][1])
+    env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path))
     env[DVC_DAEMON] = "1"
 
     _spawn(cmd, env)

dvc/dagascii.py

@@ -3,14 +3,52 @@
 import logging
 import math
 import os
+import pydoc
+import sys
 
 from grandalf.graphs import Edge, Graph, Vertex
 from grandalf.layouts import SugiyamaLayout
 from grandalf.routing import EdgeViewer, route_with_lines
 
+from dvc.env import DVC_PAGER
+from dvc.utils import format_link
+
 logger = logging.getLogger(__name__)
 
 
+DEFAULT_PAGER = "less"
+DEFAULT_PAGER_FORMATTED = "{} --chop-long-lines --clear-screen".format(
+    DEFAULT_PAGER
+)
+
+
+def make_pager(cmd):
+    def pager(text):
+        return pydoc.tempfilepager(pydoc.plain(text), cmd)
+
+    return pager
+
+
+def find_pager():
+    if not sys.stdout.isatty():
+        return pydoc.plainpager
+
+    env_pager = os.getenv(DVC_PAGER)
+    if env_pager:
+        return make_pager(env_pager)
+
+    if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
+        return make_pager(DEFAULT_PAGER_FORMATTED)
+
+    logger.warning(
+        "Unable to find `less` in the PATH. Check out "
+        "{} for more info.".format(
+            format_link("https://man.dvc.org/pipeline/show")
+        )
+    )
+    return pydoc.plainpager
+
+
 class VertexViewer:
     """Class to define vertex box boundaries that will be accounted for during
     graph building by grandalf.
@@ -163,7 +201,7 @@ class AsciiCanvas:
         self.point(x0 + width, y0 + height, "+")
 
 
-def _build_sugiyama_layout(vertices, edges):
+def _build_sugiyama_layout(vertexes, edges):
     #
     # Just a reminder about naming conventions:
     # +------------X
@@ -174,17 +212,17 @@ def _build_sugiyama_layout(vertices, edges):
     # Y
     #
 
-    vertices = {v: Vertex(f" {v} ") for v in vertices}
+    vertexes = {v: Vertex(f" {v} ") for v in vertexes}
     # NOTE: reverting edges to correctly orientate the graph
-    edges = [Edge(vertices[e], vertices[s]) for s, e in edges]
-    vertices = vertices.values()
-    graph = Graph(vertices, edges)
+    edges = [Edge(vertexes[e], vertexes[s]) for s, e in edges]
+    vertexes = vertexes.values()
+    graph = Graph(vertexes, edges)
 
-    for vertex in vertices:
+    for vertex in vertexes:
         vertex.view = VertexViewer(vertex.data)
 
     # NOTE: determine min box length to create the best layout
-    minw = min(v.view.w for v in vertices)
+    minw = min(v.view.w for v in vertexes)
 
     for edge in edges:
         edge.view = EdgeViewer()
@@ -204,11 +242,11 @@ def _build_sugiyama_layout(vertices, edges):
     return sug
 
 
-def draw(vertices, edges):
+def draw(vertexes, edges):
     """Build a DAG and draw it in ASCII.
 
     Args:
-        vertices (list): list of graph vertices.
+        vertexes (list): list of graph vertexes.
         edges (list): list of graph edges.
     """
     # pylint: disable=too-many-locals
@@ -217,7 +255,7 @@ def draw(vertices, edges):
     Xs = []  # pylint: disable=invalid-name
     Ys = []  # pylint: disable=invalid-name
 
-    sug = _build_sugiyama_layout(vertices, edges)
+    sug = _build_sugiyama_layout(vertexes, edges)
 
     for vertex in sug.g.sV:
         # NOTE: moving boxes w/2 to the left

dvc/data_cloud.py

@@ -1,14 +1,9 @@
 """Manages dvc remotes that user can use with push/pull/status commands."""
 
 import logging
-from typing import TYPE_CHECKING, Iterable, Optional
 
-from dvc_data.db import get_index
-
-if TYPE_CHECKING:
-    from dvc_data.status import CompareStatusResult
-    from dvc_objects.db import ObjectDB
-    from dvc_objects.hash_info import HashInfo
+from dvc.config import NoRemoteError
+from dvc.remote import get_remote
 
 logger = logging.getLogger(__name__)
 
@@ -27,18 +22,12 @@ class DataCloud:
     def __init__(self, repo):
         self.repo = repo
 
-    def get_remote_odb(
-        self,
-        name: Optional[str] = None,
-        command: str = "<command>",
-    ) -> "ObjectDB":
-        from dvc.config import NoRemoteError
-
+    def get_remote(self, name=None, command="<command>"):
         if not name:
             name = self.repo.config["core"].get("remote")
 
         if name:
-            return self._init_odb(name)
+            return self._init_remote(name)
 
         if bool(self.repo.config["remote"]):
             error_msg = (
@@ -55,130 +44,83 @@ class DataCloud:
 
         raise NoRemoteError(error_msg)
 
-    def _init_odb(self, name):
-        from dvc.fs import get_cloud_fs
-        from dvc_data.db import get_odb
-
-        cls, config, fs_path = get_cloud_fs(self.repo, name=name)
-        config["tmp_dir"] = self.repo.index_db_dir
-        return get_odb(cls(**config), fs_path, **config)
-
-    def _log_missing(self, status: "CompareStatusResult"):
-        if status.missing:
-            missing_desc = "\n".join(
-                f"name: {hash_info.obj_name}, {hash_info}"
-                for hash_info in status.missing
-            )
-            logger.warning(
-                "Some of the cache files do not exist neither locally "
-                f"nor on remote. Missing cache files:\n{missing_desc}"
-            )
-
-    def transfer(
-        self,
-        src_odb: "ObjectDB",
-        dest_odb: "ObjectDB",
-        objs: Iterable["HashInfo"],
-        **kwargs,
-    ):
-        from dvc.exceptions import FileTransferError
-        from dvc_data.transfer import TransferError, transfer
-
-        try:
-            return transfer(src_odb, dest_odb, objs, **kwargs)
-        except TransferError as exc:
-            raise FileTransferError(exc.fails) from exc
+    def _init_remote(self, name):
+        return get_remote(self.repo, name=name)
 
     def push(
-        self,
-        objs: Iterable["HashInfo"],
-        jobs: Optional[int] = None,
-        remote: Optional[str] = None,
-        odb: Optional["ObjectDB"] = None,
+        self, cache, jobs=None, remote=None, show_checksums=False,
     ):
         """Push data items in a cloud-agnostic way.
 
         Args:
-            objs: objects to push to the cloud.
-            jobs: number of jobs that can be running simultaneously.
-            remote: optional name of remote to push to.
+            cache (NamedCache): named checksums to push to the cloud.
+            jobs (int): number of jobs that can be running simultaneously.
+            remote (dvc.remote.base.BaseRemote): optional remote to push to.
                 By default remote from core.remote config option is used.
-            odb: optional ODB to push to. Overrides remote.
+            show_checksums (bool): show checksums instead of file names in
+                information messages.
         """
-        odb = odb or self.get_remote_odb(remote, "push")
-        return self.transfer(
-            self.repo.odb.local,
-            odb,
-            objs,
+        remote = self.get_remote(remote, "push")
+
+        return remote.push(
+            self.repo.cache.local,
+            cache,
             jobs=jobs,
-            dest_index=get_index(odb),
-            cache_odb=self.repo.odb.local,
-            validate_status=self._log_missing,
+            show_checksums=show_checksums,
         )
 
     def pull(
-        self,
-        objs: Iterable["HashInfo"],
-        jobs: Optional[int] = None,
-        remote: Optional[str] = None,
-        odb: Optional["ObjectDB"] = None,
+        self, cache, jobs=None, remote=None, show_checksums=False,
     ):
         """Pull data items in a cloud-agnostic way.
 
         Args:
-            objs: objects to pull from the cloud.
-            jobs: number of jobs that can be running simultaneously.
-            remote: optional name of remote to pull from.
+            cache (NamedCache): named checksums to pull from the cloud.
+            jobs (int): number of jobs that can be running simultaneously.
+            remote (dvc.remote.base.BaseRemote): optional remote to pull from.
                 By default remote from core.remote config option is used.
-            odb: optional ODB to pull from. Overrides remote.
+            show_checksums (bool): show checksums instead of file names in
+                information messages.
         """
-        odb = odb or self.get_remote_odb(remote, "pull")
-        return self.transfer(
-            odb,
-            self.repo.odb.local,
-            objs,
+        remote = self.get_remote(remote, "pull")
+
+        return remote.pull(
+            self.repo.cache.local,
+            cache,
             jobs=jobs,
-            src_index=get_index(odb),
-            cache_odb=self.repo.odb.local,
-            verify=odb.verify,
-            validate_status=self._log_missing,
+            show_checksums=show_checksums,
         )
 
     def status(
         self,
-        objs: Iterable["HashInfo"],
-        jobs: Optional[int] = None,
-        remote: Optional[str] = None,
-        odb: Optional["ObjectDB"] = None,
-        log_missing: bool = True,
+        cache,
+        jobs=None,
+        remote=None,
+        show_checksums=False,
+        log_missing=True,
     ):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
-            objs: objects to check status for.
-            jobs: number of jobs that can be running simultaneously.
-            remote: optional remote to compare
+            cache (NamedCache): named checksums to check status for.
+            jobs (int): number of jobs that can be running simultaneously.
+            remote (dvc.remote.base.BaseRemote): optional remote to compare
                 cache to. By default remote from core.remote config option
                 is used.
-            odb: optional ODB to check status from. Overrides remote.
-            log_missing: log warning messages if file doesn't exist
+            show_checksums (bool): show checksums instead of file names in
+                information messages.
+            log_missing (bool): log warning messages if file doesn't exist
                 neither in cache, neither in cloud.
         """
-        from dvc_data.status import compare_status
-
-        if not odb:
-            odb = self.get_remote_odb(remote, "status")
-        return compare_status(
-            self.repo.odb.local,
-            odb,
-            objs,
+        remote = self.get_remote(remote, "status")
+        return remote.status(
+            self.repo.cache.local,
+            cache,
             jobs=jobs,
+            show_checksums=show_checksums,
             log_missing=log_missing,
-            dest_index=get_index(odb),
-            cache_odb=self.repo.odb.local,
         )
 
     def get_url_for(self, remote, checksum):
-        odb = self.get_remote_odb(remote)
-        path = odb.hash_to_path(checksum)
-        return odb.fs.unstrip_protocol(path)
+        remote = self.get_remote(remote)
+        return str(remote.tree.hash_to_path_info(checksum))