dvc: bump to 2.7.3

Bumps [adlfs](https://github.com/dask/adlfs) from 2021.8.2 to 2021.9.1.
- [Release notes](https://github.com/dask/adlfs/releases)
- [Changelog](https://github.com/dask/adlfs/blob/master/CHANGELOG.md)
- [Commits](https://github.com/dask/adlfs/compare/2021.08.2...2021.09.1)

---
updated-dependencies:
- dependency-name: adlfs
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <[email protected]>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.codecov.yml

@@ -1 +1,10 @@
-
+comment: false
+coverage:
+  status:
+    project:
+      default:
+        target: 90
+        threshold: 10%
+    patch:
+      default:
+        threshold: 25%

.coveragerc

@@ -1,11 +1,16 @@
 [run]
+branch = True
+source = dvc
 omit =
     *__main__.py
 
 [report]
 exclude_lines =
     if __name__ == .__main__.:
+    if typing.TYPE_CHECKING:
+    if TYPE_CHECKING:
     pass
     raise NotImplementedError
     # pragma: no cover
     #  pragma: no cover
+    \.\.\.$

.github/CODEOWNERS

@@ -0,0 +1 @@
+*       @iterative/dvc

.github/ISSUE_TEMPLATE/bug_report.md

@@ -3,16 +3,62 @@ name: "\U0001F41B Bug Report"
 about: Create a bug report to help us improve DVC
 ---
 
-## Bug Report
+# Bug Report
 
-### Please provide information about your setup
+<!--
+## Issue name
 
-**Output of `dvc version`:**
+Issue names must follow the pattern `command: description` where the command is the dvc command that you are trying to run. The description should describe the consequence of the bug. 
+
+Example: `repro: doesn't detect input changes`
+-->
+
+## Description
+
+<!--
+A clear and concise description of what the bug is.
+-->
+
+### Reproduce
+
+<!--
+Step list of how to reproduce the bug
+-->
+
+<!--
+Example:
+
+1. dvc init
+2. Copy dataset.zip to the directory
+3. dvc add dataset.zip
+4. dvc run -d dataset.zip -o model ./train.sh
+5. modify dataset.zip
+6. dvc repro
+-->
+
+### Expected
+
+<!--
+A clear and concise description of what you expect to happen.
+-->
+
+### Environment information
+
+<!--
+This is required to ensure that we can reproduce the bug.
+-->
+
+**Output of `dvc doctor`:**
 
 ```console
-$ dvc version
+$ dvc doctor
 ```
 
 **Additional Information (if any):**
 
+<!--
+Please check https://github.com/iterative/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
+
 If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
+If the issue is regarding the performance, please attach the profiling information and the benchmark comparisons.
+-->

.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+
+updates:
+  - directory: "/"
+    package-ecosystem: "pip"
+    schedule:
+      interval: "daily"
+    labels:
+      - "maintenance"
+
+  - directory: "/"
+    package-ecosystem: "github-actions"
+    schedule:
+      interval: "daily"
+    labels:
+      - "maintenance"

.github/mergify.yml

@@ -0,0 +1,9 @@
+pull_request_rules:
+  - name: backport patches to 1.11
+    conditions:
+      - base=master
+      - label=backport-1.11
+    actions:
+      backport:
+        branches:
+          - "1.11"

.github/release-drafter.yml

@@ -6,6 +6,7 @@ categories:
     labels:
       - "feature"
       - "enhancement"
+      - "ui"
   - title: "💣 Breaking Changes "
     labels:
       - "breaking-change"

.github/workflows/check-patch.yaml

@@ -1,19 +0,0 @@
-name: Lint Python
-
-on: [push, pull_request]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.8
-    - name: Install requirements
-      run: bash ./scripts/ci/install.sh
-    - name: Check Patch
-      run: ./scripts/ci/check_patch.sh

.github/workflows/packages.yaml

@@ -0,0 +1,145 @@
+name: Build packages
+on:
+  push:
+    branches:
+    - master
+  release:
+    types: [released, prereleased]
+  workflow_dispatch:
+jobs:
+  deb:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Set up Ruby 2.4
+      uses: actions/[email protected]
+      with:
+        ruby-version: 2.4
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build deb
+      run: python scripts/build.py deb
+    - name: Publish deb
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: scripts/fpm/dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_content_type: binary/octet-stream
+  rpm:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Set up Ruby 2.4
+      uses: actions/[email protected]
+      with:
+        ruby-version: 2.4
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build rpm
+      run: python scripts/build.py rpm
+    - name: Publish rpm
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_content_type: binary/octet-stream
+  osxpkg:
+    runs-on: macos-10.15
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Set up Ruby 2.4
+      uses: actions/[email protected]
+      with:
+        ruby-version: 2.4
+    - name: Install deps
+      run: |
+        # available lxml (webdav dependency) wheels are built with an old
+        # SDK, which breaks osxpkg notarization. Building from scratch
+        # instead.
+        pip install .[all] --no-binary lxml
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build
+      run: python scripts/build.py osxpkg
+    - name: Publish
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}.pkg
+        asset_name: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_content_type: binary/octet-stream
+  exe:
+    runs-on: windows-2019
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+    - name: Pull images
+      run: |
+        dvc pull
+    - name: Build
+      run: python scripts\build.py exe
+      shell: cmd
+    - name: Publish
+      if: github.event_name == 'release'
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      uses: actions/[email protected]
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: scripts/innosetup/dvc-${{ github.event.release.tag_name }}.exe
+        asset_name: dvc-${{ github.event.release.tag_name }}.exe
+        asset_content_type: binary/octet-stream
+  pip:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.7
+      uses: actions/[email protected]
+      with:
+        python-version: 3.7
+    - name: Install
+      run: pip install ".[all,tests]"
+    - name: Build packages
+      run: ./scripts/build_package.sh
+    - name: Publish packages to PyPI
+      if: github.event_name == 'release'
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_TOKEN }}

.github/workflows/tests.yaml

@@ -0,0 +1,133 @@
+name: Tests
+
+on:
+  pull_request: {}
+  push: {}
+  schedule:
+    - cron: '5 1 * * *'  # every day at 01:05
+
+env:
+  DVC_TEST: "true"
+  HOMEBREW_NO_AUTO_UPDATE: 1
+  SHELL: /bin/bash
+
+jobs:
+  lint:
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Install requirements
+      run: |
+        pip install wheel
+        pip install ".[all,tests,terraform]" Pygments collective.checkdocs pre-commit
+    - name: install temporary dependencies
+      run: |
+        pip install git+https://github.com/isidentical/gdrivefs@service-account
+        pip install git+https://github.com/isidentical/pydata-google-auth@service-account
+    - name: Check README
+      run: python setup.py checkdocs
+    - uses: pre-commit/[email protected]
+  tests:
+    timeout-minutes: 45
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, windows-2019, macos-10.15]
+        pyv: ["3.6", "3.7", "3.8", "3.9"]
+        include:
+          - os: "macos-10.15"
+            pyv: "3.10.0-rc.1"
+        exclude:
+          - os: macos-10.15
+            pyv: "3.9"
+          - os: macos-10.15
+            pyv: "3.7"
+
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python
+      uses: actions/[email protected]
+      with:
+        python-version: ${{ matrix.pyv }}
+    - name: get pip cache dir
+      id: pip-cache-dir
+      run: |
+        echo "::set-output name=dir::$(pip cache dir)"
+    - name: set pip cache
+      id: pip-cache
+      uses: actions/[email protected]
+      with:
+        path: ${{ steps.pip-cache-dir.outputs.dir }}
+        key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: install libgit2 on Python3.10 in macOS
+      if: matrix.pyv == '3.10.0-rc.1' && matrix.os == 'macos-10.15'
+      run: |
+        brew install libgit2
+    - name: install
+      if: steps.cache.pip-cache-dir.cache-hit != 'true'
+      run: |
+        pip install --upgrade pip setuptools wheel
+        pip install -e ".[all,tests,terraform]"
+    - name: install temporary dependencies
+      run: |
+        pip install git+https://github.com/isidentical/gdrivefs@service-account
+        pip install git+https://github.com/isidentical/pydata-google-auth@service-account
+    - name: setup git
+      run: |
+        git config --global user.email "[email protected]"
+        git config --global user.name "DVC Tester"
+    - name: setup gs creds
+      shell: bash
+      env:
+        GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
+      if: env.GS_CREDS_JSON != ''
+      run: |
+        mkdir -p scripts/ci
+        echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
+    - name: disable gdrive tests
+      if: >-
+        github.event_name != 'schedule' &&
+        (matrix.pyv != '3.8' || matrix.os != 'ubuntu-18.04')
+      run: |
+        echo "extra_test_args=--disable-gdrive" >> $GITHUB_ENV
+    - name: run tests
+      timeout-minutes: 30
+      env:
+        GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID}}
+        OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET}}
+        OSS_ENDPOINT: ${{ secrets.OSS_ENDPOINT}}
+      run: >-
+        python -m tests -n=4
+        --cov-report=xml --cov-report=term
+        --all ${{ env.extra_test_args }}
+    - name: upload coverage report
+      uses: codecov/[email protected]
+      with:
+        file: ./coverage.xml
+        fail_ci_if_error: false
+  notify:
+    if: github.ref == 'refs/heads/master' && failure()
+    needs:
+      - lint
+      - tests
+    runs-on: ubuntu-latest
+    steps:
+    - name: Slack Notification
+      uses: rtCamp/[email protected]
+      env:
+        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+        SLACK_COLOR: ${{ job.status }}
+        SLACK_MESSAGE: 'CI Failed on master :boom:'
+        SLACK_TITLE: CI Status
+        SLACK_USERNAME: DVC-CI

.gitignore

@@ -5,6 +5,7 @@ neatlynx/__pycache__
 *.pyc
 .env/
 .env2.7/
+.python-version
 
 .dvc.conf.lock
 .DS_Store

.pre-commit-config.yaml

@@ -1,14 +1,24 @@
+ci:
+  skip: [pylint, dvc-pre-commit]
+
 repos:
   - hooks:
       - id: black
         language_version: python3
     repo: https://github.com/ambv/black
-    rev: 19.10b0
+    rev: 21.8b0
+  - hooks:
+      - id: codespell
+        args: 
+          - --ignore-words-list
+          - ba,datas,fo,uptodate
+    repo: https://github.com/codespell-project/codespell
+    rev: v2.1.0
   - hooks:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 5.4.2
+    rev: 5.9.3
   - hooks:
       - id: flake8
         language_version: python3
@@ -18,7 +28,18 @@ repos:
           - flake8-debugger
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.3
+    rev: 3.9.2
+  - hooks:
+      -  id: mypy
+         language_version: python3
+         files: ^dvc/
+         additional_dependencies:
+           - types-requests
+           - types-paramiko
+           - types-tabulate
+           - types-toml
+    repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v0.910'
   - repo: local
     hooks:
     - id: pylint
@@ -33,7 +54,7 @@ repos:
         id: beautysh
         language_version: python3
     repo: https://github.com/lovesegfault/beautysh
-    rev: master
+    rev: v6.1.0
   - hooks:
       - args:
           - git-hook

.pylintrc

@@ -3,7 +3,7 @@
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code.
-extension-pkg-whitelist=
+extension-pkg-whitelist=pygit2
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
@@ -15,10 +15,13 @@ ignore-patterns=
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-init-hook="import sys; \
-           from pathlib import Path; \
-           from pylint.config import find_pylintrc; \
-           sys.path.append(Path(find_pylintrc()).parent / 'tests');"
+init-hook="import sys;
+           from pathlib import Path;
+           from pylint.config import find_pylintrc;
+
+           tests_folder = Path(find_pylintrc()).parent / 'tests'
+           sys.path.append(str(tests_folder));
+           "
 
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
@@ -178,13 +181,17 @@ disable=format,
         # we use it to speedup/optimize
         import-outside-toplevel,
         fixme,
-
+        # we use cause of the exception for error message output
+        # so some of the times we don't use `raise from` to not show the cause
+        # of the error
+        raise-missing-from,
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
 # multiple time (only on the command line, not in the configuration file where
 # it should appear only once). See also the "--disable" option for examples.
-enable=c-extension-no-member
+enable=c-extension-no-member,
+       no-else-return,
 
 
 [REPORTS]
@@ -265,7 +272,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=pytest.lazy_fixture,logging.TRACE,logger.trace,sys.getwindowsversion
+generated-members=hdfs.*,pytest.lazy_fixture,logging.TRACE,logger.trace,sys.getwindowsversion
 
 # Tells whether missing members accessed in mixin class should be ignored. A
 # mixin class is detected if its name ends with "mixin" (case insensitive).
@@ -292,7 +299,7 @@ ignored-classes=optparse.Values,thread._local,_thread._local,Dvcfile
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
+ignored-modules=azure
 
 # Show a hint with possible names when a member name was not found. The aspect
 # of finding the hint is based on edit distance.
@@ -306,6 +313,7 @@ missing-member-hint-distance=1
 # showing a hint for a missing member.
 missing-member-max-choices=1
 
+signature-mutators=funcy.decorators.decorator,
 
 [VARIABLES]
 

.restyled.yaml

@@ -1,20 +0,0 @@
----
-restylers:
-  - name: black
-    image: restyled/restyler-black:v19.10b0
-    command:
-    - black
-    arguments: []
-    include:
-    - "**/*.py"
-    interpreters:
-    - python
-  - name: isort
-    image: restyled/restyler-isort:v5.4.2
-    command:
-    - isort
-    arguments: []
-    include:
-    - "**/*.py"
-    interpreters:
-    - python

.travis.yml

@@ -1,162 +0,0 @@
-cache:
-  directories:
-    - $HOME/.cache/pip
-    - $HOME/.pyenv
-dist: xenial
-env:
-  global:
-    - HOMEBREW_NO_AUTO_UPDATE=1
-    - PYENV_ROOT="$HOME/.pyenv"
-    - PATH="$PYENV_ROOT/bin:$PATH"
-    - PIP_CACHE_DIR="$HOME/.cache/pip"  # unify pip cache location for all platforms
-stages:
-  - check
-  - test
-  - build
-# Travis doesn't support testing python on Mac yet, so we need
-# to workaround it by installing it directly with brew.
-#
-# Note that we specify language as ruby-2.4.3 to help us satisfy
-# fpm dependencies, as travis runs el capitan with an old ruby
-# version, that causes fpm to crash.
-#
-# Each job will use the previous job's `stage` tag if not specified.
-jobs:
-  include:
-  # patch formatting checks
-  - name: "Check patch formatting"
-    stage: check
-    os: linux
-    language: python
-    python: 3.7
-    before_install: bash ./scripts/ci/install.sh
-    install:
-    script: ./scripts/ci/check_patch.sh
-  # test jobs
-  - name: "3.7 on Windows"
-    stage: test
-    os: windows           # Windows 10.0.17134 N/A Build 17134
-    language: shell       # 'language: python' is an error on Travis CI Windows
-    env:
-      - PYTHON_VERSION=3.7.5
-      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
-  - name: "3.8 on Windows"
-    os: windows           # Windows 10.0.17134 N/A Build 17134
-    language: shell       # 'language: python' is an error on Travis CI Windows
-    env:
-      - PYTHON_VERSION=3.8.0
-      - PATH=/c/Python38:/c/Python38/Scripts:$PATH
-  - name: "3.6 on Linux"
-    language: python
-    python: 3.6
-  - name: "3.7 on Linux"
-    language: python
-    python: 3.7
-  - name: "3.8 on Linux"
-    language: python
-    python: 3.8
-  - name: "3.7 on Mac"
-    os: osx
-    osx_image: xcode11.3
-  # build jobs
-  - name: "Mac pkg"
-    stage: build
-    os: osx
-    osx_image: xcode9.4
-    language: ruby
-    rvm: 2.4.3
-    install:
-    script: ./scripts/build_posix.sh
-  - name: "Windows pkg"
-    os: windows
-    language: shell
-    install:
-    script: ./scripts/build_windows.cmd
-    env:
-      - PYTHON_VERSION=3.7.5
-      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
-  - name: "Linux pkgs"
-    language: python
-    python: 3.7
-    before_install:
-    install:
-    script: ./scripts/build_posix.sh
-  - name: "PyPI pkgs"
-    language: python
-    python: 3.8
-    script: ./scripts/build_package.sh
-  - name: Snapcraft snap
-    language: python
-    python: 3.8
-    addons:
-     snaps:
-     - name: snapcraft
-       channel: stable
-       confinement: classic
-     - name: review-tools
-       channel: stable
-     - name: lxd
-       channel: stable
-    env:
-    - SNAPCRAFT_IMAGE_INFO: |
-        '{"build_url": "$TRAVIS_JOB_WEB_URL"}'
-    - SNAPCRAFT_BUILD_ENVIRONMENT: lxd
-    - SNAPCRAFT_BUILD_INFO: 1  # https://snapcraft.io/blog/introducing-developer-notifications-for-snap-security-updates
-    install:
-    - sudo usermod --append --groups lxd $USER
-    - sudo /snap/bin/lxd.migrate -yes
-    - sudo /snap/bin/lxd waitready
-    - sudo /snap/bin/lxd init --auto
-    script:
-    - ./scripts/build_snap.sh
-    after_failure:
-    - sudo journalctl -u snapd
-
-before_install:
-  - bash ./scripts/ci/before_install.sh
-  - source env.sh
-  - bash ./scripts/ci/check_python.sh
-install:
-  - bash ./scripts/ci/install.sh
-script:
-  - bash ./scripts/ci/script.sh
-deploy:
-  - provider: releases
-    api_key:
-      secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=
-    file_glob: true
-    file:
-      - dvc*.rpm
-      - dvc*.deb
-      - dvc*.pkg
-      - dvc*.exe
-    skip_cleanup: true
-    on:
-      condition: "$(./scripts/ci/deploy_condition.sh dvc*.rpm dvc*.deb dvc*.pkg dvc*.exe)"
-      tags: true
-      repo: iterative/dvc
-      stage: build
-  - provider: pypi
-    user:
-      secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
-    password:
-      secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
-    skip_existing: true
-    file_glob: true
-    file:
-      - dist/dvc*.whl
-      - dist/dvc*.tar.gz
-    skip_cleanup: true
-    on:
-      condition: "$(./scripts/ci/deploy_condition.sh dist/dvc*.whl dist/dvc*.tar.gz)"
-      tags: true
-      repo: iterative/dvc
-      stage: build
-  - provider: script
-    skip_cleanup: true
-    script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out"
-    on:
-      all_branches: true
-      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL"
-      repo: iterative/dvc
-      stage: build

MANIFEST.in

@@ -1,2 +1,3 @@
 include fastentrypoints.py
 include LICENSE
+recursive-include requirements *.txt

README.rst

@@ -8,7 +8,7 @@
 • `Tutorial <https://dvc.org/doc/get-started>`_
 • `Mailing List <https://sweedom.us10.list-manage.com/subscribe/post?u=a08bf93caae4063c4e6a351f6&id=24c0ecc49a>`_
 
-|Release| |CI| |Maintainability| |Coverage| |Donate| |DOI|
+|CI| |Maintainability| |Coverage| |Donate| |DOI|
 
 |PyPI| |Packages| |Brew| |Conda| |Choco| |Snap|
 
@@ -20,8 +20,9 @@ learning projects. Key features:
 #. Simple **command line** Git-like experience. Does not require installing and maintaining
    any databases. Does not depend on any proprietary online services.
 
-#. Management and versioning of **datasets** and **machine learning models**. Data is saved in
-   S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
+#. Management and versioning of **datasets** and **machine learning
+   models**. Data can be saved in S3, Google cloud, Azure, Alibaba 
+   cloud, SSH server, HDFS, or even local HDD RAID.
 
 #. Makes projects **reproducible** and **shareable**; helping to answer questions about how
    a model was built.
@@ -29,9 +30,9 @@ learning projects. Key features:
 #. Helps manage experiments with Git tags/branches and **metrics** tracking.
 
 **DVC** aims to replace spreadsheet and document sharing tools (such as Excel or Google Docs)
-which are being used frequently as both knowledge repositories and team ledgers.
-DVC also replaces both ad-hoc scripts to track, move, and deploy different model versions;
-as well as ad-hoc data file suffixes and prefixes.
+frequently used as both knowledge repositories and team ledgers.
+DVC also replaces both ad-hoc scripts to track, move, and deploy different model versions
+and ad-hoc data file suffixes and prefixes.
 
 .. contents:: **Contents**
   :backlinks: none
@@ -65,25 +66,25 @@ Quick start
 
 Please read `Get Started <https://dvc.org/doc/get-started>`_ guide for a full version. Common workflow commands include:
 
-+-----------------------------------+-------------------------------------------------------------------+
-| Step                              | Command                                                           |
-+===================================+===================================================================+
-| Track data                        | | ``$ git add train.py``                                          |
-|                                   | | ``$ dvc add images.zip``                                        |
-+-----------------------------------+-------------------------------------------------------------------+
-| Connect code and data by commands | | ``$ dvc run -d images.zip -o images/ unzip -q images.zip``      |
-|                                   | | ``$ dvc run -d images/ -d train.py -o model.p python train.py`` |
-+-----------------------------------+-------------------------------------------------------------------+
-| Make changes and reproduce        | | ``$ vi train.py``                                               |
-|                                   | | ``$ dvc repro model.p.dvc``                                     |
-+-----------------------------------+-------------------------------------------------------------------+
-| Share code                        | | ``$ git add .``                                                 |
-|                                   | | ``$ git commit -m 'The baseline model'``                        |
-|                                   | | ``$ git push``                                                  |
-+-----------------------------------+-------------------------------------------------------------------+
-| Share data and ML models          | | ``$ dvc remote add myremote -d s3://mybucket/image_cnn``        |
-|                                   | | ``$ dvc push``                                                  |
-+-----------------------------------+-------------------------------------------------------------------+
++-----------------------------------+----------------------------------------------------------------------------+
+| Step                              | Command                                                                    |
++===================================+============================================================================+
+| Track data                        | | ``$ git add train.py``                                                   |
+|                                   | | ``$ dvc add images.zip``                                                 |
++-----------------------------------+----------------------------------------------------------------------------+
+| Connect code and data by commands | | ``$ dvc run -n prepare -d images.zip -o images/ unzip -q images.zip``    |
+|                                   | | ``$ dvc run -n train -d images/ -d train.py -o model.p python train.py`` |
++-----------------------------------+----------------------------------------------------------------------------+
+| Make changes and reproduce        | | ``$ vi train.py``                                                        |
+|                                   | | ``$ dvc repro model.p.dvc``                                              |
++-----------------------------------+----------------------------------------------------------------------------+
+| Share code                        | | ``$ git add .``                                                          |
+|                                   | | ``$ git commit -m 'The baseline model'``                                 |
+|                                   | | ``$ git push``                                                           |
++-----------------------------------+----------------------------------------------------------------------------+
+| Share data and ML models          | | ``$ dvc remote add myremote -d s3://mybucket/image_cnn``                 |
+|                                   | | ``$ dvc push``                                                           |
++-----------------------------------+----------------------------------------------------------------------------+
 
 Installation
 ============
@@ -129,7 +130,11 @@ Conda (Anaconda)
 
 .. code-block:: bash
 
-   conda install -c conda-forge dvc
+   conda install -c conda-forge mamba # installs much faster than conda
+   mamba install -c conda-forge dvc
+
+Depending on the remote storage type you plan to use to keep and share your data, you might need to
+install optional dependencies: `dvc-s3`, `dvc-azure`, `dvc-gdrive`, `dvc-gs`, `dvc-oss`, `dvc-ssh`.
 
 pip (PyPI)
 ----------
@@ -178,20 +183,38 @@ Fedora / CentOS (rpm)
 Comparison to related technologies
 ==================================
 
+#. Data Engineering tools such as `AirFlow <https://airflow.apache.org/>`,
+   `Luigi <https://github.com/spotify/luigi>`, and others - in DVC data,
+   model and ML pipelines represent a single ML project focused on data
+   scientists' experience.  Data engineering tools orchestrate multiple data
+   projects and focus on efficient execution. A DVC project can be used from
+   existing data pipelines as a single execution step.
+
 #. `Git-annex <https://git-annex.branchable.com/>`_ - DVC uses the idea of storing the content of large files (which should
    not be in a Git repository) in a local key-value store, and uses file hardlinks/symlinks instead of
    copying/duplicating files.
 
-#. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with any remote storage (S3, Google Cloud, Azure, SSH,
-   etc). DVC also uses reflinks or hardlinks to avoid copy operations on checkouts; thus handling large data files
-   much more efficiently.
+#. `Git-LFS <https://git-lfs.github.com/>`_ - DVC is compatible with many
+   remote storage services (S3, Google Cloud, Azure, SSH, etc). DVC also
+   uses reflinks or hardlinks to avoid copy operations on checkouts; thus
+   handling large data files much more efficiently.
 
-#. *Makefile* (and analogues including ad-hoc scripts) - DVC tracks dependencies (in a directed acyclic graph).
+#. Makefile (and analogues including ad-hoc scripts) - DVC tracks
+   dependencies (in a directed acyclic graph).
 
 #. `Workflow Management Systems <https://en.wikipedia.org/wiki/Workflow_management_system>`_ - DVC is a workflow
    management system designed specifically to manage machine learning experiments. DVC is built on top of Git.
 
-#. `DAGsHub <https://dagshub.com/>`_ - This is a Github equivalent for DVC. Pushing Git+DVC based repositories to DAGsHub will produce in a high level project dashboard; including DVC pipelines and metrics visualizations, as well as links to any DVC-managed files present in cloud storage.
+#. `DAGsHub <https://dagshub.com/>`_ - online service to host DVC
+   projects.  It provides a useful UI around DVC repositories and integrates
+   other tools.
+
+#. `DVC Studio <https://studio.iterative.ai/>`_ - official online
+   platform for DVC projects.  It can be used to manage data and models, run
+   and track experiments, and visualize and share results.  Also, it
+   integrates with `CML (CI/CD for ML) <https://cml.dev/>` for training
+   models in the cloud or Kubernetes.
+
 
 Contributing
 ============
@@ -199,39 +222,9 @@ Contributing
 |Maintainability| |Donate|
 
 Contributions are welcome! Please see our `Contributing Guide <https://dvc.org/doc/user-guide/contributing/core>`_ for more
-details.
+details. Thanks to all our contributors!
 
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/0
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/0
-   :alt: 0
-
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/1
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/1
-   :alt: 1
-
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/2
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/2
-   :alt: 2
-
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/3
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/3
-   :alt: 3
-
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/4
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/4
-   :alt: 4
-
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/5
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/5
-   :alt: 5
-
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/6
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/6
-   :alt: 6
-
-.. image:: https://sourcerer.io/fame/efiop/iterative/dvc/images/7
-   :target: https://sourcerer.io/fame/efiop/iterative/dvc/links/7
-   :alt: 7
+|Contribs|
 
 Mailing List
 ============
@@ -254,17 +247,16 @@ Citation
 Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
 `DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
 
+Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and Source Code - Empirical Study of DVC Projects <https://mcis.cs.queensu.ca/publications/2021/saner.pdf>`_ , in Proceedings of the 28th IEEE International Conference on Software Analysis, Evolution, and Reengineering, SANER 2021. Hawaii, USA.
+
+
 .. |Banner| image:: https://dvc.org/img/logo-github-readme.png
    :target: https://dvc.org
    :alt: DVC logo
 
-.. |Release| image:: https://img.shields.io/badge/release-ok-brightgreen
-   :target: https://travis-ci.com/iterative/dvc/branches
-   :alt: Release
-
-.. |CI| image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev&logo=travis
-   :target: https://travis-ci.com/iterative/dvc/builds
-   :alt: Travis dev branch
+.. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=master
+   :target: https://github.com/iterative/dvc/actions
+   :alt: GHA Tests
 
 .. |Maintainability| image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
    :target: https://codeclimate.com/github/iterative/dvc
@@ -309,3 +301,7 @@ Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
 .. |Flowchart| image:: https://dvc.org/img/flow.gif
    :target: https://dvc.org/img/flow.gif
    :alt: how_dvc_works
+
+.. |Contribs| image:: https://contrib.rocks/image?repo=iterative/dvc
+   :target: https://github.com/iterative/dvc/graphs/contributors
+   :alt: Contributors

debian/changelog

@@ -1,5 +0,0 @@
-dvc (0.75.b1) experimental; urgency=low
-
-  * First native Debian package version.
-
- -- Dmitry Petrov <[email protected]>  Sat, 07 Dec 2019 09:49:28 +0300

debian/compat

@@ -1 +0,0 @@
-10

debian/control

@@ -1,33 +0,0 @@
-Source: dvc
-Maintainer: Dmitry Petrov <[email protected]>
-Section: python
-Priority: optional
-Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)
-Standards-Version: 4.0.0
-Homepage: http://dvc.org
-Vcs-Git: https://github.com/iterative/dvc.git
-Vcs-browser: https://github.com/iterative/dvc
-
-Package: dvc
-Architecture: all
-Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2)
-Recommends: python3-arrow (=0.14.0),
-            python3-paramiko (>=2.5.0),
-            python3-gssapi (>= 2.5.0),
-            python3-boto3 (>=1.9.86),
-            python3-pydrive (=1.3.1),
-            python3-azure-storage-blob (=12.1.0),
-            python3-oss2 (=2.6.1),
-            python3-google-cloud-storage (=1.19.0)
-Description: Git for data scientists - manage your code and data together
- Data Version Control or DVC is an open-source tool for data science and machine learning projects.
- .
- Key features:
- .
- 1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services.
- .
- 2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
- .
- 3. Makes projects reproducible and shareable; helping to answer questions about how a model was built.
- .
- 4. Helps manage experiments with Git tags/branches and metrics tracking.

debian/copyright

@@ -1,11 +0,0 @@
-Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: DVC
-Upstream-Contact: Dmitry Petrov <[email protected]>
-Source: https://github.com/iterative/dvc
-
-Files: *
-Copyright: Dmitry Petrov <[email protected]>
-License: Apache 2.0
- On Debian systems, the full text of the Apache 2.0
- License can be found in the file
- `/usr/share/common-licenses/Apache-2.0`.

debian/rules

@@ -1,6 +0,0 @@
-#!/usr/bin/make -f
-
-export PYBUILD_NAME=dvc
-%:
-	dh $@ --with python3 --buildsystem=pybuild
-

debian/source/format

@@ -1 +0,0 @@
-3.0 (native)

debian/watch

@@ -1,4 +0,0 @@
-# please also check http://pypi.debian.net/dvc/watch
-version=3
-opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
-http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))

dvc/_debug.py

@@ -0,0 +1,91 @@
+from contextlib import ExitStack, contextmanager
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from argparse import Namespace
+
+
+@contextmanager
+def instrument(html_output=False):
+    """Run a statistical profiler"""
+    try:
+        from pyinstrument import Profiler  # pylint: disable=import-error
+    except ImportError:
+        print("Failed to run profiler, pyinstrument is not installed")
+        yield
+        return
+
+    profiler = Profiler()
+
+    profiler.start()
+    yield
+    profiler.stop()
+
+    if html_output:
+        profiler.open_in_browser()
+        return
+    print(profiler.output_text(unicode=True, color=True))
+
+
+@contextmanager
+def profile(dump_path: str = None):
+    """Run a cprofile"""
+    import cProfile
+
+    prof = cProfile.Profile()
+    prof.enable()
+
+    yield
+
+    prof.disable()
+    if not dump_path:
+        prof.print_stats(sort="cumtime")
+        return
+    prof.dump_stats(dump_path)
+
+
+@contextmanager
+def debug():
+    try:
+        yield
+    except Exception:  # pylint: disable=broad-except
+        try:
+            import ipdb as pdb  # noqa: T100, pylint: disable=import-error
+        except ImportError:
+            import pdb  # noqa: T100
+        pdb.post_mortem()
+
+        raise  # prevent from jumping ahead
+
+
+@contextmanager
+def debugtools(args: "Namespace" = None, **kwargs):
+    kw = vars(args) if args else {}
+    kw.update(kwargs)
+
+    with ExitStack() as stack:
+        if kw.get("pdb"):
+            stack.enter_context(debug())
+        if kw.get("cprofile") or kw.get("cprofile_dump"):
+            stack.enter_context(profile(kw.get("cprofile_dump")))
+        if kw.get("instrument") or kw.get("instrument_open"):
+            stack.enter_context(instrument(kw.get("instrument_open", False)))
+        yield
+
+
+def add_debugging_flags(parser):
+    from argparse import SUPPRESS
+
+    parser.add_argument(
+        "--cprofile", action="store_true", default=False, help=SUPPRESS
+    )
+    parser.add_argument("--cprofile-dump", help=SUPPRESS)
+    parser.add_argument(
+        "--pdb", action="store_true", default=False, help=SUPPRESS
+    )
+    parser.add_argument(
+        "--instrument", action="store_true", default=False, help=SUPPRESS
+    )
+    parser.add_argument(
+        "--instrument-open", action="store_true", default=False, help=SUPPRESS
+    )

dvc/analytics.py

@@ -1,24 +1,8 @@
 import json
 import logging
 import os
-import platform
-import sys
-import tempfile
-import uuid
 
-import distro
-import requests
-
-from dvc import __version__
-from dvc.config import Config, to_bool
-from dvc.daemon import daemon
-from dvc.exceptions import NotDvcRepoError
-from dvc.lock import Lock, LockError
-from dvc.repo import Repo
-from dvc.scm import SCM, NoSCM
-from dvc.scm.base import SCMError
-from dvc.utils import env2bool, is_binary
-from dvc.utils.fs import makedirs
+from .env import DVC_NO_ANALYTICS
 
 logger = logging.getLogger(__name__)
 
@@ -35,6 +19,10 @@ def collect_and_send_report(args=None, return_code=None):
     report as a JSON, where the _collector_ generates it and the _sender_
     removes it after sending it.
     """
+    import tempfile
+
+    from dvc.daemon import daemon
+
     report = {}
 
     # Include command execution information on the report only when available.
@@ -46,16 +34,22 @@ def collect_and_send_report(args=None, return_code=None):
 
     with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
         json.dump(report, fobj)
-        daemon(["analytics", fobj.name])
+    daemon(["analytics", fobj.name])
 
 
 def is_enabled():
+    from dvc.config import Config, to_bool
+    from dvc.utils import env2bool
+
     if env2bool("DVC_TEST"):
         return False
 
-    enabled = to_bool(
-        Config(validate=False).get("core", {}).get("analytics", "true")
-    )
+    enabled = not os.getenv(DVC_NO_ANALYTICS)
+    if enabled:
+        enabled = to_bool(
+            Config(validate=False).get("core", {}).get("analytics", "true")
+        )
+
     logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
 
     return enabled
@@ -69,6 +63,8 @@ def send(path):
     `collect_and_send_report`. Sending happens on another process,
     thus, the need of removing such file afterwards.
     """
+    import requests
+
     url = "https://analytics.dvc.org"
     headers = {"content-type": "application/json"}
 
@@ -86,6 +82,11 @@ def send(path):
 
 
 def _scm_in_use():
+    from dvc.exceptions import NotDvcRepoError
+    from dvc.repo import Repo
+    from dvc.scm import SCM, NoSCM
+    from dvc.scm.base import SCMError
+
     try:
         scm = SCM(root_dir=Repo.find_root())
         return type(scm).__name__
@@ -99,6 +100,9 @@ def _runtime_info():
     """
     Gather information from the environment where DVC runs to fill a report.
     """
+    from dvc import __version__
+    from dvc.utils import is_binary
+
     return {
         "dvc_version": __version__,
         "is_binary": is_binary(),
@@ -109,6 +113,11 @@ def _runtime_info():
 
 
 def _system_info():
+    import platform
+    import sys
+
+    import distro
+
     system = platform.system()
 
     if system == "Windows":
@@ -147,6 +156,12 @@ def _find_or_create_user_id():
 
     IDs are generated randomly with UUID.
     """
+    import uuid
+
+    from dvc.config import Config
+    from dvc.lock import Lock, LockError
+    from dvc.utils.fs import makedirs
+
     config_dir = Config.get_dir("global")
     fname = os.path.join(config_dir, "user_id")
     lockfile = os.path.join(config_dir, "user_id.lock")

dvc/api.py

@@ -1,15 +1,9 @@
 import os
 from contextlib import _GeneratorContextManager as GCM
-from contextlib import contextmanager
 
 from funcy import reraise
 
-from dvc.exceptions import (
-    NotDvcRepoError,
-    OutputNotFoundError,
-    PathMissingError,
-)
-from dvc.external_repo import external_repo
+from dvc.exceptions import OutputNotFoundError, PathMissingError
 from dvc.path_info import PathInfo
 from dvc.repo import Repo
 
@@ -20,23 +14,22 @@ def get_url(path, repo=None, rev=None, remote=None):
     in a DVC repo. For Git repos, HEAD is used unless a rev argument is
     supplied. The default remote is tried unless a remote argument is supplied.
 
-    Raises OutputNotFoundError if the file is not a dvc-tracked file.
+    Raises OutputNotFoundError if the file is not tracked by DVC.
 
     NOTE: This function does not check for the actual existence of the file or
     directory in the remote storage.
     """
-    with _make_repo(repo, rev=rev) as _repo:
+    with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
         path_info = PathInfo(_repo.root_dir) / path
         with reraise(FileNotFoundError, PathMissingError(path, repo)):
-            metadata = _repo.repo_tree.metadata(path_info)
+            metadata = _repo.repo_fs.metadata(path_info)
 
         if not metadata.is_dvc:
             raise OutputNotFoundError(path, repo)
 
         cloud = metadata.repo.cloud
-        with _repo.state:
-            hash_info = _repo.repo_tree.get_hash(path_info)
-        return cloud.get_url_for(remote, checksum=hash_info.value)
+        md5 = metadata.repo.dvcfs.info(path_info)["md5"]
+        return cloud.get_url_for(remote, checksum=md5)
 
 
 def open(  # noqa, pylint: disable=redefined-builtin
@@ -79,7 +72,7 @@ class _OpenContextManager(GCM):
 
 
 def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
-    with _make_repo(repo, rev=rev) as _repo:
+    with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
         with _repo.open_by_relpath(
             path, remote=remote, mode=mode, encoding=encoding
         ) as fd:
@@ -98,14 +91,33 @@ def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
         return fd.read()
 
 
-@contextmanager
-def _make_repo(repo_url=None, rev=None):
-    repo_url = repo_url or os.getcwd()
-    if rev is None and os.path.exists(repo_url):
-        try:
-            yield Repo(repo_url, subrepos=True)
-            return
-        except NotDvcRepoError:
-            pass  # fallthrough to external_repo
-    with external_repo(url=repo_url, rev=rev) as repo:
-        yield repo
+def make_checkpoint():
+    """
+    Signal DVC to create a checkpoint experiment.
+
+    If the current process is being run from DVC, this function will block
+    until DVC has finished creating the checkpoint. Otherwise, this function
+    will return immediately.
+    """
+    import builtins
+    from time import sleep
+
+    from dvc.env import DVC_CHECKPOINT, DVC_ROOT
+    from dvc.stage.monitor import CheckpointTask
+
+    if os.getenv(DVC_CHECKPOINT) is None:
+        return
+
+    root_dir = os.getenv(DVC_ROOT, Repo.find_root())
+    signal_file = os.path.join(
+        root_dir, Repo.DVC_DIR, "tmp", CheckpointTask.SIGNAL_FILE
+    )
+
+    with builtins.open(signal_file, "w") as fobj:
+        # NOTE: force flushing/writing empty file to disk, otherwise when
+        # run in certain contexts (pytest) file may not actually be written
+        fobj.write("")
+        fobj.flush()
+        os.fsync(fobj.fileno())
+    while os.path.exists(signal_file):
+        sleep(0.1)

dvc/cache/__init__.py

@@ -1,176 +0,0 @@
-"""Manages cache of a DVC repo."""
-from collections import defaultdict
-
-from funcy import cached_property
-
-
-def _make_remote_property(name):
-    """
-    The config file is stored in a way that allows you to have a
-    cache for each remote.
-
-    This is needed when specifying external outputs
-    (as they require you to have an external cache location).
-
-    Imagine a config file like the following:
-
-            ['remote "dvc-storage"']
-            url = ssh://localhost/tmp
-            ask_password = true
-
-            [cache]
-            ssh = dvc-storage
-
-    This method creates a cached property, containing cache named `name`:
-
-        self.config == {'ssh': 'dvc-storage'}
-        self.ssh  # a RemoteSSH instance
-    """
-
-    def getter(self):
-        from ..tree import get_cloud_tree
-        from .base import CloudCache
-
-        remote = self.config.get(name)
-        if not remote:
-            return None
-
-        tree = get_cloud_tree(self.repo, name=remote)
-        return CloudCache(tree)
-
-    getter.__name__ = name
-    return cached_property(getter)
-
-
-class Cache:
-    """Class that manages cache locations of a DVC repo.
-
-    Args:
-        repo (dvc.repo.Repo): repo instance that this cache belongs to.
-    """
-
-    CACHE_DIR = "cache"
-
-    def __init__(self, repo):
-        from ..tree import get_cloud_tree
-        from .local import LocalCache
-
-        self.repo = repo
-        self.config = config = repo.config["cache"]
-
-        local = config.get("local")
-
-        if local:
-            settings = {"name": local}
-        else:
-            from ..config import LOCAL_COMMON
-
-            settings = {"url": config["dir"]}
-            for opt in LOCAL_COMMON.keys():
-                if opt in config:
-                    settings[str(opt)] = config.get(opt)
-
-        tree = get_cloud_tree(repo, **settings)
-        self.local = LocalCache(tree)
-
-    s3 = _make_remote_property("s3")
-    gs = _make_remote_property("gs")
-    ssh = _make_remote_property("ssh")
-    hdfs = _make_remote_property("hdfs")
-    azure = _make_remote_property("azure")
-
-
-class NamedCacheItem:
-    def __init__(self):
-        self.names = set()
-        self.children = defaultdict(NamedCacheItem)
-
-    def __eq__(self, other):
-        return self.names == other.names and self.children == other.children
-
-    def child_keys(self):
-        for key, child in self.children.items():
-            yield key
-            yield from child.child_keys()
-
-    def child_names(self):
-        for key, child in self.children.items():
-            yield key, child.names
-            yield from child.child_names()
-
-    def add(self, checksum, item):
-        self.children[checksum].update(item)
-
-    def update(self, item, suffix=""):
-        if suffix:
-            self.names.update(n + suffix for n in item.names)
-        else:
-            self.names.update(item.names)
-        for checksum, child_item in item.children.items():
-            self.children[checksum].update(child_item)
-
-
-class NamedCache:
-    # pylint: disable=protected-access
-    def __init__(self):
-        self._items = defaultdict(lambda: defaultdict(NamedCacheItem))
-        self.external = defaultdict(set)
-
-    @classmethod
-    def make(cls, scheme, checksum, name):
-        cache = cls()
-        cache.add(scheme, checksum, name)
-        return cache
-
-    def __getitem__(self, key):
-        return self._items[key]
-
-    def add(self, scheme, checksum, name):
-        """Add a mapped name for the specified checksum."""
-        self._items[scheme][checksum].names.add(name)
-
-    def add_child_cache(self, checksum, cache, suffix=""):
-        """Add/update child cache for the specified checksum."""
-        for scheme, src in cache._items.items():
-            dst = self._items[scheme][checksum].children
-            for child_checksum, item in src.items():
-                dst[child_checksum].update(item, suffix=suffix)
-
-        for repo_pair, files in cache.external.items():
-            self.external[repo_pair].update(files)
-
-    def add_external(self, url, rev, path):
-        self.external[url, rev].add(path)
-
-    def update(self, cache, suffix=""):
-        for scheme, src in cache._items.items():
-            dst = self._items[scheme]
-            for checksum, item in src.items():
-                dst[checksum].update(item, suffix=suffix)
-
-        for repo_pair, files in cache.external.items():
-            self.external[repo_pair].update(files)
-
-    def scheme_keys(self, scheme):
-        """Iterate over a flat list of all keys for the specified scheme,
-        including children.
-        """
-        for key, item in self._items[scheme].items():
-            yield key
-            yield from item.child_keys()
-
-    def scheme_names(self, scheme):
-        """Iterate over a flat list of checksum, names items for the specified
-        scheme, including children.
-        """
-        for key, item in self._items[scheme].items():
-            yield key, item.names
-            yield from item.child_names()
-
-    def dir_keys(self, scheme):
-        return (
-            key for key, item in self._items[scheme].items() if item.children
-        )
-
-    def child_keys(self, scheme, checksum):
-        return self._items[scheme][checksum].child_keys()

dvc/cache/base.py

@@ -1,694 +0,0 @@
-import json
-import logging
-from copy import copy
-from operator import itemgetter
-
-from shortuuid import uuid
-
-import dvc.prompt as prompt
-from dvc.exceptions import (
-    CheckoutError,
-    ConfirmRemoveError,
-    DvcException,
-    MergeError,
-)
-from dvc.hash_info import HashInfo
-from dvc.path_info import WindowsPathInfo
-from dvc.progress import Tqdm
-from dvc.remote.slow_link_detection import slow_link_guard
-
-from ..tree.base import RemoteActionNotImplemented
-
-logger = logging.getLogger(__name__)
-
-STATUS_OK = 1
-STATUS_MISSING = 2
-STATUS_NEW = 3
-STATUS_DELETED = 4
-
-STATUS_MAP = {
-    # (local_exists, remote_exists)
-    (True, True): STATUS_OK,
-    (False, False): STATUS_MISSING,
-    (True, False): STATUS_NEW,
-    (False, True): STATUS_DELETED,
-}
-
-
-class DirCacheError(DvcException):
-    def __init__(self, hash_):
-        super().__init__(
-            f"Failed to load dir cache for hash value: '{hash_}'."
-        )
-
-
-class CloudCache:
-    """Cloud cache class."""
-
-    DEFAULT_CACHE_TYPES = ["copy"]
-    CACHE_MODE = None
-
-    def __init__(self, tree):
-        self.tree = tree
-        self.repo = tree.repo
-
-        self.cache_types = tree.config.get("type") or copy(
-            self.DEFAULT_CACHE_TYPES
-        )
-        self.cache_type_confirmed = False
-        self._dir_info = {}
-
-    def get_dir_cache(self, hash_info):
-        assert hash_info
-
-        dir_info = self._dir_info.get(hash_info.value)
-        if dir_info:
-            return dir_info
-
-        try:
-            dir_info = self.load_dir_cache(hash_info)
-        except DirCacheError:
-            dir_info = []
-
-        self._dir_info[hash_info.value] = dir_info
-        return dir_info
-
-    def load_dir_cache(self, hash_info):
-        path_info = self.tree.hash_to_path_info(hash_info.value)
-
-        try:
-            with self.tree.open(path_info, "r") as fobj:
-                d = json.load(fobj)
-        except (ValueError, FileNotFoundError) as exc:
-            raise DirCacheError(hash_info) from exc
-
-        if not isinstance(d, list):
-            logger.error(
-                "dir cache file format error '%s' [skipping the file]",
-                path_info,
-            )
-            return []
-
-        if self.tree.PATH_CLS == WindowsPathInfo:
-            # only need to convert it for Windows
-            for info in d:
-                # NOTE: here is a BUG, see comment to .as_posix() below
-                relpath = info[self.tree.PARAM_RELPATH]
-                info[self.tree.PARAM_RELPATH] = relpath.replace(
-                    "/", self.tree.PATH_CLS.sep
-                )
-
-        return d
-
-    def changed(self, path_info, hash_info):
-        """Checks if data has changed.
-
-        A file is considered changed if:
-            - It doesn't exist on the working directory (was unlinked)
-            - Hash value is not computed (saving a new file)
-            - The hash value stored is different from the given one
-            - There's no file in the cache
-
-        Args:
-            path_info: dict with path information.
-            hash: expected hash value for this data.
-
-        Returns:
-            bool: True if data has changed, False otherwise.
-        """
-
-        logger.debug(
-            "checking if '%s'('%s') has changed.", path_info, hash_info
-        )
-
-        if not self.tree.exists(path_info):
-            logger.debug("'%s' doesn't exist.", path_info)
-            return True
-
-        if not hash_info:
-            logger.debug("hash value for '%s' is missing.", path_info)
-            return True
-
-        if self.changed_cache(hash_info):
-            logger.debug(
-                "cache for '%s'('%s') has changed.", path_info, hash_info
-            )
-            return True
-
-        actual = self.tree.get_hash(path_info)
-        if hash_info != actual:
-            logger.debug(
-                "hash value '%s' for '%s' has changed (actual '%s').",
-                hash_info,
-                actual,
-                path_info,
-            )
-            return True
-
-        logger.debug("'%s' hasn't changed.", path_info)
-        return False
-
-    def link(self, from_info, to_info):
-        self._link(from_info, to_info, self.cache_types)
-
-    def _link(self, from_info, to_info, link_types):
-        assert self.tree.isfile(from_info)
-
-        self.tree.makedirs(to_info.parent)
-
-        self._try_links(from_info, to_info, link_types)
-
-    def _verify_link(self, path_info, link_type):
-        if self.cache_type_confirmed:
-            return
-
-        is_link = getattr(self.tree, f"is_{link_type}", None)
-        if is_link and not is_link(path_info):
-            self.tree.remove(path_info)
-            raise DvcException(f"failed to verify {link_type}")
-
-        self.cache_type_confirmed = True
-
-    @slow_link_guard
-    def _try_links(self, from_info, to_info, link_types):
-        while link_types:
-            link_method = getattr(self.tree, link_types[0])
-            try:
-                self._do_link(from_info, to_info, link_method)
-                self._verify_link(to_info, link_types[0])
-                return
-
-            except DvcException as exc:
-                logger.debug(
-                    "Cache type '%s' is not supported: %s", link_types[0], exc
-                )
-                del link_types[0]
-
-        raise DvcException("no possible cache types left to try out.")
-
-    def _do_link(self, from_info, to_info, link_method):
-        if self.tree.exists(to_info):
-            raise DvcException(f"Link '{to_info}' already exists!")
-
-        link_method(from_info, to_info)
-
-        logger.debug(
-            "Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
-        )
-
-    def _save_file(self, path_info, tree, hash_info, save_link=True, **kwargs):
-        assert hash_info
-
-        cache_info = self.tree.hash_to_path_info(hash_info.value)
-        if tree == self.tree:
-            if self.changed_cache(hash_info):
-                self.tree.move(path_info, cache_info, mode=self.CACHE_MODE)
-                self.link(cache_info, path_info)
-            elif self.tree.iscopy(path_info) and self._cache_is_copy(
-                path_info
-            ):
-                # Default relink procedure involves unneeded copy
-                self.tree.unprotect(path_info)
-            else:
-                self.tree.remove(path_info)
-                self.link(cache_info, path_info)
-
-            if save_link:
-                self.tree.state.save_link(path_info)
-            # we need to update path and cache, since in case of reflink,
-            # or copy cache type moving original file results in updates on
-            # next executed command, which causes md5 recalculation
-            self.tree.state.save(path_info, hash_info.value)
-        else:
-            if self.changed_cache(hash_info):
-                with tree.open(path_info, mode="rb") as fobj:
-                    # if tree has fetch enabled, DVC out will be fetched on
-                    # open and we do not need to read/copy any data
-                    if not (
-                        tree.isdvc(path_info, strict=False) and tree.fetch
-                    ):
-                        self.tree.copy_fobj(fobj, cache_info)
-                callback = kwargs.get("download_callback")
-                if callback:
-                    callback(1)
-
-        self.tree.state.save(cache_info, hash_info.value)
-
-    def _cache_is_copy(self, path_info):
-        """Checks whether cache uses copies."""
-        if self.cache_type_confirmed:
-            return self.cache_types[0] == "copy"
-
-        if set(self.cache_types) <= {"copy"}:
-            return True
-
-        workspace_file = path_info.with_name("." + uuid())
-        test_cache_file = self.tree.path_info / ".cache_type_test_file"
-        if not self.tree.exists(test_cache_file):
-            with self.tree.open(test_cache_file, "wb") as fobj:
-                fobj.write(bytes(1))
-        try:
-            self.link(test_cache_file, workspace_file)
-        finally:
-            self.tree.remove(workspace_file)
-            self.tree.remove(test_cache_file)
-
-        self.cache_type_confirmed = True
-        return self.cache_types[0] == "copy"
-
-    def _get_dir_info_hash(self, dir_info):
-        import tempfile
-
-        from dvc.path_info import PathInfo
-        from dvc.utils import tmp_fname
-
-        # Sorting the list by path to ensure reproducibility
-        dir_info = sorted(dir_info, key=itemgetter(self.tree.PARAM_RELPATH))
-
-        tmp = tempfile.NamedTemporaryFile(delete=False).name
-        with open(tmp, "w+") as fobj:
-            json.dump(dir_info, fobj, sort_keys=True)
-
-        from_info = PathInfo(tmp)
-        to_info = self.tree.path_info / tmp_fname("")
-        self.tree.upload(from_info, to_info, no_progress_bar=True)
-
-        hash_info = self.tree.get_file_hash(to_info)
-        hash_info.value += self.tree.CHECKSUM_DIR_SUFFIX
-        hash_info.dir_info = dir_info
-
-        return hash_info, to_info
-
-    def save_dir_info(self, dir_info, hash_info=None):
-        if (
-            hash_info
-            and hash_info.name == self.tree.PARAM_CHECKSUM
-            and not self.changed_cache_file(hash_info)
-        ):
-            return hash_info
-
-        hi, tmp_info = self._get_dir_info_hash(dir_info)
-        new_info = self.tree.hash_to_path_info(hi.value)
-        if self.changed_cache_file(hi):
-            self.tree.makedirs(new_info.parent)
-            self.tree.move(tmp_info, new_info, mode=self.CACHE_MODE)
-
-        self.tree.state.save(new_info, hi.value)
-
-        return hi
-
-    def _save_dir(self, path_info, tree, hash_info, save_link=True, **kwargs):
-        if not hash_info.dir_info:
-            hash_info.dir_info = tree.cache.get_dir_cache(hash_info)
-        hi = self.save_dir_info(hash_info.dir_info, hash_info)
-        for entry in Tqdm(
-            hi.dir_info, desc="Saving " + path_info.name, unit="file"
-        ):
-            entry_info = path_info / entry[self.tree.PARAM_RELPATH]
-            entry_hash = HashInfo(
-                self.tree.PARAM_CHECKSUM, entry[self.tree.PARAM_CHECKSUM]
-            )
-            self._save_file(
-                entry_info, tree, entry_hash, save_link=False, **kwargs
-            )
-
-        if save_link:
-            self.tree.state.save_link(path_info)
-        if self.tree.exists(path_info):
-            self.tree.state.save(path_info, hi.value)
-
-        cache_info = self.tree.hash_to_path_info(hi.value)
-        self.tree.state.save(cache_info, hi.value)
-
-    def save(self, path_info, tree, hash_info, save_link=True, **kwargs):
-        if path_info.scheme != self.tree.scheme:
-            raise RemoteActionNotImplemented(
-                f"save {path_info.scheme} -> {self.tree.scheme}",
-                self.tree.scheme,
-            )
-
-        self._save(path_info, tree, hash_info, save_link, **kwargs)
-
-    def _save(self, path_info, tree, hash_info, save_link=True, **kwargs):
-        to_info = self.tree.hash_to_path_info(hash_info.value)
-        logger.debug("Saving '%s' to '%s'.", path_info, to_info)
-
-        if tree.isdir(path_info):
-            self._save_dir(path_info, tree, hash_info, save_link, **kwargs)
-        else:
-            self._save_file(path_info, tree, hash_info, save_link, **kwargs)
-
-    # Override to return path as a string instead of PathInfo for clouds
-    # which support string paths (see local)
-    def hash_to_path(self, hash_):
-        return self.tree.hash_to_path_info(hash_)
-
-    def changed_cache_file(self, hash_info):
-        """Compare the given hash with the (corresponding) actual one.
-
-        - Use `State` as a cache for computed hashes
-            + The entries are invalidated by taking into account the following:
-                * mtime
-                * inode
-                * size
-                * hash
-
-        - Remove the file from cache if it doesn't match the actual hash
-        """
-        # Prefer string path over PathInfo when possible due to performance
-        cache_info = self.hash_to_path(hash_info.value)
-        if self.tree.is_protected(cache_info):
-            logger.debug(
-                "Assuming '%s' is unchanged since it is read-only", cache_info
-            )
-            return False
-
-        actual = self.tree.get_hash(cache_info)
-
-        logger.debug(
-            "cache '%s' expected '%s' actual '%s'",
-            cache_info,
-            hash_info,
-            actual,
-        )
-
-        if not hash_info or not actual:
-            return True
-
-        if actual.value.split(".")[0] == hash_info.value.split(".")[0]:
-            # making cache file read-only so we don't need to check it
-            # next time
-            self.tree.protect(cache_info)
-            return False
-
-        if self.tree.exists(cache_info):
-            logger.warning("corrupted cache file '%s'.", cache_info)
-            self.tree.remove(cache_info)
-
-        return True
-
-    def _changed_dir_cache(self, hash_info, path_info=None, filter_info=None):
-        if self.changed_cache_file(hash_info):
-            return True
-
-        for entry in self.get_dir_cache(hash_info):
-            entry_hash = HashInfo(
-                self.tree.PARAM_CHECKSUM, entry[self.tree.PARAM_CHECKSUM]
-            )
-
-            if path_info and filter_info:
-                entry_info = path_info / entry[self.tree.PARAM_RELPATH]
-                if not entry_info.isin_or_eq(filter_info):
-                    continue
-
-            if self.changed_cache_file(entry_hash):
-                return True
-
-        return False
-
-    def changed_cache(self, hash_info, path_info=None, filter_info=None):
-        if hash_info.isdir:
-            return self._changed_dir_cache(
-                hash_info, path_info=path_info, filter_info=filter_info
-            )
-        return self.changed_cache_file(hash_info)
-
-    def already_cached(self, path_info):
-        current = self.tree.get_hash(path_info)
-
-        if not current:
-            return False
-
-        return not self.changed_cache(current)
-
-    def safe_remove(self, path_info, force=False):
-        if not self.tree.exists(path_info):
-            return
-
-        if not force and not self.already_cached(path_info):
-            msg = (
-                "file '{}' is going to be removed."
-                " Are you sure you want to proceed?".format(str(path_info))
-            )
-
-            if not prompt.confirm(msg):
-                raise ConfirmRemoveError(str(path_info))
-
-        self.tree.remove(path_info)
-
-    def _checkout_file(
-        self, path_info, hash_info, force, progress_callback=None, relink=False
-    ):
-        """The file is changed we need to checkout a new copy"""
-        added, modified = True, False
-        cache_info = self.tree.hash_to_path_info(hash_info.value)
-        if self.tree.exists(path_info):
-            logger.debug("data '%s' will be replaced.", path_info)
-            self.safe_remove(path_info, force=force)
-            added, modified = False, True
-
-        self.link(cache_info, path_info)
-        self.tree.state.save_link(path_info)
-        self.tree.state.save(path_info, hash_info.value)
-        if progress_callback:
-            progress_callback(str(path_info))
-
-        return added, modified and not relink
-
-    def _checkout_dir(
-        self,
-        path_info,
-        hash_info,
-        force,
-        progress_callback=None,
-        relink=False,
-        filter_info=None,
-    ):
-        added, modified = False, False
-        # Create dir separately so that dir is created
-        # even if there are no files in it
-        if not self.tree.exists(path_info):
-            added = True
-            self.tree.makedirs(path_info)
-
-        dir_info = self.get_dir_cache(hash_info)
-
-        logger.debug("Linking directory '%s'.", path_info)
-
-        for entry in dir_info:
-            relative_path = entry[self.tree.PARAM_RELPATH]
-            entry_hash = entry[self.tree.PARAM_CHECKSUM]
-            entry_cache_info = self.tree.hash_to_path_info(entry_hash)
-            entry_info = path_info / relative_path
-
-            if filter_info and not entry_info.isin_or_eq(filter_info):
-                continue
-
-            entry_hash_info = HashInfo(self.tree.PARAM_CHECKSUM, entry_hash)
-            if relink or self.changed(entry_info, entry_hash_info):
-                modified = True
-                self.safe_remove(entry_info, force=force)
-                self.link(entry_cache_info, entry_info)
-                self.tree.state.save(entry_info, entry_hash)
-            if progress_callback:
-                progress_callback(str(entry_info))
-
-        modified = (
-            self._remove_redundant_files(path_info, dir_info, force)
-            or modified
-        )
-
-        self.tree.state.save_link(path_info)
-        self.tree.state.save(path_info, hash_info.value)
-
-        # relink is not modified, assume it as nochange
-        return added, not added and modified and not relink
-
-    def _remove_redundant_files(self, path_info, dir_info, force):
-        existing_files = set(self.tree.walk_files(path_info))
-
-        needed_files = {
-            path_info / entry[self.tree.PARAM_RELPATH] for entry in dir_info
-        }
-        redundant_files = existing_files - needed_files
-        for path in redundant_files:
-            self.safe_remove(path, force)
-
-        return bool(redundant_files)
-
-    def checkout(
-        self,
-        path_info,
-        hash_info,
-        force=False,
-        progress_callback=None,
-        relink=False,
-        filter_info=None,
-    ):
-        if path_info.scheme not in ["local", self.tree.scheme]:
-            raise NotImplementedError
-
-        failed = None
-        skip = False
-        if not hash_info:
-            logger.warning(
-                "No file hash info found for '%s'. " "It won't be created.",
-                path_info,
-            )
-            self.safe_remove(path_info, force=force)
-            failed = path_info
-
-        elif not relink and not self.changed(path_info, hash_info):
-            logger.debug("Data '%s' didn't change.", path_info)
-            skip = True
-
-        elif self.changed_cache(
-            hash_info, path_info=path_info, filter_info=filter_info
-        ):
-            logger.warning(
-                "Cache '%s' not found. File '%s' won't be created.",
-                hash_info,
-                path_info,
-            )
-            self.safe_remove(path_info, force=force)
-            failed = path_info
-
-        if failed or skip:
-            if progress_callback:
-                progress_callback(
-                    str(path_info),
-                    self.get_files_number(
-                        self.tree.path_info, hash_info, filter_info
-                    ),
-                )
-            if failed:
-                raise CheckoutError([failed])
-            return
-
-        logger.debug(
-            "Checking out '%s' with cache '%s'.", path_info, hash_info
-        )
-
-        return self._checkout(
-            path_info,
-            hash_info,
-            force,
-            progress_callback,
-            relink,
-            filter_info,
-        )
-
-    def _checkout(
-        self,
-        path_info,
-        hash_info,
-        force=False,
-        progress_callback=None,
-        relink=False,
-        filter_info=None,
-    ):
-        if not hash_info.isdir:
-            return self._checkout_file(
-                path_info, hash_info, force, progress_callback, relink
-            )
-
-        return self._checkout_dir(
-            path_info, hash_info, force, progress_callback, relink, filter_info
-        )
-
-    def get_files_number(self, path_info, hash_info, filter_info):
-        from funcy.py3 import ilen
-
-        if not hash_info:
-            return 0
-
-        if not hash_info.isdir:
-            return 1
-
-        if not filter_info:
-            return len(self.get_dir_cache(hash_info))
-
-        return ilen(
-            filter_info.isin_or_eq(path_info / entry[self.tree.PARAM_CHECKSUM])
-            for entry in self.get_dir_cache(hash_info)
-        )
-
-    def _to_dict(self, dir_info):
-        return {
-            entry[self.tree.PARAM_RELPATH]: entry[self.tree.PARAM_CHECKSUM]
-            for entry in dir_info
-        }
-
-    def _from_dict(self, dir_dict):
-        return [
-            {
-                self.tree.PARAM_RELPATH: relpath,
-                self.tree.PARAM_CHECKSUM: checksum,
-            }
-            for relpath, checksum in dir_dict.items()
-        ]
-
-    @staticmethod
-    def _diff(ancestor, other, allow_removed=False):
-        from dictdiffer import diff
-
-        allowed = ["add"]
-        if allow_removed:
-            allowed.append("remove")
-
-        result = list(diff(ancestor, other))
-        for typ, _, _ in result:
-            if typ not in allowed:
-                raise MergeError(
-                    "unable to auto-merge directories with diff that contains "
-                    f"'{typ}'ed files"
-                )
-        return result
-
-    def _merge_dirs(self, ancestor_info, our_info, their_info):
-        from dictdiffer import patch
-
-        ancestor = self._to_dict(ancestor_info)
-        our = self._to_dict(our_info)
-        their = self._to_dict(their_info)
-
-        our_diff = self._diff(ancestor, our)
-        if not our_diff:
-            return self._from_dict(their)
-
-        their_diff = self._diff(ancestor, their)
-        if not their_diff:
-            return self._from_dict(our)
-
-        # make sure there are no conflicting files
-        self._diff(our, their, allow_removed=True)
-
-        merged = patch(our_diff + their_diff, ancestor, in_place=True)
-
-        # Sorting the list by path to ensure reproducibility
-        return sorted(
-            self._from_dict(merged), key=itemgetter(self.tree.PARAM_RELPATH)
-        )
-
-    def merge(self, ancestor_info, our_info, their_info):
-        assert our_info
-        assert their_info
-
-        if ancestor_info:
-            ancestor = self.get_dir_cache(ancestor_info)
-        else:
-            ancestor = []
-
-        our = self.get_dir_cache(our_info)
-        their = self.get_dir_cache(their_info)
-
-        merged = self._merge_dirs(ancestor, our, their)
-        return self.save_dir_info(merged)
-
-    def get_hash(self, tree, path_info):
-        hash_info = tree.get_hash(path_info)
-        if not hash_info.isdir:
-            assert hash_info.name == self.tree.PARAM_CHECKSUM
-            return hash_info
-
-        return self.save_dir_info(hash_info.dir_info, hash_info)

dvc/cache/local.py

@@ -1,472 +0,0 @@
-import errno
-import logging
-import os
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from functools import partial, wraps
-
-from funcy import cached_property, concat
-
-from dvc.exceptions import DownloadError, UploadError
-from dvc.hash_info import HashInfo
-from dvc.path_info import PathInfo
-from dvc.progress import Tqdm
-
-from ..remote.base import index_locked
-from ..tree.local import LocalTree
-from .base import (
-    STATUS_DELETED,
-    STATUS_MAP,
-    STATUS_MISSING,
-    STATUS_NEW,
-    CloudCache,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def _log_exceptions(func, operation):
-    @wraps(func)
-    def wrapper(from_info, to_info, *args, **kwargs):
-        try:
-            func(from_info, to_info, *args, **kwargs)
-            return 0
-        except Exception as exc:  # pylint: disable=broad-except
-            # NOTE: this means we ran out of file descriptors and there is no
-            # reason to try to proceed, as we will hit this error anyways.
-            # pylint: disable=no-member
-            if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
-                raise
-
-            logger.exception(
-                "failed to %s '%s' to '%s'", operation, from_info, to_info
-            )
-            return 1
-
-    return wrapper
-
-
-class LocalCache(CloudCache):
-    DEFAULT_CACHE_TYPES = ["reflink", "copy"]
-    CACHE_MODE = LocalTree.CACHE_MODE
-
-    def __init__(self, tree):
-        super().__init__(tree)
-        self.cache_dir = tree.config.get("url")
-
-    @property
-    def cache_dir(self):
-        return self.tree.path_info.fspath if self.tree.path_info else None
-
-    @cache_dir.setter
-    def cache_dir(self, value):
-        self.tree.path_info = PathInfo(value) if value else None
-
-    @classmethod
-    def supported(cls, config):  # pylint: disable=unused-argument
-        return True
-
-    @cached_property
-    def cache_path(self):
-        return os.path.abspath(self.cache_dir)
-
-    def hash_to_path(self, hash_):
-        # NOTE: `self.cache_path` is already normalized so we can simply use
-        # `os.sep` instead of `os.path.join`. This results in this helper
-        # being ~5.5 times faster.
-        return f"{self.cache_path}{os.sep}{hash_[0:2]}{os.sep}{hash_[2:]}"
-
-    def hashes_exist(
-        self, hashes, jobs=None, name=None
-    ):  # pylint: disable=unused-argument
-        return [
-            hash_
-            for hash_ in Tqdm(
-                hashes,
-                unit="file",
-                desc="Querying "
-                + ("cache in " + name if name else "local cache"),
-            )
-            if not self.changed_cache_file(
-                HashInfo(self.tree.PARAM_CHECKSUM, hash_)
-            )
-        ]
-
-    def already_cached(self, path_info):
-        assert path_info.scheme in ["", "local"]
-
-        return super().already_cached(path_info)
-
-    def _verify_link(self, path_info, link_type):
-        if link_type == "hardlink" and self.tree.getsize(path_info) == 0:
-            return
-
-        super()._verify_link(path_info, link_type)
-
-    @index_locked
-    def status(
-        self,
-        named_cache,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-        log_missing=True,
-    ):
-        # Return flattened dict containing all status info
-        dir_status, file_status, _ = self._status(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=download,
-            log_missing=log_missing,
-        )
-        return dict(dir_status, **file_status)
-
-    def _status(
-        self,
-        named_cache,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-        log_missing=True,
-    ):
-        """Return a tuple of (dir_status_info, file_status_info, dir_contents).
-
-        dir_status_info contains status for .dir files, file_status_info
-        contains status for all other files, and dir_contents is a dict of
-        {dir_hash: set(file_hash, ...)} which can be used to map
-        a .dir file to its file contents.
-        """
-        logger.debug(
-            f"Preparing to collect status from {remote.tree.path_info}"
-        )
-        md5s = set(named_cache.scheme_keys(self.tree.scheme))
-
-        logger.debug("Collecting information from local cache...")
-        local_exists = frozenset(
-            self.hashes_exist(md5s, jobs=jobs, name=self.cache_dir)
-        )
-
-        # This is a performance optimization. We can safely assume that,
-        # if the resources that we want to fetch are already cached,
-        # there's no need to check the remote storage for the existence of
-        # those files.
-        if download and local_exists == md5s:
-            remote_exists = local_exists
-        else:
-            logger.debug("Collecting information from remote cache...")
-            remote_exists = set()
-            dir_md5s = set(named_cache.dir_keys(self.tree.scheme))
-            if dir_md5s:
-                remote_exists.update(
-                    self._indexed_dir_hashes(named_cache, remote, dir_md5s)
-                )
-                md5s.difference_update(remote_exists)
-            if md5s:
-                remote_exists.update(
-                    remote.hashes_exist(
-                        md5s, jobs=jobs, name=str(remote.tree.path_info)
-                    )
-                )
-        return self._make_status(
-            named_cache,
-            show_checksums,
-            local_exists,
-            remote_exists,
-            log_missing,
-        )
-
-    def _make_status(
-        self,
-        named_cache,
-        show_checksums,
-        local_exists,
-        remote_exists,
-        log_missing,
-    ):
-        def make_names(hash_, names):
-            return {"name": hash_ if show_checksums else " ".join(names)}
-
-        dir_status = {}
-        file_status = {}
-        dir_contents = {}
-        for hash_, item in named_cache[self.tree.scheme].items():
-            if item.children:
-                dir_status[hash_] = make_names(hash_, item.names)
-                dir_contents[hash_] = set()
-                for child_hash, child in item.children.items():
-                    file_status[child_hash] = make_names(
-                        child_hash, child.names
-                    )
-                    dir_contents[hash_].add(child_hash)
-            else:
-                file_status[hash_] = make_names(hash_, item.names)
-
-        self._fill_statuses(dir_status, local_exists, remote_exists)
-        self._fill_statuses(file_status, local_exists, remote_exists)
-
-        if log_missing:
-            self._log_missing_caches(dict(dir_status, **file_status))
-
-        return dir_status, file_status, dir_contents
-
-    def _indexed_dir_hashes(self, named_cache, remote, dir_md5s):
-        # Validate our index by verifying all indexed .dir hashes
-        # still exist on the remote
-        indexed_dirs = set(remote.index.dir_hashes())
-        indexed_dir_exists = set()
-        if indexed_dirs:
-            indexed_dir_exists.update(
-                remote.tree.list_hashes_exists(indexed_dirs)
-            )
-            missing_dirs = indexed_dirs.difference(indexed_dir_exists)
-            if missing_dirs:
-                logger.debug(
-                    "Remote cache missing indexed .dir hashes '{}', "
-                    "clearing remote index".format(", ".join(missing_dirs))
-                )
-                remote.index.clear()
-
-        # Check if non-indexed (new) dir hashes exist on remote
-        dir_exists = dir_md5s.intersection(indexed_dir_exists)
-        dir_exists.update(
-            remote.tree.list_hashes_exists(dir_md5s - dir_exists)
-        )
-
-        # If .dir hash exists on the remote, assume directory contents
-        # still exists on the remote
-        for dir_hash in dir_exists:
-            file_hashes = list(
-                named_cache.child_keys(self.tree.scheme, dir_hash)
-            )
-            if dir_hash not in remote.index:
-                logger.debug(
-                    "Indexing new .dir '{}' with '{}' nested files".format(
-                        dir_hash, len(file_hashes)
-                    )
-                )
-                remote.index.update([dir_hash], file_hashes)
-            yield dir_hash
-            yield from file_hashes
-
-    @staticmethod
-    def _fill_statuses(hash_info_dir, local_exists, remote_exists):
-        # Using sets because they are way faster for lookups
-        local = set(local_exists)
-        remote = set(remote_exists)
-
-        for md5, info in hash_info_dir.items():
-            status = STATUS_MAP[(md5 in local, md5 in remote)]
-            info["status"] = status
-
-    def _get_plans(self, download, remote, status_info, status):
-        cache = []
-        path_infos = []
-        names = []
-        hashes = []
-        missing = []
-        for md5, info in Tqdm(
-            status_info.items(), desc="Analysing status", unit="file"
-        ):
-            if info["status"] == status:
-                cache.append(self.tree.hash_to_path_info(md5))
-                path_infos.append(remote.tree.hash_to_path_info(md5))
-                names.append(info["name"])
-                hashes.append(md5)
-            elif info["status"] == STATUS_MISSING:
-                missing.append(md5)
-
-        if download:
-            to_infos = cache
-            from_infos = path_infos
-        else:
-            to_infos = path_infos
-            from_infos = cache
-
-        return (from_infos, to_infos, names, hashes), missing
-
-    def _process(
-        self,
-        named_cache,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-    ):
-        logger.debug(
-            "Preparing to {} '{}'".format(
-                "download data from" if download else "upload data to",
-                remote.tree.path_info,
-            )
-        )
-
-        if download:
-            func = partial(
-                _log_exceptions(remote.tree.download, "download"),
-                dir_mode=self.tree.dir_mode,
-                file_mode=self.tree.file_mode,
-            )
-            status = STATUS_DELETED
-            desc = "Downloading"
-        else:
-            func = _log_exceptions(remote.tree.upload, "upload")
-            status = STATUS_NEW
-            desc = "Uploading"
-
-        if jobs is None:
-            jobs = remote.tree.JOBS
-
-        dir_status, file_status, dir_contents = self._status(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=download,
-        )
-
-        dir_plans, _ = self._get_plans(download, remote, dir_status, status)
-        file_plans, missing_files = self._get_plans(
-            download, remote, file_status, status
-        )
-
-        total = len(dir_plans[0]) + len(file_plans[0])
-        if total == 0:
-            return 0
-
-        with Tqdm(total=total, unit="file", desc=desc) as pbar:
-            func = pbar.wrap_fn(func)
-            with ThreadPoolExecutor(max_workers=jobs) as executor:
-                if download:
-                    from_infos, to_infos, names, _ = (
-                        d + f for d, f in zip(dir_plans, file_plans)
-                    )
-                    fails = sum(
-                        executor.map(func, from_infos, to_infos, names)
-                    )
-                else:
-                    # for uploads, push files first, and any .dir files last
-
-                    file_futures = {}
-                    for from_info, to_info, name, hash_ in zip(*file_plans):
-                        file_futures[hash_] = executor.submit(
-                            func, from_info, to_info, name
-                        )
-                    dir_futures = {}
-                    for from_info, to_info, name, dir_hash in zip(*dir_plans):
-                        # if for some reason a file contained in this dir is
-                        # missing both locally and in the remote, we want to
-                        # push whatever file content we have, but should not
-                        # push .dir file
-                        for file_hash in missing_files:
-                            if file_hash in dir_contents[dir_hash]:
-                                logger.debug(
-                                    "directory '%s' contains missing files,"
-                                    "skipping .dir file upload",
-                                    name,
-                                )
-                                break
-                        else:
-                            wait_futures = {
-                                future
-                                for file_hash, future in file_futures.items()
-                                if file_hash in dir_contents[dir_hash]
-                            }
-                            dir_futures[dir_hash] = executor.submit(
-                                self._dir_upload,
-                                func,
-                                wait_futures,
-                                from_info,
-                                to_info,
-                                name,
-                            )
-                    fails = sum(
-                        future.result()
-                        for future in concat(
-                            file_futures.values(), dir_futures.values()
-                        )
-                    )
-
-        if fails:
-            if download:
-                remote.index.clear()
-                raise DownloadError(fails)
-            raise UploadError(fails)
-
-        if not download:
-            # index successfully pushed dirs
-            for dir_hash, future in dir_futures.items():
-                if future.result() == 0:
-                    file_hashes = dir_contents[dir_hash]
-                    logger.debug(
-                        "Indexing pushed dir '{}' with "
-                        "'{}' nested files".format(dir_hash, len(file_hashes))
-                    )
-                    remote.index.update([dir_hash], file_hashes)
-
-        return len(dir_plans[0]) + len(file_plans[0])
-
-    @staticmethod
-    def _dir_upload(func, futures, from_info, to_info, name):
-        for future in as_completed(futures):
-            if future.result():
-                # do not upload this .dir file if any file in this
-                # directory failed to upload
-                logger.debug(
-                    "failed to upload full contents of '{}', "
-                    "aborting .dir file upload".format(name)
-                )
-                logger.error(f"failed to upload '{from_info}' to '{to_info}'")
-                return 1
-        return func(from_info, to_info, name)
-
-    @index_locked
-    def push(self, named_cache, remote, jobs=None, show_checksums=False):
-        return self._process(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=False,
-        )
-
-    @index_locked
-    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
-        ret = self._process(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=True,
-        )
-
-        if not remote.tree.verify:
-            for checksum in named_cache.scheme_keys("local"):
-                cache_file = self.tree.hash_to_path_info(checksum)
-                if self.tree.exists(cache_file):
-                    # We can safely save here, as existing corrupted files will
-                    # be removed upon status, while files corrupted during
-                    # download will not be moved from tmp_file
-                    # (see `BaseTree.download()`)
-                    self.tree.state.save(cache_file, checksum)
-
-        return ret
-
-    @staticmethod
-    def _log_missing_caches(hash_info_dict):
-        missing_caches = [
-            (md5, info)
-            for md5, info in hash_info_dict.items()
-            if info["status"] == STATUS_MISSING
-        ]
-        if missing_caches:
-            missing_desc = "\n".join(
-                "name: {}, md5: {}".format(info["name"], md5)
-                for md5, info in missing_caches
-            )
-            msg = (
-                "Some of the cache files do not exist neither locally "
-                "nor on remote. Missing cache files:\n{}".format(missing_desc)
-            )
-            logger.warning(msg)

dvc/checkout.py

@@ -0,0 +1,351 @@
+import logging
+
+from shortuuid import uuid
+
+from dvc import prompt
+from dvc.exceptions import (
+    CacheLinkError,
+    CheckoutError,
+    ConfirmRemoveError,
+    DvcException,
+)
+from dvc.ignore import DvcIgnoreFilter
+from dvc.objects import check, load
+from dvc.objects.db.slow_link_detection import (  # type: ignore[attr-defined]
+    slow_link_guard,
+)
+from dvc.objects.errors import ObjectFormatError
+from dvc.objects.stage import stage
+from dvc.types import Optional
+
+logger = logging.getLogger(__name__)
+
+
+def _changed(path_info, fs, obj, cache, state=None):
+    logger.trace("checking if '%s'('%s') has changed.", path_info, obj)
+
+    try:
+        check(cache, obj)
+    except (FileNotFoundError, ObjectFormatError):
+        logger.debug(
+            "cache for '%s'('%s') has changed.", path_info, obj.hash_info
+        )
+        return True
+
+    actual = state.get(path_info, fs) if state else None
+    if actual is None:
+        try:
+            actual = stage(
+                cache, path_info, fs, obj.hash_info.name, dry_run=True
+            )[1].hash_info
+        except FileNotFoundError:
+            logger.debug("'%s' doesn't exist.", path_info)
+            return True
+
+    if obj.hash_info != actual:
+        logger.debug(
+            "hash value '%s' for '%s' has changed (actual '%s').",
+            obj.hash_info,
+            actual,
+            path_info,
+        )
+        return True
+
+    logger.trace("'%s' hasn't changed.", path_info)
+    return False
+
+
+def _remove(path_info, fs, cache, force=False):
+    if not fs.exists(path_info):
+        return
+
+    if force:
+        fs.remove(path_info)
+        return
+
+    current = stage(cache, path_info, fs, fs.PARAM_CHECKSUM, dry_run=True)[
+        1
+    ].hash_info
+    try:
+        obj = load(cache, current)
+        check(cache, obj)
+    except (FileNotFoundError, ObjectFormatError):
+        msg = (
+            f"file/directory '{path_info}' is going to be removed. "
+            "Are you sure you want to proceed?"
+        )
+
+        if not prompt.confirm(msg):
+            raise ConfirmRemoveError(str(path_info))
+
+    fs.remove(path_info)
+
+
+def _verify_link(cache, path_info, link_type):
+    if link_type == "hardlink" and cache.fs.getsize(path_info) == 0:
+        return
+
+    if cache.cache_type_confirmed:
+        return
+
+    is_link = getattr(cache.fs, f"is_{link_type}", None)
+    if is_link and not is_link(path_info):
+        cache.fs.remove(path_info)
+        raise DvcException(f"failed to verify {link_type}")
+
+    cache.cache_type_confirmed = True
+
+
+def _do_link(cache, from_info, to_info, link_method):
+    if cache.fs.exists(to_info):
+        raise DvcException(f"Link '{to_info}' already exists!")
+
+    link_method(from_info, to_info)
+
+    logger.debug(
+        "Created '%s': %s -> %s", cache.cache_types[0], from_info, to_info
+    )
+
+
+@slow_link_guard
+def _try_links(cache, from_info, to_info, link_types):
+    while link_types:
+        link_method = getattr(cache.fs, link_types[0])
+        try:
+            _do_link(cache, from_info, to_info, link_method)
+            _verify_link(cache, to_info, link_types[0])
+            return
+
+        except DvcException as exc:
+            logger.debug(
+                "Cache type '%s' is not supported: %s", link_types[0], exc
+            )
+            del link_types[0]
+
+    raise CacheLinkError([to_info])
+
+
+def _link(cache, from_info, to_info):
+    cache.makedirs(to_info.parent)
+    try:
+        _try_links(cache, from_info, to_info, cache.cache_types)
+    except FileNotFoundError as exc:
+        raise CheckoutError([to_info]) from exc
+
+
+def _cache_is_copy(cache, path_info):
+    """Checks whether cache uses copies."""
+    if cache.cache_type_confirmed:
+        return cache.cache_types[0] == "copy"
+
+    if set(cache.cache_types) <= {"copy"}:
+        return True
+
+    workspace_file = path_info.with_name("." + uuid())
+    test_cache_file = cache.path_info / ".cache_type_test_file"
+    if not cache.fs.exists(test_cache_file):
+        cache.makedirs(test_cache_file.parent)
+        with cache.fs.open(test_cache_file, "wb") as fobj:
+            fobj.write(bytes(1))
+    try:
+        _link(cache, test_cache_file, workspace_file)
+    finally:
+        cache.fs.remove(workspace_file)
+        cache.fs.remove(test_cache_file)
+
+    cache.cache_type_confirmed = True
+    return cache.cache_types[0] == "copy"
+
+
+def _checkout_file(
+    path_info,
+    fs,
+    obj,
+    cache,
+    force,
+    progress_callback=None,
+    relink=False,
+    state=None,
+):
+    """The file is changed we need to checkout a new copy"""
+    modified = False
+    cache_info = cache.hash_to_path_info(obj.hash_info.value)
+    if fs.exists(path_info):
+        if not relink and _changed(path_info, fs, obj, cache, state=state):
+            modified = True
+            _remove(path_info, fs, cache, force=force)
+            _link(cache, cache_info, path_info)
+        else:
+            if fs.iscopy(path_info) and _cache_is_copy(cache, path_info):
+                cache.unprotect(path_info)
+            else:
+                _remove(path_info, fs, cache, force=force)
+                _link(cache, cache_info, path_info)
+    else:
+        _link(cache, cache_info, path_info)
+        modified = True
+
+    if state:
+        state.save(path_info, fs, obj.hash_info)
+
+    if progress_callback:
+        progress_callback(str(path_info))
+
+    return modified
+
+
+def _remove_redundant_files(
+    path_info, fs, obj, cache, force, dvcignore: Optional[DvcIgnoreFilter]
+):
+    if dvcignore:
+        existing_files = set(dvcignore.walk_files(fs, path_info))
+    else:
+        existing_files = set(fs.walk_files(path_info))
+
+    needed_files = {path_info.joinpath(*key) for key, _ in obj}
+    redundant_files = existing_files - needed_files
+    for path in redundant_files:
+        _remove(path, fs, cache, force)
+
+    return bool(redundant_files)
+
+
+def _checkout_dir(
+    path_info,
+    fs,
+    obj,
+    cache,
+    force,
+    progress_callback=None,
+    relink=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
+    state=None,
+):
+    modified = False
+    # Create dir separately so that dir is created
+    # even if there are no files in it
+    if not fs.exists(path_info):
+        modified = True
+        fs.makedirs(path_info)
+
+    logger.debug("Linking directory '%s'.", path_info)
+
+    failed = []
+    for entry_key, entry_obj in obj:
+        try:
+            entry_modified = _checkout_file(
+                path_info.joinpath(*entry_key),
+                fs,
+                entry_obj,
+                cache,
+                force,
+                progress_callback,
+                relink,
+                state=None,
+            )
+            if entry_modified:
+                modified = True
+        except CheckoutError as exc:
+            failed.extend(exc.target_infos)
+
+    modified = (
+        _remove_redundant_files(
+            path_info, fs, obj, cache, force, dvcignore=dvcignore
+        )
+        or modified
+    )
+
+    if failed:
+        raise CheckoutError(failed)
+
+    if state:
+        state.save(path_info, fs, obj.hash_info)
+
+    # relink is not modified, assume it as nochange
+    return modified and not relink
+
+
+def _checkout(
+    path_info,
+    fs,
+    obj,
+    cache,
+    force=False,
+    progress_callback=None,
+    relink=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
+    state=None,
+):
+    if not obj.hash_info.isdir:
+        ret = _checkout_file(
+            path_info, fs, obj, cache, force, progress_callback, relink, state
+        )
+    else:
+        ret = _checkout_dir(
+            path_info,
+            fs,
+            obj,
+            cache,
+            force,
+            progress_callback,
+            relink,
+            dvcignore=dvcignore,
+            state=state,
+        )
+
+    if state:
+        state.save_link(path_info, fs)
+
+    return ret
+
+
+def checkout(
+    path_info,
+    fs,
+    obj,
+    cache,
+    force=False,
+    progress_callback=None,
+    relink=False,
+    quiet=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
+    state=None,
+):
+    if path_info.scheme not in ["local", cache.fs.scheme]:
+        raise NotImplementedError
+
+    failed = None
+    skip = False
+    if not obj:
+        if not quiet:
+            logger.warning(
+                "No file hash info found for '%s'. It won't be created.",
+                path_info,
+            )
+        _remove(path_info, fs, cache, force=force)
+        failed = path_info
+
+    elif not relink and not _changed(path_info, fs, obj, cache, state=state):
+        logger.trace("Data '%s' didn't change.", path_info)  # type: ignore
+        skip = True
+
+    if failed or skip:
+        if progress_callback and obj:
+            progress_callback(str(path_info), len(obj))
+        if failed:
+            raise CheckoutError([failed])
+        return
+
+    logger.debug("Checking out '%s' with cache '%s'.", path_info, obj)
+
+    return _checkout(
+        path_info,
+        fs,
+        obj,
+        cache,
+        force,
+        progress_callback,
+        relink,
+        dvcignore=dvcignore,
+        state=state,
+    )

dvc/cli.py

@@ -1,8 +1,10 @@
 """DVC command line interface"""
 import argparse
 import logging
+import os
 import sys
 
+from ._debug import add_debugging_flags
 from .command import (
     add,
     cache,
@@ -26,7 +28,9 @@ from .command import (
     imp_url,
     init,
     install,
+    live,
     ls,
+    machine,
     metrics,
     move,
     params,
@@ -36,6 +40,7 @@ from .command import (
     repro,
     root,
     run,
+    stage,
     unprotect,
     update,
     version,
@@ -79,8 +84,11 @@ COMMANDS = [
     update,
     git_hook,
     plots,
+    stage,
     experiments,
     check_ignore,
+    live,
+    machine,
 ]
 
 
@@ -136,15 +144,7 @@ def get_parent_parser():
     in order to prevent some weird behavior.
     """
     parent_parser = argparse.ArgumentParser(add_help=False)
-
-    parent_parser.add_argument(
-        "--cprofile",
-        action="store_true",
-        default=False,
-        help=argparse.SUPPRESS,
-    )
-    parent_parser.add_argument("--cprofile-dump", help=argparse.SUPPRESS)
-
+    add_debugging_flags(parent_parser)
     log_level_group = parent_parser.add_mutually_exclusive_group()
     log_level_group.add_argument(
         "-q", "--quiet", action="count", default=0, help="Be quiet."
@@ -191,6 +191,14 @@ def get_main_parser():
         help="Show program's version.",
     )
 
+    parser.add_argument(
+        "--cd",
+        default=os.path.curdir,
+        metavar="<path>",
+        help="Change to directory before executing.",
+        type=str,
+    )
+
     # Sub commands
     subparsers = parser.add_subparsers(
         title="Available Commands",

dvc/command/add.py

@@ -3,13 +3,17 @@ import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 
 logger = logging.getLogger(__name__)
 
 
 class CmdAdd(CmdBase):
     def run(self):
+        from dvc.exceptions import (
+            DvcException,
+            RecursiveAddingWhileUsingFilename,
+        )
+
         try:
             if len(self.args.targets) > 1 and self.args.file:
                 raise RecursiveAddingWhileUsingFilename()
@@ -20,6 +24,12 @@ class CmdAdd(CmdBase):
                 no_commit=self.args.no_commit,
                 fname=self.args.file,
                 external=self.args.external,
+                glob=self.args.glob,
+                desc=self.args.desc,
+                out=self.args.out,
+                remote=self.args.remote,
+                to_remote=self.args.to_remote,
+                jobs=self.args.jobs,
             )
 
         except DvcException:
@@ -57,12 +67,56 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Allow targets that are outside of the DVC repository.",
     )
+    parser.add_argument(
+        "--glob",
+        action="store_true",
+        default=False,
+        help="Allows targets containing shell-style wildcards.",
+    )
     parser.add_argument(
         "--file",
-        help="Specify name of the DVC-file this command will generate.",
+        help="Specify name of the .dvc file this command will generate.",
         metavar="<filename>",
     )
     parser.add_argument(
-        "targets", nargs="+", help="Input files/directories to add.",
+        "-o",
+        "--out",
+        help="Destination path to put files to.",
+        metavar="<path>",
+    )
+    parser.add_argument(
+        "--to-remote",
+        action="store_true",
+        default=False,
+        help="Download it directly to the remote",
+    )
+    parser.add_argument(
+        "-r",
+        "--remote",
+        help="Remote storage to download to",
+        metavar="<name>",
+    )
+    parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
+    )
+    parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the data (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
+    parser.add_argument(
+        "targets", nargs="+", help="Input files/directories to add."
     ).complete = completion.FILE
     parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -1,4 +1,5 @@
 import logging
+import os
 from abc import ABC, abstractmethod
 
 logger = logging.getLogger(__name__)
@@ -6,11 +7,11 @@ logger = logging.getLogger(__name__)
 
 def fix_subparsers(subparsers):
     """Workaround for bug in Python 3. See more info at:
-        https://bugs.python.org/issue16308
-        https://github.com/iterative/dvc/issues/769
+    https://bugs.python.org/issue16308
+    https://github.com/iterative/dvc/issues/769
 
-        Args:
-            subparsers: subparsers to fix.
+    Args:
+        subparsers: subparsers to fix.
     """
     subparsers.required = True
     subparsers.dest = "cmd"
@@ -28,24 +29,20 @@ def append_doc_link(help_message, path):
 
 
 class CmdBase(ABC):
+    UNINITIALIZED = False
+
     def __init__(self, args):
         from dvc.repo import Repo
-        from dvc.updater import Updater
 
-        self.repo = Repo()
+        os.chdir(args.cd)
+
+        self.repo = Repo(uninitialized=self.UNINITIALIZED)
         self.config = self.repo.config
         self.args = args
-        hardlink_lock = self.config["core"].get("hardlink_lock", False)
-        updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
-        updater.check()
 
-    @property
-    def default_targets(self):
-        """Default targets for `dvc repro`."""
-        from dvc.dvcfile import PIPELINE_FILE
-
-        logger.debug(f"assuming default target '{PIPELINE_FILE}'.")
-        return [PIPELINE_FILE]
+    def do_run(self):
+        with self.repo:
+            return self.run()
 
     @abstractmethod
     def run(self):
@@ -55,3 +52,8 @@ class CmdBase(ABC):
 class CmdBaseNoRepo(CmdBase):
     def __init__(self, args):  # pylint: disable=super-init-not-called
         self.args = args
+
+        os.chdir(args.cd)
+
+    def do_run(self):
+        return self.run()

dvc/command/cache.py

@@ -3,12 +3,27 @@ import argparse
 from dvc.command import completion
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
+from dvc.ui import ui
 
 
 class CmdCacheDir(CmdConfig):
     def run(self):
-        with self.config.edit(level=self.args.level) as edit:
-            edit["cache"]["dir"] = self.args.value
+        if self.args.value is None and not self.args.unset:
+            if self.args.level:
+                conf = self.config.read(level=self.args.level)
+            else:
+                # Use merged config with default values
+                conf = self.config
+            self._check(conf, False, "cache", "dir")
+            ui.write(conf["cache"]["dir"])
+            return 0
+        with self.config.edit(level=self.args.level) as conf:
+            if self.args.unset:
+                self._check(conf, False, "cache", "dir")
+                del conf["cache"]["dir"]
+            else:
+                self._check(conf, False, "cache")
+                conf["cache"]["dir"] = self.args.value
         return 0
 
 
@@ -55,6 +70,8 @@ def add_parser(subparsers, parent_parser):
         "value",
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "
-        "config file location.",
+        "config file location. If no path is provided, it returns the "
+        "current cache directory.",
+        nargs="?",
     ).complete = completion.DIR
     cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/check_ignore.py

@@ -1,17 +1,14 @@
 import argparse
-import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 class CmdCheckIgnore(CmdBase):
     def __init__(self, args):
         super().__init__(args)
-        self.ignore_filter = self.repo.tree.dvcignore
+        self.ignore_filter = self.repo.dvcignore
 
     def _show_results(self, result):
         if not result.match and not self.args.non_matching:
@@ -23,9 +20,9 @@ class CmdCheckIgnore(CmdBase):
                 patterns = patterns[-1:]
 
             for pattern in patterns:
-                logger.info("{}\t{}".format(pattern, result.file))
+                ui.write(pattern, result.file, sep="\t")
         else:
-            logger.info(result.file)
+            ui.write(result.file)
 
     def _check_one_file(self, target):
         result = self.ignore_filter.check_ignore(target)
@@ -55,6 +52,8 @@ class CmdCheckIgnore(CmdBase):
         return ret
 
     def _check_args(self):
+        from dvc.exceptions import DvcException
+
         if not self.args.stdin and not self.args.targets:
             raise DvcException("`targets` or `--stdin` needed")
 
@@ -121,6 +120,6 @@ def add_parser(subparsers, parent_parser):
         help="Read paths from standard input instead of providing `targets`.",
     )
     parser.add_argument(
-        "targets", nargs="*", help="File or directory paths to check",
+        "targets", nargs="*", help="File or directory paths to check"
     ).complete = completion.FILE
     parser.set_defaults(func=CmdCheckIgnore)

dvc/command/checkout.py

@@ -1,44 +1,35 @@
 import argparse
-import logging
 import operator
 
-import colorama
-
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import CheckoutError
-from dvc.utils.humanize import get_summary
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 def log_changes(stats):
-    colors = [
-        ("modified", colorama.Fore.YELLOW,),
-        ("added", colorama.Fore.GREEN),
-        ("deleted", colorama.Fore.RED,),
-    ]
+    colors = {
+        "modified": "yellow",
+        "added": "green",
+        "deleted": "red",
+    }
 
-    for state, color in colors:
+    for state, color in colors.items():
         entries = stats.get(state)
 
         if not entries:
             continue
 
         for entry in entries:
-            logger.info(
-                "{color}{state}{nc}{spacing}{entry}".format(
-                    color=color,
-                    state=state[0].upper(),
-                    nc=colorama.Fore.RESET,
-                    spacing="\t",
-                    entry=entry,
-                )
+            ui.write(
+                f"[{color}]{state[0].upper()}", entry, styled=True, sep="\t"
             )
 
 
 class CmdCheckout(CmdBase):
     def run(self):
+        from dvc.utils.humanize import get_summary
+
         stats, exc = None, None
         try:
             stats = self.repo.checkout(
@@ -57,7 +48,7 @@ class CmdCheckout(CmdBase):
             msg = get_summary(
                 sorted(stats.items(), key=operator.itemgetter(0))
             )
-            logger.info(msg or default_message)
+            ui.write(msg or default_message)
         else:
             log_changes(stats)
 

dvc/command/commit.py

@@ -3,13 +3,14 @@ import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
 
 class CmdCommit(CmdBase):
     def run(self):
+        from dvc.exceptions import DvcException
+
         if not self.args.targets:
             self.args.targets = [None]
 
@@ -32,7 +33,10 @@ class CmdCommit(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    COMMIT_HELP = "Save changed data to cache and update DVC-files."
+    COMMIT_HELP = (
+        "Record changes to files or directories tracked by DVC"
+        " by storing the current versions in the cache."
+    )
 
     commit_parser = subparsers.add_parser(
         "commit",
@@ -65,7 +69,7 @@ def add_parser(subparsers, parent_parser):
     commit_parser.add_argument(
         "targets",
         nargs="*",
-        help="DVC-files to commit. Optional. "
-        "(Finds all DVC-files in the workspace by default.)",
-    ).complete = completion.DVC_FILE
+        help="stages or .dvc files to commit. Optional. "
+        "(Finds all DVC files in the workspace by default.)",
+    ).complete = completion.DVCFILES_AND_STAGE
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -4,14 +4,12 @@ import logging
 import shtab
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 FILE = shtab.FILE
 DIR = shtab.DIRECTORY
-DVC_FILE = {
-    "bash": "_dvc_compgen_DVCFiles",
-    "zsh": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'",
-}
+
 PREAMBLE = {
     "bash": """
 # $1=COMP_WORDS[1]
@@ -21,9 +19,65 @@ _dvc_compgen_DVCFiles() {
   compgen -f -X '!*Dvcfile' -- $1
   compgen -f -X '!*dvc.yaml' -- $1
 }
-""",
-    "zsh": "",
+
+_dvc_compgen_stages() {
+    local _dvc_stages=($(dvc stage list -q --names-only))
+    compgen -W "${_dvc_stages[*]}" -- $1
 }
+_dvc_compgen_stages_and_files() {
+    _dvc_compgen_DVCFiles $1
+    _dvc_compgen_stages $1
+}
+
+_dvc_compgen_exps() {
+    local _dvc_exps=($(dvc exp list -q --all --names-only))
+    compgen -W "${_dvc_exps[*]}" -- $1
+}
+    """,
+    "zsh": """
+_dvc_compadd_DVCFiles() {
+    _files -g '(*?.dvc|Dvcfile|dvc.yaml)'
+}
+_dvc_compadd_stages() {
+    # this will also show up the description of the stages
+    _describe 'stages' "($(_dvc_stages_output))"
+}
+
+_dvc_stages_output() {
+  dvc stage list -q | awk '{
+    # escape possible `:` on the stage name
+    sub(/:/, "\\\\\\\\:", $1);
+    # read all of the columns except the first
+    # reading `out` from $2, so as not to have a leading whitespace
+    out=$2; for(i=3;i<=NF;i++){out=out" "$i};
+    # print key, ":" and then single-quote the description
+    # colon is a delimiter used by `_describe` to separate field/description
+    print $1":""\\047"out"\\047"
+    # single quote -> \\047
+    }'
+}
+
+_dvc_compadd_stages_and_files() {
+    _dvc_compadd_DVCFiles
+    _dvc_compadd_stages
+}
+
+_dvc_compadd_exps() {
+    _describe 'experiments' "($(dvc exp list -q -a --names-only))"
+}
+    """,
+}
+
+DVC_FILE = {"bash": "_dvc_compgen_DVCFiles", "zsh": "_dvc_compadd_DVCFiles"}
+
+STAGE = {"bash": "_dvc_compgen_stages", "zsh": "_dvc_compadd_stages"}
+
+DVCFILES_AND_STAGE = {
+    "bash": "_dvc_compgen_stages_and_files",
+    "zsh": "_dvc_compadd_stages_and_files",
+}
+
+EXPERIMENT = {"bash": "_dvc_compgen_exps", "zsh": "_dvc_compadd_exps"}
 
 
 class CmdCompletion(CmdBaseNoRepo):
@@ -33,7 +87,7 @@ class CmdCompletion(CmdBaseNoRepo):
         parser = get_main_parser()
         shell = self.args.shell
         script = shtab.complete(parser, shell=shell, preamble=PREAMBLE)
-        print(script)
+        ui.write(script, force=True)
         return 0
 
 

dvc/command/config.py

@@ -1,33 +1,118 @@
 import argparse
 import logging
+import os
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
-from dvc.config import Config, ConfigError
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
+NAME_REGEX = r"^(?P<remote>remote\.)?(?P<section>[^\.]*)\.(?P<option>[^\.]*)$"
+
+
+def _name_type(value):
+    import re
+
+    match = re.match(NAME_REGEX, value)
+    if not match:
+        raise argparse.ArgumentTypeError(
+            "name argument should look like "
+            "remote.name.option or "
+            "section.option"
+        )
+    return (
+        bool(match.group("remote")),
+        match.group("section").lower(),
+        match.group("option").lower(),
+    )
+
 
 class CmdConfig(CmdBaseNoRepo):
     def __init__(self, args):
+        from dvc.config import Config
+
         super().__init__(args)
 
         self.config = Config(validate=False)
 
     def run(self):
-        section, opt = self.args.name.lower().strip().split(".", 1)
+        if self.args.show_origin:
+            if any((self.args.value, self.args.unset)):
+                logger.error(
+                    "--show-origin can't be used together with any of these "
+                    "options: -u/--unset, value"
+                )
+                return 1
+
+        if self.args.list:
+            return self._list()
+
+        if self.args.name is None:
+            logger.error("name argument is required")
+            return 1
+
+        remote, section, opt = self.args.name
 
         if self.args.value is None and not self.args.unset:
-            conf = self.config.load_one(self.args.level)
-            self._check(conf, section, opt)
-            logger.info(conf[section][opt])
-            return 0
+            return self._get(remote, section, opt)
 
+        return self._set(remote, section, opt)
+
+    def _list(self):
+        if any((self.args.name, self.args.value, self.args.unset)):
+            logger.error(
+                "-l/--list can't be used together with any of these "
+                "options: -u/--unset, name, value"
+            )
+            return 1
+
+        levels = [self.args.level] if self.args.level else self.config.LEVELS
+        for level in levels:
+            conf = self.config.read(level)
+            prefix = self._config_file_prefix(
+                self.args.show_origin, self.config, level
+            )
+            configs = list(self._format_config(conf, prefix))
+            if configs:
+                ui.write("\n".join(configs))
+
+        return 0
+
+    def _get(self, remote, section, opt):
+        from dvc.config import ConfigError
+
+        levels = (
+            [self.args.level] if self.args.level else self.config.LEVELS[::-1]
+        )
+
+        for level in levels:
+            conf = self.config.read(level)
+            if remote:
+                conf = conf["remote"]
+
+            try:
+                self._check(conf, remote, section, opt)
+            except ConfigError:
+                if self.args.level:
+                    raise
+            else:
+                prefix = self._config_file_prefix(
+                    self.args.show_origin, self.config, level
+                )
+                ui.write(prefix, conf[section][opt], sep="")
+                break
+
+        return 0
+
+    def _set(self, remote, section, opt):
         with self.config.edit(self.args.level) as conf:
+            if remote:
+                conf = conf["remote"]
             if self.args.unset:
-                self._check(conf, section, opt)
+                self._check(conf, remote, section, opt)
                 del conf[section][opt]
             else:
-                self._check(conf, section)
+                self._check(conf, remote, section)
                 conf[section][opt] = self.args.value
 
         if self.args.name == "cache.type":
@@ -39,14 +124,39 @@ class CmdConfig(CmdBaseNoRepo):
 
         return 0
 
-    def _check(self, conf, section, opt=None):
+    def _check(self, conf, remote, section, opt=None):
+        from dvc.config import ConfigError
+
+        name = "remote" if remote else "section"
         if section not in conf:
-            msg = "section {} doesn't exist"
-            raise ConfigError(msg.format(self.args.name))
+            raise ConfigError(f"{name} '{section}' doesn't exist")
 
         if opt and opt not in conf[section]:
-            msg = "option {} doesn't exist"
-            raise ConfigError(msg.format(self.args.name))
+            raise ConfigError(
+                f"option '{opt}' doesn't exist in {name} '{section}'"
+            )
+
+    @staticmethod
+    def _format_config(config, prefix=""):
+        from dvc.utils.flatten import flatten
+
+        for key, value in flatten(config).items():
+            yield f"{prefix}{key}={value}"
+
+    @staticmethod
+    def _config_file_prefix(show_origin, config, level):
+        from dvc.repo import Repo
+
+        if not show_origin:
+            return ""
+
+        level = level or "repo"
+        fname = config.files[level]
+
+        if level in ["local", "repo"]:
+            fname = os.path.relpath(fname, start=Repo.find_root())
+
+        return fname + "\t"
 
 
 parent_config_parser = argparse.ArgumentParser(add_help=False)
@@ -65,14 +175,21 @@ level_group.add_argument(
     const="system",
     help="Use system config.",
 )
+level_group.add_argument(
+    "--project",
+    dest="level",
+    action="store_const",
+    const="repo",
+    help="Use project config (.dvc/config).",
+)
 level_group.add_argument(
     "--local",
     dest="level",
     action="store_const",
     const="local",
-    help="Use local config.",
+    help="Use local config (.dvc/config.local).",
 )
-parent_config_parser.set_defaults(level="repo")
+parent_config_parser.set_defaults(level=None)
 
 
 def add_parser(subparsers, parent_parser):
@@ -92,6 +209,24 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         help="Unset option.",
     )
-    config_parser.add_argument("name", help="Option name.")
+    config_parser.add_argument(
+        "name",
+        nargs="?",
+        type=_name_type,
+        help="Option name (section.option or remote.name.option).",
+    )
     config_parser.add_argument("value", nargs="?", help="Option value.")
+    config_parser.add_argument(
+        "-l",
+        "--list",
+        default=False,
+        action="store_true",
+        help="List all defined config values.",
+    )
+    config_parser.add_argument(
+        "--show-origin",
+        default=False,
+        action="store_true",
+        help="Show the source file containing each config value.",
+    )
     config_parser.set_defaults(func=CmdConfig)

dvc/command/daemon.py

@@ -67,6 +67,6 @@ def add_parser(subparsers, parent_parser):
         help=DAEMON_ANALYTICS_HELP,
     )
     daemon_analytics_parser.add_argument(
-        "target", help="Analytics file.",
+        "target", help="Analytics file."
     ).complete = completion.FILE
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/dag.py

@@ -1,10 +1,7 @@
 import argparse
-import logging
 
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException
-
-logger = logging.getLogger(__name__)
+from dvc.ui import ui
 
 
 def _show_ascii(G):
@@ -30,58 +27,87 @@ def _show_dot(G):
     return dot_file.getvalue()
 
 
-def _build(G, target=None, full=False):
+def _collect_targets(repo, target, outs):
+    if not target:
+        return []
+
+    pairs = repo.stage.collect_granular(target)
+    if not outs:
+        return [stage.addressing for stage, _ in pairs]
+
+    targets = []
+
+    outs_trie = repo.index.outs_trie
+    for stage, info in pairs:
+        if not info:
+            targets.extend([str(out) for out in stage.outs])
+            continue
+
+        for out in outs_trie.itervalues(prefix=info.parts):  # noqa: B301
+            targets.extend(str(out))
+
+    return targets
+
+
+def _transform(index, outs):
     import networkx as nx
 
-    from dvc.repo.graph import get_pipeline, get_pipelines
+    from dvc.stage import Stage
 
-    if target:
-        H = get_pipeline(get_pipelines(G), target)
-        if not full:
-            descendants = nx.descendants(G, target)
+    def _relabel(node) -> str:
+        return node.addressing if isinstance(node, Stage) else str(node)
+
+    G = index.outs_graph if outs else index.graph
+    return nx.relabel_nodes(G, _relabel, copy=True)
+
+
+def _filter(G, targets, full):
+    import networkx as nx
+
+    if not targets:
+        return G
+
+    H = G.copy()
+    if not full:
+        descendants = set()
+        for target in targets:
+            descendants.update(nx.descendants(G, target))
             descendants.add(target)
-            H.remove_nodes_from(set(G.nodes()) - descendants)
-    else:
-        H = G
+        H.remove_nodes_from(set(G.nodes()) - descendants)
 
-    def _relabel(stage):
-        return stage.addressing
+    undirected = H.to_undirected()
+    connected = set()
+    for target in targets:
+        connected.update(nx.node_connected_component(undirected, target))
 
-    return nx.relabel_nodes(H, _relabel, copy=False)
+    H.remove_nodes_from(set(H.nodes()) - connected)
+
+    return H
+
+
+def _build(repo, target=None, full=False, outs=False):
+    targets = _collect_targets(repo, target, outs)
+    G = _transform(repo.index, outs)
+    return _filter(G, targets, full)
 
 
 class CmdDAG(CmdBase):
     def run(self):
-        try:
-            target = None
-            if self.args.target:
-                stages = self.repo.collect(self.args.target)
-                if len(stages) > 1:
-                    logger.error(
-                        f"'{self.args.target}' contains more than one stage "
-                        "{stages}, please specify one stage"
-                    )
-                    return 1
-                target = stages[0]
+        G = _build(
+            self.repo,
+            target=self.args.target,
+            full=self.args.full,
+            outs=self.args.outs,
+        )
 
-            G = _build(self.repo.graph, target=target, full=self.args.full,)
+        if self.args.dot:
+            ui.write(_show_dot(G))
+        else:
+            from dvc.utils.pager import pager
 
-            if self.args.dot:
-                logger.info(_show_dot(G))
-            else:
-                from dvc.utils.pager import pager
+            pager(_show_ascii(G))
 
-                pager(_show_ascii(G))
-
-            return 0
-        except DvcException:
-            msg = "failed to show "
-            if self.args.target:
-                msg += f"a pipeline for '{target}'"
-            else:
-                msg += "pipelines"
-            logger.exception(msg)
-            return 1
+        return 0
 
 
 def add_parser(subparsers, parent_parser):
@@ -108,6 +134,13 @@ def add_parser(subparsers, parent_parser):
             "showing DAG consisting only of ancestors."
         ),
     )
+    dag_parser.add_argument(
+        "-o",
+        "--outs",
+        action="store_true",
+        default=False,
+        help="Print output files instead of stages.",
+    )
     dag_parser.add_argument(
         "target",
         nargs="?",

dvc/command/data_sync.py

@@ -3,25 +3,29 @@ import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.checkout import log_changes
-from dvc.exceptions import CheckoutError, DvcException
-from dvc.utils.humanize import get_summary
 
 logger = logging.getLogger(__name__)
 
 
 class CmdDataBase(CmdBase):
     def log_summary(self, stats):
+        from dvc.ui import ui
+        from dvc.utils.humanize import get_summary
+
         default_msg = "Everything is up to date."
-        logger.info(get_summary(stats.items()) or default_msg)
+        ui.write(get_summary(stats.items()) or default_msg)
 
 
 class CmdDataPull(CmdDataBase):
     def log_summary(self, stats):
+        from dvc.command.checkout import log_changes
+
         log_changes(stats)
         super().log_summary(stats)
 
     def run(self):
+        from dvc.exceptions import CheckoutError, DvcException
+
         try:
             stats = self.repo.pull(
                 targets=self.args.targets,
@@ -34,6 +38,7 @@ class CmdDataPull(CmdDataBase):
                 force=self.args.force,
                 recursive=self.args.recursive,
                 run_cache=self.args.run_cache,
+                glob=self.args.glob,
             )
             self.log_summary(stats)
         except (CheckoutError, DvcException) as exc:
@@ -46,6 +51,8 @@ class CmdDataPull(CmdDataBase):
 
 class CmdDataPush(CmdDataBase):
     def run(self):
+        from dvc.exceptions import DvcException
+
         try:
             processed_files_count = self.repo.push(
                 targets=self.args.targets,
@@ -57,6 +64,7 @@ class CmdDataPush(CmdDataBase):
                 with_deps=self.args.with_deps,
                 recursive=self.args.recursive,
                 run_cache=self.args.run_cache,
+                glob=self.args.glob,
             )
             self.log_summary({"pushed": processed_files_count})
         except DvcException:
@@ -67,6 +75,8 @@ class CmdDataPush(CmdDataBase):
 
 class CmdDataFetch(CmdDataBase):
     def run(self):
+        from dvc.exceptions import DvcException
+
         try:
             processed_files_count = self.repo.fetch(
                 targets=self.args.targets,
@@ -130,7 +140,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     pull_parser.add_argument(
-        "-r", "--remote", help="Remote storage to pull from", metavar="<name>",
+        "-r", "--remote", help="Remote storage to pull from", metavar="<name>"
     )
     pull_parser.add_argument(
         "-a",
@@ -147,6 +157,7 @@ def add_parser(subparsers, _parent_parser):
         help="Fetch cache for all tags.",
     )
     pull_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -179,6 +190,12 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Fetch run history for all stages.",
     )
+    pull_parser.add_argument(
+        "--glob",
+        action="store_true",
+        default=False,
+        help="Pull cache for targets matching shell-style wildcards.",
+    )
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
@@ -192,7 +209,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     push_parser.add_argument(
-        "-r", "--remote", help="Remote storage to push to", metavar="<name>",
+        "-r", "--remote", help="Remote storage to push to", metavar="<name>"
     )
     push_parser.add_argument(
         "-a",
@@ -209,6 +226,7 @@ def add_parser(subparsers, _parent_parser):
         help="Push cache for all tags.",
     )
     push_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -234,12 +252,17 @@ def add_parser(subparsers, _parent_parser):
         default=False,
         help="Push run history for all stages.",
     )
+    push_parser.add_argument(
+        "--glob",
+        action="store_true",
+        default=False,
+        help="Allows targets containing shell-style wildcards.",
+    )
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
     FETCH_HELP = (
-        "Get tracked files or directories from remote storage"
-        " into the cache."
+        "Download files or directories from remote storage to the cache."
     )
 
     fetch_parser = subparsers.add_parser(
@@ -250,10 +273,7 @@ def add_parser(subparsers, _parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     fetch_parser.add_argument(
-        "-r",
-        "--remote",
-        help="Remote storage to fetch from",
-        metavar="<name>",
+        "-r", "--remote", help="Remote storage to fetch from", metavar="<name>"
     )
     fetch_parser.add_argument(
         "-a",
@@ -270,6 +290,7 @@ def add_parser(subparsers, _parent_parser):
         help="Fetch cache for all tags.",
     )
     fetch_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -350,6 +371,7 @@ def add_parser(subparsers, _parent_parser):
         "for all tags.",
     )
     status_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,

dvc/command/destroy.py

@@ -1,15 +1,16 @@
 import argparse
 import logging
 
-import dvc.prompt as prompt
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
 
 class CmdDestroy(CmdBase):
     def run(self):
+        from dvc.exceptions import DvcException
+        from dvc.ui import ui
+
         try:
             statement = (
                 "This will destroy all information about your pipelines,"
@@ -18,7 +19,7 @@ class CmdDestroy(CmdBase):
                 "Are you sure you want to continue?"
             )
 
-            if not self.args.force and not prompt.confirm(statement):
+            if not self.args.force and not ui.confirm(statement):
                 raise DvcException(
                     "cannot destroy without a confirmation from the user."
                     " Use `-f` to force."
@@ -32,7 +33,7 @@ class CmdDestroy(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    DESTROY_HELP = "Remove DVC-files, local DVC config and data cache."
+    DESTROY_HELP = "Remove DVC files, local DVC config and data cache."
 
     destroy_parser = subparsers.add_parser(
         "destroy",

dvc/command/diff.py

@@ -3,10 +3,9 @@ import json
 import logging
 import os
 
-import colorama
-
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -18,31 +17,32 @@ def _digest(checksum):
 
 
 def _show_md(diff, show_hash=False, hide_missing=False):
-    from dvc.utils.diff import table
-
-    header = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
+    headers = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
     rows = []
-    statuses = ["added", "deleted", "modified"]
+    statuses = ["added", "deleted", "renamed", "modified"]
     if not hide_missing:
         statuses.append("not in cache")
+
     for status in statuses:
         entries = diff.get(status, [])
         if not entries:
             continue
         for entry in entries:
             path = entry["path"]
+            if isinstance(path, dict):
+                path = f"{path['old']} -> {path['new']}"
             if show_hash:
                 check_sum = _digest(entry.get("hash", ""))
                 rows.append([status, check_sum, path])
             else:
                 rows.append([status, path])
 
-    return table(header, rows, True)
+    ui.table(rows, headers=headers, markdown=True)
 
 
 class CmdDiff(CmdBase):
     @staticmethod
-    def _format(diff, hide_missing=False):
+    def _show_diff(diff, hide_missing=False):
         """
         Given a diff structure, generate a string of paths separated
         by new lines and grouped together by their state.
@@ -69,16 +69,16 @@ class CmdDiff(CmdBase):
         """
 
         colors = {
-            "added": colorama.Fore.GREEN,
-            "modified": colorama.Fore.YELLOW,
-            "deleted": colorama.Fore.RED,
-            "not in cache": colorama.Fore.YELLOW,
+            "added": "green",
+            "modified": "yellow",
+            "deleted": "red",
+            "renamed": "green",
+            "not in cache": "yellow",
         }
 
         summary = {}
-        groups = []
 
-        states = ["added", "deleted", "modified"]
+        states = ["added", "deleted", "renamed", "modified"]
         if not hide_missing:
             states.append("not in cache")
         for state in states:
@@ -88,66 +88,66 @@ class CmdDiff(CmdBase):
             if not entries:
                 continue
 
-            content = []
+            header = state.capitalize()
+            ui.write(f"[{colors[state]}]{header}[/]:", styled=True)
 
             for entry in entries:
                 path = entry["path"]
+                if isinstance(path, dict):
+                    path = f"{path['old']} -> {path['new']}"
                 checksum = entry.get("hash")
                 summary[state] += 1 if not path.endswith(os.sep) else 0
-                content.append(
+                ui.write(
                     "{space}{checksum}{separator}{path}".format(
                         space="    ",
                         checksum=_digest(checksum) if checksum else "",
                         separator="  " if checksum else "",
-                        path=entry["path"],
+                        path=path,
                     )
                 )
 
-            groups.append(
-                "{color}{header}{nc}:\n{content}".format(
-                    color=colors[state],
-                    header=state.capitalize(),
-                    nc=colorama.Fore.RESET,
-                    content="\n".join(content),
-                )
-            )
+            ui.write()
 
         if not sum(summary.values()):
             return None
 
-        fmt = (
-            "files summary: {added} added, {deleted} deleted,"
-            " {modified} modified"
+        states_summary = ", ".join(
+            f"{summary[state]} {state}"
+            for state in states
+            if summary[state] > 0
         )
-        if not hide_missing:
-            fmt += ", {not in cache} not in cache"
-        groups.append(fmt.format_map(summary))
-
-        return "\n\n".join(groups)
+        ui.write("files summary:", states_summary)
 
     def run(self):
+        from dvc.exceptions import DvcException
+
         try:
-            diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
+            diff = self.repo.diff(
+                self.args.a_rev, self.args.b_rev, self.args.targets
+            )
             show_hash = self.args.show_hash
             hide_missing = self.args.b_rev or self.args.hide_missing
             if hide_missing:
                 del diff["not in cache"]
 
             for key, entries in diff.items():
-                entries = sorted(entries, key=lambda entry: entry["path"])
+                entries = sorted(
+                    entries,
+                    key=lambda entry: entry["path"]["old"]
+                    if isinstance(entry["path"], dict)
+                    else entry["path"],
+                )
                 if not show_hash:
                     for entry in entries:
                         del entry["hash"]
                 diff[key] = entries
 
             if self.args.show_json:
-                logger.info(json.dumps(diff))
+                ui.write(json.dumps(diff))
             elif self.args.show_md:
-                logger.info(_show_md(diff, show_hash, hide_missing))
+                _show_md(diff, show_hash, hide_missing)
             elif diff:
-                output = self._format(diff, hide_missing)
-                if output:
-                    logger.info(output)
+                self._show_diff(diff, hide_missing)
 
         except DvcException:
             logger.exception("failed to get diff")
@@ -167,6 +167,15 @@ def add_parser(subparsers, parent_parser):
         help=DIFF_DESCRIPTION,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    diff_parser.add_argument(
+        "--targets",
+        nargs="*",
+        help=(
+            "Specific DVC-tracked files to compare. "
+            "Accepts one or more file paths."
+        ),
+        metavar="<paths>",
+    ).complete = completion.FILE
     diff_parser.add_argument(
         "a_rev",
         help="Old Git commit to compare (defaults to HEAD)",

dvc/command/freeze.py

@@ -40,7 +40,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     freeze_parser.add_argument(
-        "targets", nargs="+", help="Stages or .dvc files to freeze",
+        "targets", nargs="+", help="Stages or .dvc files to freeze"
     ).complete = completion.DVC_FILE
     freeze_parser.set_defaults(func=CmdFreeze)
 
@@ -53,6 +53,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unfreeze_parser.add_argument(
-        "targets", nargs="+", help="Stages or .dvc files to unfreeze",
+        "targets", nargs="+", help="Stages or .dvc files to unfreeze"
     ).complete = completion.DVC_FILE
     unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/gc.py

@@ -2,8 +2,8 @@ import argparse
 import logging
 import os
 
-import dvc.prompt as prompt
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -30,6 +30,8 @@ class CmdGC(CmdBase):
             msg += " and all git branches"
         elif self.args.all_tags:
             msg += " and all git tags"
+        elif self.args.all_experiments:
+            msg += " and all experiments"
 
         if self.args.repos:
             msg += " of the current and the following repos:"
@@ -42,13 +44,14 @@ class CmdGC(CmdBase):
         logger.warning(msg)
 
         msg = "Are you sure you want to proceed?"
-        if not self.args.force and not prompt.confirm(msg):
+        if not self.args.force and not ui.confirm(msg):
             return 1
 
         self.repo.gc(
             all_branches=self.args.all_branches,
             all_tags=self.args.all_tags,
             all_commits=self.args.all_commits,
+            all_experiments=self.args.all_experiments,
             cloud=self.args.cloud,
             remote=self.args.remote,
             force=self.args.force,
@@ -94,11 +97,18 @@ def add_parser(subparsers, parent_parser):
         help="Keep data files for all Git tags.",
     )
     gc_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
         help="Keep data files for all Git commits.",
     )
+    gc_parser.add_argument(
+        "--all-experiments",
+        action="store_true",
+        default=False,
+        help="Keep data files for all experiments.",
+    )
     gc_parser.add_argument(
         "-c",
         "--cloud",

dvc/command/get.py

@@ -12,12 +12,13 @@ logger = logging.getLogger(__name__)
 class CmdGet(CmdBaseNoRepo):
     def _show_url(self):
         from dvc.api import get_url
+        from dvc.ui import ui
 
         try:
             url = get_url(
                 self.args.path, repo=self.args.url, rev=self.args.rev
             )
-            logger.info(url)
+            ui.write(url, force=True)
         except DvcException:
             logger.exception("failed to show URL")
             return 1
@@ -39,6 +40,7 @@ class CmdGet(CmdBaseNoRepo):
                 path=self.args.path,
                 out=self.args.out,
                 rev=self.args.rev,
+                jobs=self.args.jobs,
             )
             return 0
         except DvcException:
@@ -63,7 +65,7 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path", help="Path to a file or directory within the repository",
+        "path", help="Path to a file or directory within the repository"
     ).complete = completion.FILE
     get_parser.add_argument(
         "-o",
@@ -84,4 +86,15 @@ def add_parser(subparsers, parent_parser):
         help="Print the storage location (URL) the target data would be "
         "downloaded from, and exit.",
     )
+    get_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
+    )
     get_parser.set_defaults(func=CmdGet)

dvc/command/get_url.py

@@ -14,7 +14,7 @@ class CmdGetUrl(CmdBaseNoRepo):
         from dvc.repo import Repo
 
         try:
-            Repo.get_url(self.args.url, out=self.args.out)
+            Repo.get_url(self.args.url, out=self.args.out, jobs=self.args.jobs)
             return 0
         except DvcException:
             logger.exception(f"failed to get '{self.args.url}'")
@@ -34,6 +34,17 @@ def add_parser(subparsers, parent_parser):
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
     get_parser.add_argument(
-        "out", nargs="?", help="Destination path to put data to.",
+        "out", nargs="?", help="Destination path to put data to."
     ).complete = completion.DIR
+    get_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
+    )
     get_parser.set_defaults(func=CmdGetUrl)

dvc/command/git_hook.py

@@ -69,14 +69,13 @@ class CmdMergeDriver(CmdHookBase):
         dvc = Repo()
 
         try:
-            with dvc.state:
-                ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
-                our = Dvcfile(dvc, self.args.our, verify=False)
-                their = Dvcfile(dvc, self.args.their, verify=False)
+            ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
+            our = Dvcfile(dvc, self.args.our, verify=False)
+            their = Dvcfile(dvc, self.args.their, verify=False)
 
-                our.merge(ancestor, their)
+            our.merge(ancestor, their)
 
-                return 0
+            return 0
         finally:
             dvc.close()
 
@@ -106,7 +105,7 @@ def add_parser(subparsers, parent_parser):
         help=PRE_COMMIT_HELP,
     )
     pre_commit_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
     )
     pre_commit_parser.set_defaults(func=CmdPreCommit)
 
@@ -118,7 +117,7 @@ def add_parser(subparsers, parent_parser):
         help=POST_CHECKOUT_HELP,
     )
     post_checkout_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
     )
     post_checkout_parser.set_defaults(func=CmdPostCheckout)
 
@@ -130,7 +129,7 @@ def add_parser(subparsers, parent_parser):
         help=PRE_PUSH_HELP,
     )
     pre_push_parser.add_argument(
-        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
+        "args", nargs="*", help="Arguments passed by GIT or pre-commit tool."
     )
     pre_push_parser.set_defaults(func=CmdPrePush)
 
@@ -147,9 +146,7 @@ def add_parser(subparsers, parent_parser):
         help="Ancestor's version of the conflicting file.",
     )
     merge_driver_parser.add_argument(
-        "--our",
-        required=True,
-        help="Current version of the conflicting file.",
+        "--our", required=True, help="Current version of the conflicting file."
     )
     merge_driver_parser.add_argument(
         "--their",

dvc/command/imp.py

@@ -15,7 +15,11 @@ class CmdImport(CmdBase):
                 self.args.url,
                 self.args.path,
                 out=self.args.out,
+                fname=self.args.file,
                 rev=self.args.rev,
+                no_exec=self.args.no_exec,
+                desc=self.args.desc,
+                jobs=self.args.jobs,
             )
         except DvcException:
             logger.exception(
@@ -44,7 +48,7 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path", help="Path to a file or directory within the repository",
+        "path", help="Path to a file or directory within the repository"
     ).complete = completion.FILE
     import_parser.add_argument(
         "-o",
@@ -59,4 +63,35 @@ def add_parser(subparsers, parent_parser):
         help="Git revision (e.g. SHA, branch, tag)",
         metavar="<commit>",
     )
+    import_parser.add_argument(
+        "--file",
+        help="Specify name of the .dvc file this command will generate.",
+        metavar="<filename>",
+    )
+    import_parser.add_argument(
+        "--no-exec",
+        action="store_true",
+        default=False,
+        help="Only create .dvc file without actually downloading it.",
+    )
+    import_parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the data (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
+    import_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
+    )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -16,6 +16,10 @@ class CmdImportUrl(CmdBase):
                 out=self.args.out,
                 fname=self.args.file,
                 no_exec=self.args.no_exec,
+                remote=self.args.remote,
+                to_remote=self.args.to_remote,
+                desc=self.args.desc,
+                jobs=self.args.jobs,
             )
         except DvcException:
             logger.exception(
@@ -54,17 +58,49 @@ def add_parser(subparsers, parent_parser):
         "remote://remote_name/path/to/file/or/dir (see `dvc remote`)",
     )
     import_parser.add_argument(
-        "out", nargs="?", help="Destination path to put files to.",
+        "out", nargs="?", help="Destination path to put files to."
     ).complete = completion.DIR
     import_parser.add_argument(
         "--file",
-        help="Specify name of the DVC-file this command will generate.",
+        help="Specify name of the .dvc file this command will generate.",
         metavar="<filename>",
     ).complete = completion.DIR
     import_parser.add_argument(
         "--no-exec",
         action="store_true",
         default=False,
-        help="Only create DVC-file without actually downloading it.",
+        help="Only create .dvc file without actually downloading it.",
+    )
+    import_parser.add_argument(
+        "--to-remote",
+        action="store_true",
+        default=False,
+        help="Download it directly to the remote",
+    )
+    import_parser.add_argument(
+        "-r",
+        "--remote",
+        help="Remote storage to download to",
+        metavar="<name>",
+    )
+    import_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
+    )
+    import_parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the data (optional). "
+            "This doesn't affect any DVC operations."
+        ),
     )
     import_parser.set_defaults(func=CmdImportUrl)

dvc/command/init.py

@@ -1,24 +1,54 @@
 import argparse
 import logging
 
+import colorama
+
+from dvc import analytics
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.utils import boxify
+from dvc.utils import format_link as fmt_link
 
 logger = logging.getLogger(__name__)
 
 
+def _welcome_message():
+    from dvc.ui import ui
+
+    if analytics.is_enabled():
+        ui.write(
+            boxify(
+                "DVC has enabled anonymous aggregate usage analytics.\n"
+                "Read the analytics documentation (and how to opt-out) here:\n"
+                + fmt_link("https://dvc.org/doc/user-guide/analytics"),
+                border_color="red",
+            )
+        )
+
+    msg = (
+        "{yellow}What's next?{nc}\n"
+        "{yellow}------------{nc}\n"
+        f"- Check out the documentation: {fmt_link('https://dvc.org/doc')}\n"
+        f"- Get help and share ideas: {fmt_link('https://dvc.org/chat')}\n"
+        f"- Star us on GitHub: {fmt_link('https://github.com/iterative/dvc')}"
+    ).format(yellow=colorama.Fore.YELLOW, nc=colorama.Fore.RESET)
+
+    ui.write(msg)
+
+
 class CmdInit(CmdBaseNoRepo):
     def run(self):
         from dvc.exceptions import InitError
         from dvc.repo import Repo
 
         try:
-            self.repo = Repo.init(
+            with Repo.init(
                 ".",
                 no_scm=self.args.no_scm,
                 force=self.args.force,
                 subdir=self.args.subdir,
-            )
-            self.config = self.repo.config
+            ) as repo:
+                self.config = repo.config
+                _welcome_message()
         except InitError:
             logger.exception("failed to initiate DVC")
             return 1

dvc/command/live.py

@@ -0,0 +1,93 @@
+import argparse
+from pathlib import Path
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, fix_subparsers
+from dvc.ui import ui
+
+
+class CmdLive(CmdBase):
+    UNINITIALIZED = True
+
+    def _run(self, target, revs=None):
+        metrics, plots = self.repo.live.show(target=target, revs=revs)
+
+        if plots:
+            html_path = Path.cwd() / (self.args.target + "_html")
+            from dvc.render.utils import render
+
+            index_path = render(self.repo, plots, metrics, html_path)
+            ui.write(index_path.as_uri())
+            return 0
+        return 1
+
+
+class CmdLiveShow(CmdLive):
+    def run(self):
+        return self._run(self.args.target)
+
+
+class CmdLiveDiff(CmdLive):
+    def run(self):
+        return self._run(self.args.target, self.args.revs)
+
+
+def shared_parent_parser():
+    parent_parser = argparse.ArgumentParser(add_help=False)
+    parent_parser.add_argument(
+        "target", help="Logs dir to produce summary from"
+    ).complete = completion.DIR
+    parent_parser.add_argument(
+        "-o",
+        "--out",
+        default=None,
+        help="Destination path to save plots to",
+        metavar="<path>",
+    ).complete = completion.DIR
+    return parent_parser
+
+
+def add_parser(subparsers, parent_parser):
+    LIVE_DESCRIPTION = (
+        "Commands to visualize and compare dvclive-produced logs."
+    )
+    live_parser = subparsers.add_parser(
+        "live",
+        parents=[parent_parser],
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=LIVE_DESCRIPTION,
+    )
+    live_subparsers = live_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc live CMD --help` to display command-specific help.",
+    )
+
+    fix_subparsers(live_subparsers)
+
+    SHOW_HELP = "Visualize dvclive directory content."
+    live_show_parser = live_subparsers.add_parser(
+        "show",
+        parents=[parent_parser, shared_parent_parser()],
+        help=SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    live_show_parser.set_defaults(func=CmdLiveShow)
+
+    DIFF_HELP = (
+        "Show multiple versions of dvclive data, "
+        "by plotting it in single view."
+    )
+    live_diff_parser = live_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser, shared_parent_parser()],
+        help=DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    live_diff_parser.add_argument(
+        "--revs",
+        nargs="*",
+        default=None,
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
+    )
+    live_diff_parser.set_defaults(func=CmdLiveDiff)

dvc/command/ls/__init__.py

@@ -1,11 +1,11 @@
 import argparse
 import logging
-import sys
 
 from dvc.command import completion
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.command.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
@@ -34,9 +34,13 @@ class CmdList(CmdBaseNoRepo):
                 recursive=self.args.recursive,
                 dvc_only=self.args.dvc_only,
             )
-            if entries:
-                entries = _prettify(entries, sys.stdout.isatty())
-                logger.info("\n".join(entries))
+            if self.args.show_json:
+                import json
+
+                ui.write(json.dumps(entries))
+            elif entries:
+                entries = _prettify(entries, with_color=True)
+                ui.write("\n".join(entries))
             return 0
         except DvcException:
             logger.exception(f"failed to list '{self.args.url}'")
@@ -65,6 +69,9 @@ def add_parser(subparsers, parent_parser):
     list_parser.add_argument(
         "--dvc-only", action="store_true", help="Show only DVC outputs."
     )
+    list_parser.add_argument(
+        "--show-json", action="store_true", help="Show output in JSON format."
+    )
     list_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/machine.py

@@ -0,0 +1,296 @@
+import argparse
+
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.command.config import CmdConfig
+from dvc.config import ConfigError
+from dvc.ui import ui
+from dvc.utils import format_link
+
+
+class MachineDisabledError(ConfigError):
+    def __init__(self):
+        super().__init__("Machine feature is disabled")
+
+
+class CmdMachineConfig(CmdConfig):
+    def __init__(self, args):
+
+        super().__init__(args)
+        if not self.config["feature"].get("machine", False):
+            raise MachineDisabledError
+
+        if getattr(self.args, "name", None):
+            self.args.name = self.args.name.lower()
+
+    def _check_exists(self, conf):
+        if self.args.name not in conf["machine"]:
+            raise ConfigError(f"machine '{self.args.name}' doesn't exist.")
+
+
+class CmdMachineAdd(CmdMachineConfig):
+    def run(self):
+        from dvc.machine import validate_name
+
+        validate_name(self.args.name)
+
+        if self.args.default:
+            ui.write(f"Setting '{self.args.name}' as a default machine.")
+
+        with self.config.edit(self.args.level) as conf:
+            if self.args.name in conf["machine"] and not self.args.force:
+                raise ConfigError(
+                    "machine '{}' already exists. Use `-f|--force` to "
+                    "overwrite it.".format(self.args.name)
+                )
+
+            conf["machine"][self.args.name] = {"cloud": self.args.cloud}
+            if self.args.default:
+                conf["core"]["machine"] = self.args.name
+
+        return 0
+
+
+class CmdMachineRemove(CmdMachineConfig):
+    def run(self):
+        with self.config.edit(self.args.level) as conf:
+            self._check_exists(conf)
+            del conf["machine"][self.args.name]
+
+        up_to_level = self.args.level or "repo"
+        # Remove core.machine refs to this machine in any shadowing configs
+        for level in reversed(self.config.LEVELS):
+            with self.config.edit(level) as conf:
+                if conf["core"].get("machine") == self.args.name:
+                    del conf["core"]["machine"]
+
+            if level == up_to_level:
+                break
+
+        return 0
+
+
+class CmdMachineModify(CmdMachineConfig):
+    def run(self):
+        from dvc.config import merge
+
+        with self.config.edit(self.args.level) as conf:
+            merged = self.config.load_config_to_level(self.args.level)
+            merge(merged, conf)
+            self._check_exists(merged)
+
+            if self.args.name not in conf["machine"]:
+                conf["machine"][self.args.name] = {}
+            section = conf["machine"][self.args.name]
+            if self.args.unset:
+                section.pop(self.args.option, None)
+            else:
+                section[self.args.option] = self.args.value
+        return 0
+
+
+class CmdMachineDefault(CmdMachineConfig):
+    def run(self):
+        if self.args.name is None and not self.args.unset:
+            conf = self.config.read(self.args.level)
+            try:
+                print(conf["core"]["machine"])
+            except KeyError:
+                ui.write("No default machine set")
+                return 1
+        else:
+            with self.config.edit(self.args.level) as conf:
+                if self.args.unset:
+                    conf["core"].pop("machine", None)
+                else:
+                    merged_conf = self.config.load_config_to_level(
+                        self.args.level
+                    )
+                    if (
+                        self.args.name in conf["machine"]
+                        or self.args.name in merged_conf["machine"]
+                    ):
+                        conf["core"]["machine"] = self.args.name
+                    else:
+                        raise ConfigError(
+                            "default machine must be present in machine "
+                            "list."
+                        )
+        return 0
+
+
+class CmdMachineCreate(CmdBase):
+    def run(self):
+        if self.repo.machine is None:
+            raise MachineDisabledError
+
+        self.repo.machine.create(self.args.name)
+        return 0
+
+
+class CmdMachineDestroy(CmdBase):
+    def run(self):
+        if self.repo.machine is None:
+            raise MachineDisabledError
+
+        self.repo.machine.destroy(self.args.name)
+        return 0
+
+
+class CmdMachineSsh(CmdBase):
+    def run(self):
+        if self.repo.machine is None:
+            raise MachineDisabledError
+
+        self.repo.machine.run_shell(self.args.name)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    from dvc.command.config import parent_config_parser
+
+    machine_HELP = "Set up and manage cloud machines."
+    machine_parser = subparsers.add_parser(
+        "machine",
+        parents=[parent_parser],
+        description=append_doc_link(machine_HELP, "machine"),
+        # NOTE: suppress help during development to hide command
+        # help=machine_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    machine_subparsers = machine_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc machine CMD --help` for " "command-specific help.",
+    )
+
+    fix_subparsers(machine_subparsers)
+
+    machine_ADD_HELP = "Add a new data machine."
+    machine_add_parser = machine_subparsers.add_parser(
+        "add",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_ADD_HELP, "machine/add"),
+        help=machine_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_add_parser.add_argument("name", help="Name of the machine")
+    machine_add_parser.add_argument(
+        "cloud",
+        help="Machine cloud. See full list of supported clouds at {}".format(
+            format_link(
+                "https://github.com/iterative/"
+                "terraform-provider-iterative#machine"
+            )
+        ),
+    )
+    machine_add_parser.add_argument(
+        "-d",
+        "--default",
+        action="store_true",
+        default=False,
+        help="Set as default machine.",
+    )
+    machine_add_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force overwriting existing configs",
+    )
+    machine_add_parser.set_defaults(func=CmdMachineAdd)
+
+    machine_DEFAULT_HELP = "Set/unset the default machine."
+    machine_default_parser = machine_subparsers.add_parser(
+        "default",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_DEFAULT_HELP, "machine/default"),
+        help=machine_DEFAULT_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_default_parser.add_argument(
+        "name", nargs="?", help="Name of the machine"
+    )
+    machine_default_parser.add_argument(
+        "-u",
+        "--unset",
+        action="store_true",
+        default=False,
+        help="Unset default machine.",
+    )
+    machine_default_parser.set_defaults(func=CmdMachineDefault)
+
+    machine_MODIFY_HELP = "Modify the configuration of an machine."
+    machine_modify_parser = machine_subparsers.add_parser(
+        "modify",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_MODIFY_HELP, "machine/modify"),
+        help=machine_MODIFY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_modify_parser.add_argument("name", help="Name of the machine")
+    machine_modify_parser.add_argument(
+        "option", help="Name of the option to modify."
+    )
+    machine_modify_parser.add_argument(
+        "value", nargs="?", help="(optional) Value of the option."
+    )
+    machine_modify_parser.add_argument(
+        "-u",
+        "--unset",
+        default=False,
+        action="store_true",
+        help="Unset option.",
+    )
+    machine_modify_parser.set_defaults(func=CmdMachineModify)
+
+    machine_REMOVE_HELP = "Remove an machine."
+    machine_remove_parser = machine_subparsers.add_parser(
+        "remove",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_REMOVE_HELP, "machine/remove"),
+        help=machine_REMOVE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_remove_parser.add_argument(
+        "name", help="Name of the machine to remove."
+    )
+    machine_remove_parser.set_defaults(func=CmdMachineRemove)
+
+    machine_CREATE_HELP = "Create and start a machine instance."
+    machine_create_parser = machine_subparsers.add_parser(
+        "create",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_CREATE_HELP, "machine/create"),
+        help=machine_CREATE_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_create_parser.add_argument(
+        "name", help="Name of the machine to create."
+    )
+    machine_create_parser.set_defaults(func=CmdMachineCreate)
+
+    machine_DESTROY_HELP = "Destroy an machine instance."
+    machine_destroy_parser = machine_subparsers.add_parser(
+        "destroy",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_DESTROY_HELP, "machine/destroy"),
+        help=machine_DESTROY_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_destroy_parser.add_argument(
+        "name", help="Name of the machine instance to destroy."
+    )
+    machine_destroy_parser.set_defaults(func=CmdMachineDestroy)
+
+    machine_SSH_HELP = "Connect to a machine via SSH."
+    machine_ssh_parser = machine_subparsers.add_parser(
+        "ssh",
+        parents=[parent_config_parser, parent_parser],
+        description=append_doc_link(machine_SSH_HELP, "machine/ssh"),
+        help=machine_SSH_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    machine_ssh_parser.add_argument(
+        "name", help="Name of the machine instance to connect to."
+    )
+    machine_ssh_parser.set_defaults(func=CmdMachineSsh)

dvc/command/metrics.py

@@ -3,7 +3,9 @@ import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
-from dvc.exceptions import BadMetricError, DvcException
+from dvc.exceptions import DvcException
+from dvc.ui import ui
+from dvc.utils.serialize import encode_exception
 
 logger = logging.getLogger(__name__)
 
@@ -11,34 +13,11 @@ logger = logging.getLogger(__name__)
 DEFAULT_PRECISION = 5
 
 
-def show_metrics(
-    metrics, all_branches=False, all_tags=False, all_commits=False
-):
-    from dvc.utils.diff import format_dict
-    from dvc.utils.flatten import flatten
-
-    # When `metrics` contains a `None` key, it means that some files
-    # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
-    missing = metrics.pop(None, None)
-
-    for branch, val in metrics.items():
-        if all_branches or all_tags or all_commits:
-            logger.info(f"{branch}:")
-
-        for fname, metric in val.items():
-            if not isinstance(metric, dict):
-                logger.info("\t{}: {}".format(fname, str(metric)))
-                continue
-
-            logger.info(f"\t{fname}:")
-            for key, value in flatten(format_dict(metric)).items():
-                logger.info(f"\t\t{key}: {value}")
-
-    if missing:
-        raise BadMetricError(missing)
+class CmdMetricsBase(CmdBase):
+    UNINITIALIZED = True
 
 
-class CmdMetricsShow(CmdBase):
+class CmdMetricsShow(CmdMetricsBase):
     def run(self):
         try:
             metrics = self.repo.metrics.show(
@@ -48,63 +27,31 @@ class CmdMetricsShow(CmdBase):
                 all_commits=self.args.all_commits,
                 recursive=self.args.recursive,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(metrics))
-            else:
-                show_metrics(
-                    metrics,
-                    self.args.all_branches,
-                    self.args.all_tags,
-                    self.args.all_commits,
-                )
         except DvcException:
-            logger.exception("failed to show metrics")
+            logger.exception("")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            ui.write(json.dumps(metrics, default=encode_exception))
+        else:
+            from dvc.compare import show_metrics
+
+            show_metrics(
+                metrics,
+                markdown=self.args.show_md,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
+                precision=self.args.precision or DEFAULT_PRECISION,
+                round_digits=True,
+            )
+
         return 0
 
 
-def _show_diff(diff, markdown=False, no_path=False, old=False, precision=None):
-    from collections import OrderedDict
-
-    from dvc.utils.diff import table
-
-    if precision is None:
-        precision = DEFAULT_PRECISION
-
-    def _round(val):
-        if isinstance(val, float):
-            return round(val, precision)
-
-        return val
-
-    rows = []
-    for fname, mdiff in diff.items():
-        sorted_mdiff = OrderedDict(sorted(mdiff.items()))
-        for metric, change in sorted_mdiff.items():
-            row = [] if no_path else [fname]
-            row.append(metric)
-            if old:
-                row.append(_round(change.get("old")))
-            row.append(_round(change["new"]))
-            row.append(_round(change.get("diff", "diff not supported")))
-            rows.append(row)
-
-    header = [] if no_path else ["Path"]
-    header.append("Metric")
-    if old:
-        header.extend(["Old", "New"])
-    else:
-        header.append("Value")
-    header.append("Change")
-
-    return table(header, rows, markdown)
-
-
-class CmdMetricsDiff(CmdBase):
+class CmdMetricsDiff(CmdMetricsBase):
     def run(self):
         try:
             diff = self.repo.metrics.diff(
@@ -114,26 +61,26 @@ class CmdMetricsDiff(CmdBase):
                 recursive=self.args.recursive,
                 all=self.args.all,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(diff))
-            else:
-                table = _show_diff(
-                    diff,
-                    self.args.show_md,
-                    self.args.no_path,
-                    self.args.old,
-                    precision=self.args.precision,
-                )
-                if table:
-                    logger.info(table)
-
         except DvcException:
             logger.exception("failed to show metrics diff")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            ui.write(json.dumps(diff))
+        else:
+            from dvc.compare import show_diff
+
+            show_diff(
+                diff,
+                title="Metric",
+                markdown=self.args.show_md,
+                no_path=self.args.no_path,
+                precision=self.args.precision or DEFAULT_PRECISION,
+                round_digits=True,
+            )
+
         return 0
 
 
@@ -167,8 +114,8 @@ def add_parser(subparsers, parent_parser):
         "targets",
         nargs="*",
         help=(
-            "Limit command scope to these metric files. Using -R, "
-            "directories to search metric files in can also be given."
+            "Limit command scope to these metrics files. Using -R, "
+            "directories to search metrics files in can also be given."
         ),
     ).complete = completion.FILE
     metrics_show_parser.add_argument(
@@ -186,6 +133,7 @@ def add_parser(subparsers, parent_parser):
         help="Show metrics for all tags.",
     )
     metrics_show_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -197,6 +145,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show output in JSON format.",
     )
+    metrics_show_parser.add_argument(
+        "--show-md",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
     metrics_show_parser.add_argument(
         "-R",
         "--recursive",
@@ -204,9 +158,18 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help=(
             "If any target is a directory, recursively search and process "
-            "metric files."
+            "metrics files."
         ),
     )
+    metrics_show_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics to `n` digits precision after the decimal point. "
+            f"Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
     METRICS_DIFF_HELP = (
@@ -232,8 +195,11 @@ def add_parser(subparsers, parent_parser):
         "--targets",
         nargs="*",
         help=(
-            "Limit command scope to these metric files. Using -R, "
-            "directories to search metric files in can also be given."
+            "Specific metrics file(s) to compare "
+            "(even if not found as `metrics` in `dvc.yaml`). "
+            "Using -R, directories to search metrics files in "
+            "can also be given."
+            "Shows all tracked metrics by default."
         ),
         metavar="<paths>",
     ).complete = completion.FILE
@@ -244,7 +210,7 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help=(
             "If any target is a directory, recursively search and process "
-            "metric files."
+            "metrics files."
         ),
     )
     metrics_diff_parser.add_argument(
@@ -265,12 +231,6 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show tabulated output in the Markdown format (GFM).",
     )
-    metrics_diff_parser.add_argument(
-        "--old",
-        action="store_true",
-        default=False,
-        help="Show old metric value.",
-    )
     metrics_diff_parser.add_argument(
         "--no-path",
         action="store_true",

dvc/command/move.py

@@ -25,7 +25,7 @@ def add_parser(subparsers, parent_parser):
     MOVE_HELP = "Rename or move a DVC controlled data file or a directory."
     MOVE_DESCRIPTION = (
         "Rename or move a DVC controlled data file or a directory.\n"
-        "It renames and modifies the corresponding DVC-file to reflect the"
+        "It renames and modifies the corresponding .dvc file to reflect the"
         " changes."
     )
 
@@ -37,9 +37,9 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     move_parser.add_argument(
-        "src", help="Source path to a data file or directory.",
+        "src", help="Source path to a data file or directory."
     ).complete = completion.FILE
     move_parser.add_argument(
-        "dst", help="Destination path.",
+        "dst", help="Destination path."
     ).complete = completion.FILE
     move_parser.set_defaults(func=CmdMove)

dvc/command/params.py

@@ -1,56 +1,45 @@
 import argparse
 import logging
-from collections import OrderedDict
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
 
-def _show_diff(diff, markdown=False, no_path=False):
-    from dvc.utils.diff import table
-
-    rows = []
-    for fname, pdiff in diff.items():
-        sorted_pdiff = OrderedDict(sorted(pdiff.items()))
-        for param, change in sorted_pdiff.items():
-            row = [] if no_path else [fname]
-            row.append(param)
-            row.append(change["old"])
-            row.append(change["new"])
-            rows.append(row)
-
-    header = [] if no_path else ["Path"]
-    header.append("Param")
-    header.append("Old")
-    header.append("New")
-
-    return table(header, rows, markdown)
-
-
 class CmdParamsDiff(CmdBase):
+    UNINITIALIZED = True
+
     def run(self):
         try:
             diff = self.repo.params.diff(
                 a_rev=self.args.a_rev,
                 b_rev=self.args.b_rev,
+                targets=self.args.targets,
                 all=self.args.all,
+                deps=self.args.deps,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(diff))
-            else:
-                table = _show_diff(diff, self.args.show_md, self.args.no_path)
-                if table:
-                    logger.info(table)
-
         except DvcException:
             logger.exception("failed to show params diff")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            ui.write(json.dumps(diff))
+        else:
+            from dvc.compare import show_diff
+
+            show_diff(
+                diff,
+                title="Param",
+                markdown=self.args.show_md,
+                no_path=self.args.no_path,
+                show_changes=False,
+            )
+
         return 0
 
 
@@ -91,12 +80,28 @@ def add_parser(subparsers, parent_parser):
         nargs="?",
         help=("New Git commit to compare (defaults to the current workspace)"),
     )
+    params_diff_parser.add_argument(
+        "--targets",
+        nargs="*",
+        help=(
+            "Specific params file(s) to compare "
+            "(even if not found as `params` in `dvc.yaml`). "
+            "Shows all tracked params by default."
+        ),
+        metavar="<paths>",
+    ).complete = completion.FILE
     params_diff_parser.add_argument(
         "--all",
         action="store_true",
         default=False,
         help="Show unchanged params as well.",
     )
+    params_diff_parser.add_argument(
+        "--deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     params_diff_parser.add_argument(
         "--show-json",
         action="store_true",

dvc/command/plots.py

@@ -1,45 +1,30 @@
 import argparse
 import logging
-import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
-from dvc.schema import PLOT_PROPS
+from dvc.render.utils import find_vega, render
+from dvc.ui import ui
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
-PAGE_HTML = """<!DOCTYPE html>
-<html>
-<head>
-    <title>DVC Plot</title>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
-</head>
-<body>
-    {divs}
-</body>
-</html>"""
-
-DIV_HTML = """<div id = "{id}"></div>
-<script type = "text/javascript">
-    var spec = {vega_json};
-    vegaEmbed('#{id}', spec);
-</script>"""
-
 
 class CmdPlots(CmdBase):
     def _func(self, *args, **kwargs):
         raise NotImplementedError
 
     def _props(self):
+        from dvc.schema import PLOT_PROPS
+
         # Pass only props specified by user, to not shadow ones from plot def
         props = {p: getattr(self.args, p) for p in PLOT_PROPS}
         return {k: v for k, v in props.items() if v is not None}
 
     def run(self):
+        from pathlib import Path
+
         if self.args.show_vega:
             if not self.args.targets:
                 logger.error("please specify a target for `--show-vega`")
@@ -51,40 +36,62 @@ class CmdPlots(CmdBase):
                 return 1
 
         try:
-            plots = self._func(targets=self.args.targets, props=self._props())
+            plots_data = self._func(
+                targets=self.args.targets, props=self._props()
+            )
+
+            if not plots_data:
+                ui.error_write(
+                    "No plots were loaded, "
+                    "visualization file will not be created."
+                )
 
             if self.args.show_vega:
                 target = self.args.targets[0]
-                logger.info(plots[target])
+                plot_json = find_vega(self.repo, plots_data, target)
+                if plot_json:
+                    ui.write(plot_json)
                 return 0
 
-            divs = [
-                DIV_HTML.format(id=f"plot{i}", vega_json=plot)
-                for i, plot in enumerate(plots.values())
-            ]
-            html = PAGE_HTML.format(divs="\n".join(divs))
-            path = self.args.out or "plots.html"
-
-            with open(path, "w") as fobj:
-                fobj.write(html)
-
-            logger.info(
-                "file://{}".format(os.path.join(self.repo.root_dir, path))
+            rel: str = self.args.out or "dvc_plots"
+            path: Path = (Path.cwd() / rel).resolve()
+            index_path = render(
+                self.repo,
+                plots_data,
+                path=path,
+                html_template_path=self.args.html_template,
             )
 
+            assert index_path.is_absolute()
+            url = index_path.as_uri()
+            ui.write(url)
+
+            if self.args.open:
+                import webbrowser
+
+                opened = webbrowser.open(index_path)
+                if not opened:
+                    ui.error_write(
+                        "Failed to open. Please try opening it manually."
+                    )
+                    return 1
+            return 0
+
         except DvcException:
             logger.exception("")
             return 1
 
-        return 0
-
 
 class CmdPlotsShow(CmdPlots):
+    UNINITIALIZED = True
+
     def _func(self, *args, **kwargs):
         return self.repo.plots.show(*args, **kwargs)
 
 
 class CmdPlotsDiff(CmdPlots):
+    UNINITIALIZED = True
+
     def _func(self, *args, **kwargs):
         return self.repo.plots.diff(
             *args,
@@ -97,7 +104,7 @@ class CmdPlotsDiff(CmdPlots):
 class CmdPlotsModify(CmdPlots):
     def run(self):
         self.repo.plots.modify(
-            self.args.target, props=self._props(), unset=self.args.unset,
+            self.args.target, props=self._props(), unset=self.args.unset
         )
         return 0
 
@@ -105,7 +112,7 @@ class CmdPlotsModify(CmdPlots):
 def add_parser(subparsers, parent_parser):
     PLOTS_HELP = (
         "Commands to visualize and compare plot metrics in structured files "
-        "(JSON, YAML, CSV, TSV)"
+        "(JSON, YAML, CSV, TSV)."
     )
 
     plots_parser = subparsers.add_parser(
@@ -122,7 +129,7 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(plots_subparsers)
 
-    SHOW_HELP = "Generate plots from metric files."
+    SHOW_HELP = "Generate plots from metrics files."
     plots_show_parser = plots_subparsers.add_parser(
         "show",
         parents=[parent_parser],
@@ -133,7 +140,9 @@ def add_parser(subparsers, parent_parser):
     plots_show_parser.add_argument(
         "targets",
         nargs="*",
-        help="Plots files to visualize. Shows all plots by default.",
+        help="Files to visualize (supports any file, "
+        "even when not found as `plots` in `dvc.yaml`). "
+        "Shows all plots by default.",
     ).complete = completion.FILE
     _add_props_arguments(plots_show_parser)
     _add_output_arguments(plots_show_parser)
@@ -153,8 +162,12 @@ def add_parser(subparsers, parent_parser):
     plots_diff_parser.add_argument(
         "--targets",
         nargs="*",
-        help="Plots file to visualize. Shows all plots by default.",
-        metavar="<path>",
+        help=(
+            "Specific plots file(s) to visualize "
+            "(even if not found as `plots` in `dvc.yaml`). "
+            "Shows all tracked plots by default."
+        ),
+        metavar="<paths>",
     ).complete = completion.FILE
     plots_diff_parser.add_argument(
         "-e",
@@ -164,13 +177,13 @@ def add_parser(subparsers, parent_parser):
         help=argparse.SUPPRESS,
     )
     plots_diff_parser.add_argument(
-        "revisions", nargs="*", default=None, help="Git commits to plot from",
+        "revisions", nargs="*", default=None, help="Git commits to plot from"
     )
     _add_props_arguments(plots_diff_parser)
     _add_output_arguments(plots_diff_parser)
     plots_diff_parser.set_defaults(func=CmdPlotsDiff)
 
-    PLOTS_MODIFY_HELP = "Modify display properties of plot metric files."
+    PLOTS_MODIFY_HELP = "Modify display properties of plot metrics files."
     plots_modify_parser = plots_subparsers.add_parser(
         "modify",
         parents=[parent_parser],
@@ -179,7 +192,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     plots_modify_parser.add_argument(
-        "target", help="Metric file to set properties to",
+        "target", help="Metric file to set properties to"
     ).complete = completion.FILE
     _add_props_arguments(plots_modify_parser)
     plots_modify_parser.add_argument(
@@ -243,3 +256,15 @@ def _add_output_arguments(parser):
         default=False,
         help="Show output in Vega format.",
     )
+    parser.add_argument(
+        "--open",
+        action="store_true",
+        default=False,
+        help="Open plot file directly in the browser.",
+    )
+    parser.add_argument(
+        "--html-template",
+        default=None,
+        help="Custom HTML template for VEGA visualization.",
+        metavar="<path>",
+    )

dvc/command/remote.py

@@ -1,13 +1,10 @@
 import argparse
-import logging
 
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-from dvc.config import ConfigError, merge
+from dvc.ui import ui
 from dvc.utils import format_link
 
-logger = logging.getLogger(__name__)
-
 
 class CmdRemote(CmdConfig):
     def __init__(self, args):
@@ -17,14 +14,18 @@ class CmdRemote(CmdConfig):
             self.args.name = self.args.name.lower()
 
     def _check_exists(self, conf):
+        from dvc.config import ConfigError
+
         if self.args.name not in conf["remote"]:
             raise ConfigError(f"remote '{self.args.name}' doesn't exist.")
 
 
 class CmdRemoteAdd(CmdRemote):
     def run(self):
+        from dvc.config import ConfigError
+
         if self.args.default:
-            logger.info(f"Setting '{self.args.name}' as a default remote.")
+            ui.write(f"Setting '{self.args.name}' as a default remote.")
 
         with self.config.edit(self.args.level) as conf:
             if self.args.name in conf["remote"] and not self.args.force:
@@ -46,13 +47,14 @@ class CmdRemoteRemove(CmdRemote):
             self._check_exists(conf)
             del conf["remote"][self.args.name]
 
+        up_to_level = self.args.level or "repo"
         # Remove core.remote refs to this remote in any shadowing configs
         for level in reversed(self.config.LEVELS):
             with self.config.edit(level) as conf:
                 if conf["core"].get("remote") == self.args.name:
                     del conf["core"]["remote"]
 
-            if level == self.args.level:
+            if level == up_to_level:
                 break
 
         return 0
@@ -60,6 +62,8 @@ class CmdRemoteRemove(CmdRemote):
 
 class CmdRemoteModify(CmdRemote):
     def run(self):
+        from dvc.config import merge
+
         with self.config.edit(self.args.level) as conf:
             merged = self.config.load_config_to_level(self.args.level)
             merge(merged, conf)
@@ -77,13 +81,14 @@ class CmdRemoteModify(CmdRemote):
 
 class CmdRemoteDefault(CmdRemote):
     def run(self):
+        from dvc.config import ConfigError
 
         if self.args.name is None and not self.args.unset:
-            conf = self.config.load_one(self.args.level)
+            conf = self.config.read(self.args.level)
             try:
                 print(conf["core"]["remote"])
             except KeyError:
-                logger.info("No default remote set")
+                ui.write("No default remote set")
                 return 1
         else:
             with self.config.edit(self.args.level) as conf:
@@ -107,9 +112,9 @@ class CmdRemoteDefault(CmdRemote):
 
 class CmdRemoteList(CmdRemote):
     def run(self):
-        conf = self.config.load_one(self.args.level)
+        conf = self.config.read(self.args.level)
         for name, conf in conf["remote"].items():
-            logger.info("{}\t{}".format(name, conf["url"]))
+            ui.write(name, conf["url"], sep="\t")
         return 0
 
 
@@ -119,6 +124,8 @@ class CmdRemoteRename(CmdRemote):
             conf["core"]["remote"] = self.args.new
 
     def run(self):
+        from dvc.config import ConfigError
+
         all_config = self.config.load_config_to_level(None)
         if self.args.new in all_config.get("remote", {}):
             raise ConfigError(
@@ -133,8 +140,9 @@ class CmdRemoteRename(CmdRemote):
             del conf["remote"][self.args.name]
             self._rename_default(conf)
 
+        up_to_level = self.args.level or "repo"
         for level in reversed(self.config.LEVELS):
-            if level == self.args.level:
+            if level == up_to_level:
                 break
             with self.config.edit(level) as level_conf:
                 self._rename_default(level_conf)

dvc/command/remove.py

@@ -14,14 +14,15 @@ class CmdRemove(CmdBase):
             try:
                 self.repo.remove(target, outs=self.args.outs)
             except DvcException:
-                logger.exception(f"failed to remove '{target}'")
+                logger.exception("")
                 return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     REMOVE_HELP = (
-        "Remove stage entry, remove .gitignore entry and unprotect outputs"
+        "Remove stages from dvc.yaml and/or"
+        " stop tracking files or directories."
     )
     remove_parser = subparsers.add_parser(
         "remove",
@@ -37,6 +38,8 @@ def add_parser(subparsers, parent_parser):
         help="Remove outputs as well.",
     )
     remove_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to remove.",
+        "targets",
+        nargs="+",
+        help=".dvc files or stages from dvc.yaml to remove.",
     ).complete = completion.DVC_FILE
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -1,83 +1,56 @@
 import argparse
-import logging
-import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.metrics import show_metrics
 from dvc.command.status import CmdDataStatus
-from dvc.dvcfile import PIPELINE_FILE
-from dvc.exceptions import DvcException
-
-logger = logging.getLogger(__name__)
 
 
 class CmdRepro(CmdBase):
     def run(self):
-        saved_dir = os.path.realpath(os.curdir)
-        if self.args.cwd:
-            os.chdir(self.args.cwd)
+        from dvc.ui import ui
 
-        # Dirty hack so the for loop below can at least enter once
-        if self.args.all_pipelines:
-            self.args.targets = [None]
-        elif not self.args.targets:
-            self.args.targets = self.default_targets
+        stages = self.repo.reproduce(**self._repro_kwargs)
+        if len(stages) == 0:
+            ui.write(CmdDataStatus.UP_TO_DATE_MSG)
+        else:
+            ui.write(
+                "Use `dvc push` to send your updates to " "remote storage."
+            )
 
-        ret = 0
-        for target in self.args.targets:
-            try:
-                stages = self.repo.reproduce(
-                    target,
-                    single_item=self.args.single_item,
-                    force=self.args.force,
-                    dry=self.args.dry,
-                    interactive=self.args.interactive,
-                    pipeline=self.args.pipeline,
-                    all_pipelines=self.args.all_pipelines,
-                    run_cache=not self.args.no_run_cache,
-                    no_commit=self.args.no_commit,
-                    downstream=self.args.downstream,
-                    recursive=self.args.recursive,
-                    force_downstream=self.args.force_downstream,
-                    experiment=self.args.experiment,
-                    queue=self.args.queue,
-                    run_all=self.args.run_all,
-                    jobs=self.args.jobs,
-                    params=self.args.params,
-                )
+        if self.args.metrics:
+            from dvc.compare import show_metrics
 
-                if len(stages) == 0:
-                    logger.info(CmdDataStatus.UP_TO_DATE_MSG)
+            metrics = self.repo.metrics.show()
+            show_metrics(metrics)
 
-                if self.args.metrics:
-                    metrics = self.repo.metrics.show()
-                    show_metrics(metrics)
-            except DvcException:
-                logger.exception("")
-                ret = 1
-                break
+        return 0
 
-        os.chdir(saved_dir)
-        return ret
+    @property
+    def _repro_kwargs(self):
+        return {
+            "targets": self.args.targets,
+            "single_item": self.args.single_item,
+            "force": self.args.force,
+            "dry": self.args.dry,
+            "interactive": self.args.interactive,
+            "pipeline": self.args.pipeline,
+            "all_pipelines": self.args.all_pipelines,
+            "run_cache": not self.args.no_run_cache,
+            "no_commit": self.args.no_commit,
+            "downstream": self.args.downstream,
+            "recursive": self.args.recursive,
+            "force_downstream": self.args.force_downstream,
+            "pull": self.args.pull,
+            "glob": self.args.glob,
+        }
 
 
-def add_parser(subparsers, parent_parser):
-    REPRO_HELP = (
-        "Reproduce complete or partial pipelines by executing their stages."
-    )
-    repro_parser = subparsers.add_parser(
-        "repro",
-        parents=[parent_parser],
-        description=append_doc_link(REPRO_HELP, "repro"),
-        help=REPRO_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
+def add_arguments(repro_parser):
     repro_parser.add_argument(
         "targets",
         nargs="*",
-        help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.",
-    ).complete = completion.DVC_FILE
+        help="Stages to reproduce. 'dvc.yaml' by default.",
+    ).complete = completion.DVCFILES_AND_STAGE
     repro_parser.add_argument(
         "-f",
         "--force",
@@ -93,13 +66,6 @@ def add_parser(subparsers, parent_parser):
         help="Reproduce only single data item without recursive dependencies "
         "check.",
     )
-    repro_parser.add_argument(
-        "-c",
-        "--cwd",
-        default=os.path.curdir,
-        help="Directory within your repo to reproduce from.",
-        metavar="<path>",
-    )
     repro_parser.add_argument(
         "-m",
         "--metrics",
@@ -126,8 +92,8 @@ def add_parser(subparsers, parent_parser):
         "--pipeline",
         action="store_true",
         default=False,
-        help="Reproduce the whole pipeline that the specified stage file "
-        "belongs to.",
+        help="Reproduce the whole pipeline that the specified targets "
+        "belong to.",
     )
     repro_parser.add_argument(
         "-P",
@@ -172,29 +138,32 @@ def add_parser(subparsers, parent_parser):
         help="Start from the specified stages when reproducing pipelines.",
     )
     repro_parser.add_argument(
-        "-e",
-        "--experiment",
+        "--pull",
         action="store_true",
         default=False,
-        help=argparse.SUPPRESS,
+        help=(
+            "Try automatically pulling missing cache for outputs restored "
+            "from the run-cache."
+        ),
     )
     repro_parser.add_argument(
-        "--params",
-        action="append",
-        default=[],
-        help=argparse.SUPPRESS,
-        metavar="[<filename>:]<params_list>",
-    )
-    repro_parser.add_argument(
-        "--queue", action="store_true", default=False, help=argparse.SUPPRESS
-    )
-    repro_parser.add_argument(
-        "--run-all",
+        "--glob",
         action="store_true",
         default=False,
-        help=argparse.SUPPRESS,
+        help="Allows targets containing shell-style wildcards.",
     )
-    repro_parser.add_argument(
-        "-j", "--jobs", type=int, help=argparse.SUPPRESS, metavar="<number>"
+
+
+def add_parser(subparsers, parent_parser):
+    REPRO_HELP = (
+        "Reproduce complete or partial pipelines by executing their stages."
     )
+    repro_parser = subparsers.add_parser(
+        "repro",
+        parents=[parent_parser],
+        description=append_doc_link(REPRO_HELP, "repro"),
+        help=REPRO_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    add_arguments(repro_parser)
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/root.py

@@ -10,8 +10,9 @@ logger = logging.getLogger(__name__)
 class CmdRoot(CmdBaseNoRepo):
     def run(self):
         from dvc.repo import Repo
+        from dvc.ui import ui
 
-        logger.info(relpath(Repo.find_root()))
+        ui.write(relpath(Repo.find_root()))
         return 0
 
 

dvc/command/run.py

@@ -2,6 +2,7 @@ import argparse
 import logging
 
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.command.stage import parse_cmd
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -20,8 +21,9 @@ class CmdRun(CmdBase):
                 self.args.plots_no_cache,
                 self.args.outs_persist,
                 self.args.outs_persist_no_cache,
+                self.args.checkpoints,
                 self.args.params,
-                self.args.command,
+                self.args.cmd,
             ]
         ):  # pragma: no cover
             logger.error(
@@ -31,60 +33,30 @@ class CmdRun(CmdBase):
             )
             return 1
 
+        kwargs = vars(self.args)
+        kwargs.update(
+            {
+                "cmd": parse_cmd(self.args.cmd),
+                "fname": kwargs.pop("file"),
+                "no_exec": (self.args.no_exec or bool(self.args.checkpoints)),
+                "run_cache": not kwargs.pop("no_run_cache"),
+            }
+        )
         try:
-            self.repo.run(
-                cmd=self._parsed_cmd(),
-                outs=self.args.outs,
-                outs_no_cache=self.args.outs_no_cache,
-                metrics=self.args.metrics,
-                metrics_no_cache=self.args.metrics_no_cache,
-                plots=self.args.plots,
-                plots_no_cache=self.args.plots_no_cache,
-                deps=self.args.deps,
-                params=self.args.params,
-                fname=self.args.file,
-                wdir=self.args.wdir,
-                no_exec=self.args.no_exec,
-                force=self.args.force,
-                run_cache=not self.args.no_run_cache,
-                no_commit=self.args.no_commit,
-                outs_persist=self.args.outs_persist,
-                outs_persist_no_cache=self.args.outs_persist_no_cache,
-                always_changed=self.args.always_changed,
-                name=self.args.name,
-                single_stage=self.args.single_stage,
-                external=self.args.external,
-            )
+            self.repo.run(**kwargs)
         except DvcException:
             logger.exception("")
             return 1
 
         return 0
 
-    def _parsed_cmd(self):
-        """
-        We need to take into account two cases:
-
-        - ['python code.py foo bar']: Used mainly with dvc as a library
-        - ['echo', 'foo bar']: List of arguments received from the CLI
-
-        The second case would need quoting, as it was passed through:
-                dvc run echo "foo bar"
-        """
-        if len(self.args.command) < 2:
-            return " ".join(self.args.command)
-
-        return " ".join(self._quote_argument(arg) for arg in self.args.command)
-
-    def _quote_argument(self, argument):
-        if " " not in argument or '"' in argument:
-            return argument
-
-        return f'"{argument}"'
-
 
 def add_parser(subparsers, parent_parser):
-    RUN_HELP = "Generate a stage file from a command and execute the command."
+    from dvc.command.stage import _add_common_args
+
+    RUN_HELP = (
+        "Generate a dvc.yaml file from a command and execute the command."
+    )
     run_parser = subparsers.add_parser(
         "run",
         parents=[parent_parser],
@@ -92,93 +64,15 @@ def add_parser(subparsers, parent_parser):
         help=RUN_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    run_parser.add_argument("-n", "--name", help="Stage name.")
     run_parser.add_argument(
-        "-d",
-        "--deps",
-        action="append",
-        default=[],
-        help="Declare dependencies for reproducible cmd.",
-        metavar="<path>",
-    )
-    run_parser.add_argument(
-        "-n", "--name", help="Stage name.",
-    )
-    run_parser.add_argument(
-        "-o",
-        "--outs",
-        action="append",
-        default=[],
-        help="Declare output file or directory.",
-        metavar="<filename>",
-    )
-    run_parser.add_argument(
-        "-O",
-        "--outs-no-cache",
-        action="append",
-        default=[],
-        help="Declare output file or directory "
-        "(do not put into DVC cache).",
-        metavar="<filename>",
-    )
-    run_parser.add_argument(
-        "-p",
-        "--params",
-        action="append",
-        default=[],
-        help="Declare parameter to use as additional dependency.",
-        metavar="[<filename>:]<params_list>",
-    )
-    run_parser.add_argument(
-        "-m",
-        "--metrics",
-        action="append",
-        default=[],
-        help="Declare output metric file.",
-        metavar="<path>",
-    )
-    run_parser.add_argument(
-        "-M",
-        "--metrics-no-cache",
-        action="append",
-        default=[],
-        help="Declare output metric file (do not put into DVC cache).",
-        metavar="<path>",
-    )
-    run_parser.add_argument(
-        "--plots",
-        action="append",
-        default=[],
-        help="Declare output plot file.",
-        metavar="<path>",
-    )
-    run_parser.add_argument(
-        "--plots-no-cache",
-        action="append",
-        default=[],
-        help="Declare output plot file (do not put into DVC cache).",
-        metavar="<path>",
-    )
-    run_parser.add_argument(
-        "--file", metavar="<filename>", help=argparse.SUPPRESS,
-    )
-    run_parser.add_argument(
-        "-w",
-        "--wdir",
-        help="Directory within your repo to run your command in.",
-        metavar="<path>",
+        "--file", metavar="<filename>", help=argparse.SUPPRESS
     )
     run_parser.add_argument(
         "--no-exec",
         action="store_true",
         default=False,
-        help="Only create stage file without actually running it.",
-    )
-    run_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Overwrite existing stage",
+        help="Only create dvc.yaml without actually running it.",
     )
     run_parser.add_argument(
         "--no-run-cache",
@@ -195,41 +89,11 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Don't put files/directories into cache.",
     )
-    run_parser.add_argument(
-        "--outs-persist",
-        action="append",
-        default=[],
-        help="Declare output file or directory that will not be "
-        "removed upon repro.",
-        metavar="<filename>",
-    )
-    run_parser.add_argument(
-        "--outs-persist-no-cache",
-        action="append",
-        default=[],
-        help="Declare output file or directory that will not be "
-        "removed upon repro (do not put into DVC cache).",
-        metavar="<filename>",
-    )
-    run_parser.add_argument(
-        "--always-changed",
-        action="store_true",
-        default=False,
-        help="Always consider this DVC-file as changed.",
-    )
     run_parser.add_argument(
         "--single-stage",
         action="store_true",
         default=False,
         help=argparse.SUPPRESS,
     )
-    run_parser.add_argument(
-        "--external",
-        action="store_true",
-        default=False,
-        help="Allow outputs that are outside of the DVC repository.",
-    )
-    run_parser.add_argument(
-        "command", nargs=argparse.REMAINDER, help="Command to execute."
-    )
+    _add_common_args(run_parser)
     run_parser.set_defaults(func=CmdRun)

dvc/command/stage.py

@@ -0,0 +1,365 @@
+import argparse
+import logging
+from itertools import chain, filterfalse
+from typing import TYPE_CHECKING, Dict, Iterable, List
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
+from dvc.utils.cli_parse import parse_params
+from dvc.utils.humanize import truncate_text
+
+if TYPE_CHECKING:
+    from dvc.output import Output
+    from dvc.stage import Stage
+
+logger = logging.getLogger(__name__)
+
+MAX_TEXT_LENGTH = 80
+
+
+def generate_description(stage: "Stage") -> str:
+    def part_desc(outs: Iterable["Output"]) -> str:
+        return ", ".join(out.def_path for out in outs)
+
+    if not stage.deps and not stage.outs:
+        return "No outputs or dependencies"
+
+    if not stage.outs and stage.deps:
+        return "Depends on " + part_desc(stage.deps)
+
+    def is_plot_or_metric(out: "Output"):
+        return bool(out.plot) or bool(out.metric)
+
+    desc: List[str] = []
+
+    outs = list(filterfalse(is_plot_or_metric, stage.outs))
+    if outs:
+        desc.append("Outputs " + part_desc(outs))
+
+    plots_and_metrics = list(filter(is_plot_or_metric, stage.outs))
+    if plots_and_metrics:
+        desc.append("Reports " + part_desc(plots_and_metrics))
+
+    return "; ".join(desc)
+
+
+def prepare_description(
+    stage: "Stage", max_length: int = MAX_TEXT_LENGTH
+) -> str:
+    desc = stage.short_description() or generate_description(stage)
+    return truncate_text(desc, max_length)
+
+
+def prepare_stages_data(
+    stages: Iterable["Stage"],
+    description: bool = True,
+    max_length: int = MAX_TEXT_LENGTH,
+) -> Dict[str, str]:
+    return {
+        stage.addressing: prepare_description(stage, max_length=max_length)
+        if description
+        else ""
+        for stage in stages
+    }
+
+
+class CmdStageList(CmdBase):
+    def _get_stages(self) -> Iterable["Stage"]:
+        if self.args.all:
+            stages: List["Stage"] = self.repo.index.stages  # type: ignore
+            logger.trace(  # type: ignore[attr-defined]
+                "%d no. of stages found", len(stages)
+            )
+            return stages
+
+        # removing duplicates while maintaining order
+        collected = chain.from_iterable(
+            self.repo.stage.collect(
+                target=target, recursive=self.args.recursive, accept_group=True
+            )
+            for target in self.args.targets
+        )
+        return dict.fromkeys(collected).keys()
+
+    def run(self):
+        from dvc.ui import ui
+
+        def log_error(relpath: str, exc: Exception):
+            if self.args.fail:
+                raise exc
+            logger.debug("Stages from %s failed to load", relpath)
+
+        # silence stage collection error by default
+        self.repo.stage_collection_error_handler = log_error
+
+        stages = self._get_stages()
+        names_only = self.args.names_only
+
+        data = prepare_stages_data(stages, description=not names_only)
+        ui.table(data.items())
+
+        return 0
+
+
+def parse_cmd(commands: List[str]) -> str:
+    """
+    We need to take into account two cases:
+
+    - ['python code.py foo bar']: Used mainly with dvc as a library
+    - ['echo', 'foo bar']: List of arguments received from the CLI
+
+    The second case would need quoting, as it was passed through:
+            dvc run echo "foo bar"
+    """
+
+    def quote_argument(arg: str):
+        if not arg:
+            return '""'
+        if " " in arg and '"' not in arg:
+            return f'"{arg}"'
+        return arg
+
+    if len(commands) < 2:
+        return " ".join(commands)
+    return " ".join(map(quote_argument, commands))
+
+
+class CmdStageAdd(CmdBase):
+    def run(self):
+        kwargs = vars(self.args)
+        kwargs.update(
+            {
+                "cmd": parse_cmd(kwargs.pop("cmd")),
+                "params": parse_params(self.args.params),
+            }
+        )
+        self.repo.stage.add(**kwargs)
+        return 0
+
+
+def _add_common_args(parser):
+    parser.add_argument(
+        "-d",
+        "--deps",
+        action="append",
+        default=[],
+        help="Declare dependencies for reproducible cmd.",
+        metavar="<path>",
+    ).complete = completion.FILE
+    parser.add_argument(
+        "-o",
+        "--outs",
+        action="append",
+        default=[],
+        help="Declare output file or directory.",
+        metavar="<filename>",
+    ).complete = completion.FILE
+    parser.add_argument(
+        "-O",
+        "--outs-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory "
+        "(do not put into DVC cache).",
+        metavar="<filename>",
+    ).complete = completion.FILE
+    parser.add_argument(
+        "-p",
+        "--params",
+        action="append",
+        default=[],
+        help="Declare parameter to use as additional dependency.",
+        metavar="[<filename>:]<params_list>",
+    ).complete = completion.FILE
+    parser.add_argument(
+        "-m",
+        "--metrics",
+        action="append",
+        default=[],
+        help="Declare output metrics file.",
+        metavar="<path>",
+    )
+    parser.add_argument(
+        "-M",
+        "--metrics-no-cache",
+        action="append",
+        default=[],
+        help="Declare output metrics file (do not put into DVC cache).",
+        metavar="<path>",
+    )
+    parser.add_argument(
+        "--plots",
+        action="append",
+        default=[],
+        help="Declare output plot file.",
+        metavar="<path>",
+    )
+    parser.add_argument(
+        "--plots-no-cache",
+        action="append",
+        default=[],
+        help="Declare output plot file (do not put into DVC cache).",
+        metavar="<path>",
+    )
+    parser.add_argument(
+        "--live", help="Declare output as dvclive.", metavar="<path>"
+    )
+    parser.add_argument(
+        "--live-no-cache",
+        help="Declare output as dvclive (do not put into DVC cache).",
+        metavar="<path>",
+    )
+    parser.add_argument(
+        "--live-no-summary",
+        action="store_true",
+        default=False,
+        help="Signal dvclive logger to not dump latest metrics file.",
+    )
+    parser.add_argument(
+        "--live-no-html",
+        action="store_true",
+        default=False,
+        help="Signal dvclive logger to not produce training report.",
+    )
+    parser.add_argument(
+        "-w",
+        "--wdir",
+        help="Directory within your repo to run your command in.",
+        metavar="<path>",
+    )
+    parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Overwrite existing stage",
+    )
+    parser.add_argument(
+        "--outs-persist",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro.",
+        metavar="<filename>",
+    )
+    parser.add_argument(
+        "--outs-persist-no-cache",
+        action="append",
+        default=[],
+        help="Declare output file or directory that will not be "
+        "removed upon repro (do not put into DVC cache).",
+        metavar="<filename>",
+    )
+    parser.add_argument(
+        "-c",
+        "--checkpoints",
+        action="append",
+        default=[],
+        help="Declare checkpoint output file or directory for 'dvc exp run'. "
+        "Not compatible with 'dvc repro'.",
+        metavar="<filename>",
+    ).complete = completion.FILE
+    parser.add_argument(
+        "--always-changed",
+        action="store_true",
+        default=False,
+        help="Always consider this DVC-file as changed.",
+    )
+    parser.add_argument(
+        "--external",
+        action="store_true",
+        default=False,
+        help="Allow outputs that are outside of the DVC repository.",
+    )
+    parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the stage (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
+    parser.add_argument(
+        "cmd",
+        nargs=argparse.REMAINDER,
+        help="Command to execute.",
+        metavar="command",
+    )
+
+
+def add_parser(subparsers, parent_parser):
+    STAGES_HELP = "Commands to list and create stages."
+
+    stage_parser = subparsers.add_parser(
+        "stage",
+        parents=[parent_parser],
+        description=append_doc_link(STAGES_HELP, "stage"),
+        help=STAGES_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    stage_subparsers = stage_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc stage CMD --help` to display command-specific help.",
+    )
+
+    fix_subparsers(stage_subparsers)
+
+    STAGE_ADD_HELP = "Create stage"
+    stage_add_parser = stage_subparsers.add_parser(
+        "add",
+        parents=[parent_parser],
+        description=append_doc_link(STAGE_ADD_HELP, "stage/add"),
+        help=STAGE_ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    stage_add_parser.add_argument(
+        "-n", "--name", help="Name of the stage to add", required=True
+    )
+    _add_common_args(stage_add_parser)
+    stage_add_parser.set_defaults(func=CmdStageAdd)
+
+    STAGE_LIST_HELP = "List stages."
+    stage_list_parser = stage_subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description=append_doc_link(STAGE_LIST_HELP, "stage/list"),
+        help=STAGE_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    stage_list_parser.add_argument(
+        "targets",
+        nargs="*",
+        default=["dvc.yaml"],
+        help=(
+            "Show stages from a dvc.yaml/.dvc file or a directory. "
+            "'dvc.yaml' by default"
+        ),
+    )
+    stage_list_parser.add_argument(
+        "--all",
+        action="store_true",
+        default=False,
+        help="List all of the stages in the repo.",
+    )
+    stage_list_parser.add_argument(
+        "--fail",
+        action="store_true",
+        default=False,
+        help="Fail immediately, do not suppress any syntax errors.",
+    )
+    stage_list_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="List all stages inside the specified directory.",
+    )
+    stage_list_parser.add_argument(
+        "--names-only",
+        action="store_true",
+        default=False,
+        help="List only stage names.",
+    )
+    stage_list_parser.set_defaults(func=CmdStageList)

dvc/command/status.py

@@ -2,6 +2,8 @@ import logging
 
 from dvc.command.data_sync import CmdDataBase
 from dvc.exceptions import DvcException
+from dvc.ui import ui
+from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
@@ -10,6 +12,11 @@ class CmdDataStatus(CmdDataBase):
     STATUS_LEN = 20
     STATUS_INDENT = "\t"
     UP_TO_DATE_MSG = "Data and pipelines are up to date."
+    IN_SYNC_MSG = "Cache and remote '{remote}' are in sync."
+    EMPTY_PROJECT_MSG = (
+        "There are no data or pipelines tracked in this project yet.\n"
+        "See {link} to get started!"
+    ).format(link=format_link("https://dvc.org/doc/start"))
 
     def _normalize(self, s):
         s += ":"
@@ -20,7 +27,7 @@ class CmdDataStatus(CmdDataBase):
         ind = indent * self.STATUS_INDENT
 
         if isinstance(status, str):
-            logger.info(f"{ind}{status}")
+            ui.write(f"{ind}{status}")
             return
 
         if isinstance(status, list):
@@ -32,25 +39,32 @@ class CmdDataStatus(CmdDataBase):
 
         for key, value in status.items():
             if isinstance(value, str):
-                logger.info("{}{}{}".format(ind, self._normalize(value), key))
+                ui.write(f"{ind}{self._normalize(value)}{key}")
             elif value:
-                logger.info(f"{ind}{key}:")
+                ui.write(f"{ind}{key}:")
                 self._show(value, indent + 1)
 
     def run(self):
+        from dvc.repo import lock_repo
+
         indent = 1 if self.args.cloud else 0
-        try:
-            st = self.repo.status(
-                targets=self.args.targets,
-                jobs=self.args.jobs,
-                cloud=self.args.cloud,
-                remote=self.args.remote,
-                all_branches=self.args.all_branches,
-                all_tags=self.args.all_tags,
-                all_commits=self.args.all_commits,
-                with_deps=self.args.with_deps,
-                recursive=self.args.recursive,
-            )
+
+        with lock_repo(self.repo):
+            try:
+                st = self.repo.status(
+                    targets=self.args.targets,
+                    jobs=self.args.jobs,
+                    cloud=self.args.cloud,
+                    remote=self.args.remote,
+                    all_branches=self.args.all_branches,
+                    all_tags=self.args.all_tags,
+                    all_commits=self.args.all_commits,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("")
+                return 1
 
             if self.args.quiet:
                 return bool(st)
@@ -58,13 +72,22 @@ class CmdDataStatus(CmdDataBase):
             if self.args.show_json:
                 import json
 
-                logger.info(json.dumps(st))
-            elif st:
+                ui.write(json.dumps(st))
+                return 0
+
+            if st:
                 self._show(st, indent)
+                return 0
+
+            # additional hints for the user
+            if not self.repo.index.stages:
+                ui.write(self.EMPTY_PROJECT_MSG)
+            elif self.args.cloud or self.args.remote:
+                remote = self.args.remote or self.repo.config["core"].get(
+                    "remote"
+                )
+                ui.write(self.IN_SYNC_MSG.format(remote=remote))
             else:
-                logger.info(self.UP_TO_DATE_MSG)
+                ui.write(self.UP_TO_DATE_MSG)
 
-        except DvcException:
-            logger.exception("failed to obtain data status")
-            return 1
         return 0

dvc/command/unprotect.py

@@ -23,7 +23,7 @@ class CmdUnprotect(CmdBase):
 def add_parser(subparsers, parent_parser):
     UNPROTECT_HELP = (
         "Unprotect tracked files or directories (when hardlinks or symlinks "
-        "have been enabled with `dvc config cache.type`)"
+        "have been enabled with `dvc config cache.type`)."
     )
     unprotect_parser = subparsers.add_parser(
         "unprotect",
@@ -33,6 +33,6 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unprotect_parser.add_argument(
-        "targets", nargs="+", help="Data files/directories to unprotect.",
+        "targets", nargs="+", help="Data files/directories to unprotect."
     ).complete = completion.FILE
     unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/command/update.py

@@ -16,6 +16,9 @@ class CmdUpdate(CmdBase):
                 targets=self.args.targets,
                 rev=self.args.rev,
                 recursive=self.args.recursive,
+                to_remote=self.args.to_remote,
+                remote=self.args.remote,
+                jobs=self.args.jobs,
             )
         except DvcException:
             logger.exception("failed update data")
@@ -33,7 +36,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     update_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to update.",
+        "targets", nargs="+", help=".dvc files to update."
     ).complete = completion.DVC_FILE
     update_parser.add_argument(
         "--rev",
@@ -48,4 +51,27 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Update all stages in the specified directory.",
     )
+    update_parser.add_argument(
+        "--to-remote",
+        action="store_true",
+        default=False,
+        help="Update data directly on the remote",
+    )
+    update_parser.add_argument(
+        "-r",
+        "--remote",
+        help="Remote storage to perform updates to",
+        metavar="<name>",
+    )
+    update_parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
+        metavar="<number>",
+    )
     update_parser.set_defaults(func=CmdUpdate)

dvc/command/version.py

@@ -2,15 +2,20 @@ import argparse
 import logging
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
-from dvc.info import get_dvc_info
+from dvc.ui import ui
 
 logger = logging.getLogger(__name__)
 
 
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
+        from dvc.info import get_dvc_info
+        from dvc.updater import notify_updates
+
         dvc_info = get_dvc_info()
-        logger.info(dvc_info)
+        ui.write(dvc_info, force=True)
+
+        notify_updates()
         return 0
 
 
@@ -24,5 +29,6 @@ def add_parser(subparsers, parent_parser):
         description=append_doc_link(VERSION_HELP, "version"),
         help=VERSION_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
+        aliases=["doctor"],
     )
     version_parser.set_defaults(func=CmdVersion)

dvc/compare.py

@@ -0,0 +1,329 @@
+from collections import abc
+from itertools import chain, repeat, zip_longest
+from operator import itemgetter
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    ItemsView,
+    Iterable,
+    Iterator,
+    List,
+    Mapping,
+    MutableSequence,
+    Sequence,
+    Tuple,
+    Union,
+    overload,
+)
+
+from funcy import reraise
+
+if TYPE_CHECKING:
+    from dvc.ui.table import CellT
+
+
+class Column(List["CellT"]):
+    pass
+
+
+def with_value(value, default):
+    return default if value is None else value
+
+
+class TabularData(MutableSequence[Sequence["CellT"]]):
+    def __init__(self, columns: Sequence[str], fill_value: str = ""):
+        self._columns: Dict[str, Column] = {name: Column() for name in columns}
+        self._keys: List[str] = list(columns)
+        self._fill_value = fill_value
+
+    @property
+    def columns(self) -> List[Column]:
+        return list(map(self.column, self.keys()))
+
+    def column(self, name: str) -> Column:
+        return self._columns[name]
+
+    def items(self) -> ItemsView[str, Column]:
+        projection = {k: self.column(k) for k in self.keys()}
+        return projection.items()
+
+    def keys(self) -> List[str]:
+        return self._keys
+
+    def _iter_col_row(
+        self, row: Sequence["CellT"]
+    ) -> Iterator[Tuple["CellT", Column]]:
+        for val, col in zip_longest(row, self.columns):
+            if col is None:
+                break
+            yield with_value(val, self._fill_value), col
+
+    def append(self, value: Sequence["CellT"]) -> None:
+        for val, col in self._iter_col_row(value):
+            col.append(val)
+
+    def extend(self, values: Iterable[Sequence["CellT"]]) -> None:
+        for row in values:
+            self.append(row)
+
+    def insert(self, index: int, value: Sequence["CellT"]) -> None:
+        for val, col in self._iter_col_row(value):
+            col.insert(index, val)
+
+    def __iter__(self) -> Iterator[List["CellT"]]:
+        return map(list, zip(*self.columns))
+
+    def __getattr__(self, item: str) -> Column:
+        with reraise(KeyError, AttributeError):
+            return self.column(item)
+
+    def __getitem__(self, item: Union[int, slice]):
+        func = itemgetter(item)
+        it = map(func, self.columns)
+        if isinstance(item, slice):
+            it = map(list, zip(*it))
+        return list(it)
+
+    @overload
+    def __setitem__(self, item: int, value: Sequence["CellT"]) -> None:
+        ...
+
+    @overload
+    def __setitem__(
+        self, item: slice, value: Iterable[Sequence["CellT"]]
+    ) -> None:
+        ...
+
+    def __setitem__(self, item, value) -> None:
+        it = value
+        if isinstance(item, slice):
+            n = len(self.columns)
+            normalized_rows = (
+                chain(val, repeat(self._fill_value, n - len(val)))
+                for val in value
+            )
+            # we need to transpose those rows into columnar format
+            # as we work in terms of column-based arrays
+            it = zip(*normalized_rows)
+
+        for i, col in self._iter_col_row(it):
+            col[item] = i
+
+    def __delitem__(self, item: Union[int, slice]) -> None:
+        for col in self.columns:
+            del col[item]
+
+    def __len__(self) -> int:
+        return len(self.columns[0])
+
+    @property
+    def shape(self) -> Tuple[int, int]:
+        return len(self.columns), len(self)
+
+    def drop(self, *col_names: str) -> None:
+        for col_name in col_names:
+            self._keys.remove(col_name)
+            self._columns.pop(col_name)
+
+    def rename(self, from_col_name: str, to_col_name: str) -> None:
+        self._columns[to_col_name] = self._columns.pop(from_col_name)
+        self._keys[self._keys.index(from_col_name)] = to_col_name
+
+    def project(self, *col_names: str) -> None:
+        self.drop(*(set(self._keys) - set(col_names)))
+        self._keys = list(col_names)
+
+    def is_empty(self, col_name: str) -> bool:
+        col = self.column(col_name)
+        return not any(item != self._fill_value for item in col)
+
+    def to_csv(self) -> str:
+        import csv
+        from io import StringIO
+
+        buff = StringIO()
+        writer = csv.writer(buff)
+        writer.writerow(self.keys())
+
+        for row in self:
+            writer.writerow(row)
+        return buff.getvalue()
+
+    def add_column(self, name: str) -> None:
+        self._columns[name] = Column([self._fill_value] * len(self))
+        self._keys.append(name)
+
+    def row_from_dict(self, d: Mapping[str, "CellT"]) -> None:
+        keys = self.keys()
+        for key in d:
+            if key not in keys:
+                self.add_column(key)
+
+        row: List["CellT"] = [
+            with_value(d.get(key), self._fill_value) for key in self.keys()
+        ]
+        self.append(row)
+
+    def render(self, **kwargs: Any):
+        from dvc.ui import ui
+
+        if kwargs.pop("show_csv", False):
+            ui.write(self.to_csv(), end="")
+        else:
+            ui.table(self, headers=self.keys(), **kwargs)
+
+    def as_dict(
+        self, cols: Iterable[str] = None
+    ) -> Iterable[Dict[str, "CellT"]]:
+        keys = self.keys() if cols is None else set(cols)
+        return [
+            {k: self._columns[k][i] for k in keys} for i in range(len(self))
+        ]
+
+
+def _normalize_float(val: float, precision: int):
+    return f"{val:.{precision}g}"
+
+
+def _format_field(
+    val: Any, precision: int = None, round_digits: bool = False
+) -> str:
+    def _format(_val):
+        if isinstance(_val, float) and precision:
+            func = round if round_digits else _normalize_float
+            return func(_val, precision)
+        if isinstance(_val, abc.Mapping):
+            return {k: _format(v) for k, v in _val.items()}
+        if isinstance(_val, list):
+            return [_format(x) for x in _val]
+        return _val
+
+    return str(_format(val))
+
+
+def diff_table(
+    diff,
+    title: str,
+    old: bool = True,
+    no_path: bool = False,
+    show_changes: bool = True,
+    precision: int = None,
+    round_digits: bool = False,
+    on_empty_diff: str = None,
+) -> TabularData:
+    headers: List[str] = ["Path", title, "Old", "New", "Change"]
+    fill_value = "-"
+    td = TabularData(headers, fill_value=fill_value)
+
+    for fname, diff_in_file in diff.items():
+        for item, change in sorted(diff_in_file.items()):
+            old_value = with_value(change.get("old"), fill_value)
+            new_value = with_value(change.get("new"), fill_value)
+            diff_value = with_value(
+                change.get("diff", on_empty_diff), fill_value
+            )
+            td.append(
+                [
+                    fname,
+                    str(item),
+                    _format_field(old_value, precision, round_digits),
+                    _format_field(new_value, precision, round_digits),
+                    _format_field(diff_value, precision, round_digits),
+                ]
+            )
+
+    if no_path:
+        td.drop("Path")
+
+    if not show_changes:
+        td.drop("Change")
+
+    if not old:
+        td.drop("Old")
+        td.rename("New", "Value")
+
+    return td
+
+
+def show_diff(
+    diff,
+    title: str,
+    old: bool = True,
+    no_path: bool = False,
+    show_changes: bool = True,
+    precision: int = None,
+    round_digits: bool = False,
+    on_empty_diff: str = None,
+    markdown: bool = False,
+) -> None:
+    td = diff_table(
+        diff,
+        title=title,
+        old=old,
+        no_path=no_path,
+        show_changes=show_changes,
+        precision=precision,
+        round_digits=round_digits,
+        on_empty_diff=on_empty_diff,
+    )
+    td.render(markdown=markdown)
+
+
+def metrics_table(
+    metrics,
+    all_branches: bool = False,
+    all_tags: bool = False,
+    all_commits: bool = False,
+    precision: int = None,
+    round_digits: bool = False,
+):
+    from dvc.utils.diff import format_dict
+    from dvc.utils.flatten import flatten
+
+    td = TabularData(["Revision", "Path"], fill_value="-")
+
+    for branch, val in metrics.items():
+        for fname, metric in val.get("data", {}).items():
+            row_data: Dict[str, str] = {"Revision": branch, "Path": fname}
+            metric = metric.get("data", {})
+            flattened = (
+                flatten(format_dict(metric))
+                if isinstance(metric, dict)
+                else {"": metric}
+            )
+            row_data.update(
+                {
+                    k: _format_field(v, precision, round_digits)
+                    for k, v in flattened.items()
+                }
+            )
+            td.row_from_dict(row_data)
+
+    rev, path, *metrics_headers = td.keys()
+    td.project(rev, path, *sorted(metrics_headers))
+
+    if not any([all_branches, all_tags, all_commits]):
+        td.drop("Revision")
+
+    return td
+
+
+def show_metrics(
+    metrics,
+    markdown: bool = False,
+    all_branches: bool = False,
+    all_tags: bool = False,
+    all_commits: bool = False,
+    precision: int = None,
+    round_digits: bool = False,
+) -> None:
+    td = metrics_table(
+        metrics,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        all_commits=all_commits,
+        precision=precision,
+        round_digits=round_digits,
+    )
+    td.render(markdown=markdown)

dvc/config.py

@@ -4,25 +4,11 @@ import os
 import re
 from contextlib import contextmanager
 from functools import partial
-from urllib.parse import urlparse
 
-import configobj
-from funcy import cached_property, compact, re_find, walk_values
-from voluptuous import (
-    ALLOW_EXTRA,
-    All,
-    Any,
-    Coerce,
-    Invalid,
-    Lower,
-    Optional,
-    Range,
-    Schema,
-)
+from funcy import cached_property, compact, memoize, re_find
 
 from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.path_info import PathInfo
-from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
@@ -34,197 +20,43 @@ class ConfigError(DvcException):
         super().__init__(f"config file error: {msg}")
 
 
-class NoRemoteError(ConfigError):
+class RemoteConfigError(ConfigError):
     pass
 
 
-def supported_cache_type(types):
-    """Checks if link type config option consists only of valid values.
-
-    Args:
-        types (list/string): type(s) of links that dvc should try out.
-    """
-    if types is None:
-        return None
-    if isinstance(types, str):
-        types = [typ.strip() for typ in types.split(",")]
-
-    unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
-    if unsupported:
-        raise Invalid(
-            "Unsupported cache type(s): {}".format(", ".join(unsupported))
-        )
-
-    return types
-
-
-# Checks that value is either true or false and converts it to bool
-to_bool = Bool = All(
-    Lower,
-    Any("true", "false"),
-    lambda v: v == "true",
-    msg="expected true or false",
-)
-
-
-def Choices(*choices):
-    """Checks that value belongs to the specified set of values
-
-    Args:
-        *choices: pass allowed values as arguments, or pass a list or
-            tuple as a single argument
-    """
-    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
-
-
-def ByUrl(mapping):
-    schemas = walk_values(Schema, mapping)
-
-    def validate(data):
-        if "url" not in data:
-            raise Invalid("expected 'url'")
-
-        parsed = urlparse(data["url"])
-        # Windows absolute paths should really have scheme == "" (local)
-        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
-            return schemas[""](data)
-        if parsed.scheme not in schemas:
-            raise Invalid(f"Unsupported URL type {parsed.scheme}://")
-
-        return schemas[parsed.scheme](data)
-
-    return validate
-
-
-class RelPath(str):
+class NoRemoteError(RemoteConfigError):
     pass
 
 
-REMOTE_COMMON = {
-    "url": str,
-    "checksum_jobs": All(Coerce(int), Range(1)),
-    Optional("no_traverse"): Bool,  # obsoleted
-    "verify": Bool,
-}
-LOCAL_COMMON = {
-    "type": supported_cache_type,
-    Optional("protected", default=False): Bool,  # obsoleted
-    "shared": All(Lower, Choices("group")),
-    Optional("slow_link_warning", default=True): Bool,
-}
-HTTP_COMMON = {
-    "auth": All(Lower, Choices("basic", "digest", "custom")),
-    "custom_auth_header": str,
-    "user": str,
-    "password": str,
-    "ask_password": Bool,
-    "ssl_verify": Bool,
-    "method": str,
-}
-WEBDAV_COMMON = {
-    "user": str,
-    "password": str,
-    "ask_password": Bool,
-    "token": str,
-    "cert_path": str,
-    "key_path": str,
-    "timeout": Coerce(int),
-}
+class RemoteNotFoundError(RemoteConfigError):
+    pass
 
-SCHEMA = {
-    "core": {
-        "remote": Lower,
-        "checksum_jobs": All(Coerce(int), Range(1)),
-        Optional("interactive", default=False): Bool,
-        Optional("analytics", default=True): Bool,
-        Optional("hardlink_lock", default=False): Bool,
-        Optional("no_scm", default=False): Bool,
-        Optional("autostage", default=False): Bool,
-        Optional("experiments", default=False): Bool,
-        Optional("check_update", default=True): Bool,
-    },
-    "cache": {
-        "local": str,
-        "s3": str,
-        "gs": str,
-        "hdfs": str,
-        "ssh": str,
-        "azure": str,
-        # This is for default local cache
-        "dir": str,
-        **LOCAL_COMMON,
-    },
-    "remote": {
-        str: ByUrl(
-            {
-                "": {**LOCAL_COMMON, **REMOTE_COMMON},
-                "s3": {
-                    "region": str,
-                    "profile": str,
-                    "credentialpath": str,
-                    "endpointurl": str,
-                    "access_key_id": str,
-                    "secret_access_key": str,
-                    Optional("listobjects", default=False): Bool,  # obsoleted
-                    Optional("use_ssl", default=True): Bool,
-                    "sse": str,
-                    "sse_kms_key_id": str,
-                    "acl": str,
-                    "grant_read": str,
-                    "grant_read_acp": str,
-                    "grant_write_acp": str,
-                    "grant_full_control": str,
-                    **REMOTE_COMMON,
-                },
-                "gs": {
-                    "projectname": str,
-                    "credentialpath": str,
-                    **REMOTE_COMMON,
-                },
-                "ssh": {
-                    "type": supported_cache_type,
-                    "port": Coerce(int),
-                    "user": str,
-                    "password": str,
-                    "ask_password": Bool,
-                    "keyfile": str,
-                    "timeout": Coerce(int),
-                    "gss_auth": Bool,
-                    **REMOTE_COMMON,
-                },
-                "hdfs": {"user": str, **REMOTE_COMMON},
-                "azure": {"connection_string": str, **REMOTE_COMMON},
-                "oss": {
-                    "oss_key_id": str,
-                    "oss_key_secret": str,
-                    "oss_endpoint": str,
-                    **REMOTE_COMMON,
-                },
-                "gdrive": {
-                    "gdrive_use_service_account": Bool,
-                    "gdrive_client_id": str,
-                    "gdrive_client_secret": str,
-                    "gdrive_user_credentials_file": str,
-                    "gdrive_service_account_email": str,
-                    "gdrive_service_account_user_email": str,
-                    "gdrive_service_account_p12_file_path": str,
-                    Optional("gdrive_trash_only", default=False): Bool,
-                    **REMOTE_COMMON,
-                },
-                "http": {**HTTP_COMMON, **REMOTE_COMMON},
-                "https": {**HTTP_COMMON, **REMOTE_COMMON},
-                "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON},
-                "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON},
-                "remote": {str: object},  # Any of the above options are valid
-            }
-        )
-    },
-    "state": {
-        "row_limit": All(Coerce(int), Range(1)),
-        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),
-    },
-}
-COMPILED_SCHEMA = Schema(SCHEMA)
+
+class MachineConfigError(ConfigError):
+    pass
+
+
+class NoMachineError(MachineConfigError):
+    pass
+
+
+class MachineNotFoundError(MachineConfigError):
+    pass
+
+
+@memoize
+def get_compiled_schema():
+    from voluptuous import Schema
+
+    from .config_schema import SCHEMA
+
+    return Schema(SCHEMA)
+
+
+def to_bool(value):
+    from .config_schema import Bool
+
+    return Bool(value)
 
 
 class Config(dict):
@@ -251,9 +83,9 @@ class Config(dict):
     CONFIG_LOCAL = "config.local"
 
     def __init__(
-        self, dvc_dir=None, validate=True, tree=None,
+        self, dvc_dir=None, validate=True, fs=None, config=None
     ):  # pylint: disable=super-init-not-called
-        from dvc.tree.local import LocalTree
+        from dvc.fs.local import LocalFileSystem
 
         self.dvc_dir = dvc_dir
 
@@ -267,10 +99,10 @@ class Config(dict):
         else:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
 
-        self.wtree = LocalTree(None, {"url": self.dvc_dir})
-        self.tree = tree or self.wtree
+        self.wfs = LocalFileSystem(url=self.dvc_dir)
+        self.fs = fs or self.wfs
 
-        self.load(validate=validate)
+        self.load(validate=validate, config=config)
 
     @classmethod
     def get_dir(cls, level):
@@ -310,7 +142,7 @@ class Config(dict):
         open(config_file, "w+").close()
         return Config(dvc_dir)
 
-    def load(self, validate=True):
+    def load(self, validate=True, config=None):
         """Loads config from all the config files.
 
         Raises:
@@ -318,6 +150,9 @@ class Config(dict):
         """
         conf = self.load_config_to_level()
 
+        if config is not None:
+            merge(conf, config)
+
         if validate:
             conf = self.validate(conf)
 
@@ -328,32 +163,36 @@ class Config(dict):
         if not self["cache"].get("dir") and self.dvc_dir:
             self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
-    def _get_tree(self, level):
-        # NOTE: this might be a GitTree, which doesn't see things outside of
+    def _get_fs(self, level):
+        # NOTE: this might be a Gitfs, which doesn't see things outside of
         # the repo.
-        return self.tree if level == "repo" else self.wtree
+        return self.fs if level == "repo" else self.wfs
 
     def _load_config(self, level):
-        filename = self.files[level]
-        tree = self._get_tree(level)
+        from configobj import ConfigObj
 
-        if tree.exists(filename, use_dvcignore=False):
-            with tree.open(filename) as fobj:
-                conf_obj = configobj.ConfigObj(fobj)
+        filename = self.files[level]
+        fs = self._get_fs(level)
+
+        if fs.exists(filename):
+            with fs.open(filename) as fobj:
+                conf_obj = ConfigObj(fobj)
         else:
-            conf_obj = configobj.ConfigObj()
-        return _parse_remotes(_lower_keys(conf_obj.dict()))
+            conf_obj = ConfigObj()
+        return _parse_named(_lower_keys(conf_obj.dict()))
 
     def _save_config(self, level, conf_dict):
+        from configobj import ConfigObj
+
         filename = self.files[level]
-        tree = self._get_tree(level)
+        fs = self._get_fs(level)
 
         logger.debug(f"Writing '{filename}'.")
 
-        tree.makedirs(os.path.dirname(filename))
+        fs.makedirs(os.path.dirname(filename))
 
-        config = configobj.ConfigObj(_pack_remotes(conf_dict))
-        with tree.open(filename, "wb") as fobj:
+        config = ConfigObj(_pack_named(conf_dict))
+        with fs.open(filename, "wb") as fobj:
             config.write(fobj)
         config.filename = filename
 
@@ -362,7 +201,7 @@ class Config(dict):
         conf = self._load_paths(conf, self.files[level])
 
         # Auto-verify sections
-        for key in COMPILED_SCHEMA.schema:
+        for key in get_compiled_schema().schema:
             conf.setdefault(key, {})
 
         return conf
@@ -372,14 +211,26 @@ class Config(dict):
         abs_conf_dir = os.path.abspath(os.path.dirname(filename))
 
         def resolve(path):
+            from .config_schema import RelPath
+
             if os.path.isabs(path) or re.match(r"\w+://", path):
                 return path
+
+            # on windows convert slashes to backslashes
+            # to have path compatible with abs_conf_dir
+            if os.path.sep == "\\" and "/" in path:
+                path = path.replace("/", "\\")
+
             return RelPath(os.path.join(abs_conf_dir, path))
 
         return Config._map_dirs(conf, resolve)
 
     @staticmethod
     def _to_relpath(conf_dir, path):
+        from dvc.utils import relpath
+
+        from .config_schema import RelPath
+
         if re.match(r"\w+://", path):
             return path
 
@@ -397,11 +248,22 @@ class Config(dict):
 
     @staticmethod
     def _map_dirs(conf, func):
+        from voluptuous import ALLOW_EXTRA, Schema
+
         dirs_schema = {
             "cache": {"dir": func},
             "remote": {
-                str: {"url": func, "gdrive_user_credentials_file": func}
+                str: {
+                    "url": func,
+                    "gdrive_user_credentials_file": func,
+                    "gdrive_service_account_json_file_path": func,
+                    "credentialpath": func,
+                    "keyfile": func,
+                    "cert_path": func,
+                    "key_path": func,
+                }
             },
+            "machine": {str: {"startup_script": func}},
         }
         return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
 
@@ -414,8 +276,16 @@ class Config(dict):
                 merge(merged_conf, self.load_one(merge_level))
         return merged_conf
 
+    def read(self, level=None):
+        # NOTE: we read from a merged config by default, same as git config
+        if level is None:
+            return self.load_config_to_level()
+        return self.load_one(level)
+
     @contextmanager
-    def edit(self, level="repo"):
+    def edit(self, level=None, validate=True):
+        # NOTE: we write to repo config by default, same as git config
+        level = level or "repo"
         if level in {"repo", "local"} and self.dvc_dir is None:
             raise ConfigError("Not inside a DVC repo")
 
@@ -426,40 +296,46 @@ class Config(dict):
 
         merged_conf = self.load_config_to_level(level)
         merge(merged_conf, conf)
-        self.validate(merged_conf)
+
+        if validate:
+            self.validate(merged_conf)
 
         self._save_config(level, conf)
-        self.load()
+        self.load(validate=validate)
 
     @staticmethod
     def validate(data):
+        from voluptuous import Invalid
+
         try:
-            return COMPILED_SCHEMA(data)
+            return get_compiled_schema()(data)
         except Invalid as exc:
             raise ConfigError(str(exc)) from None
 
 
-def _parse_remotes(conf):
-    result = {"remote": {}}
+def _parse_named(conf):
+    result = {"remote": {}, "machine": {}}
 
     for section, val in conf.items():
-        name = re_find(r'^\s*remote\s*"(.*)"\s*$', section)
-        if name:
-            result["remote"][name] = val
+        match = re_find(r'^\s*(remote|machine)\s*"(.*)"\s*$', section)
+        if match:
+            key, name = match
+            result[key][name] = val
         else:
             result[section] = val
 
     return result
 
 
-def _pack_remotes(conf):
+def _pack_named(conf):
     # Drop empty sections
     result = compact(conf)
 
     # Transform remote.name -> 'remote "name"'
-    for name, val in conf["remote"].items():
-        result[f'remote "{name}"'] = val
-    result.pop("remote", None)
+    for key in ("remote", "machine"):
+        for name, val in conf[key].items():
+            result[f'{key} "{name}"'] = val
+        result.pop(key, None)
 
     return result
 

dvc/config_schema.py

@@ -0,0 +1,243 @@
+import os
+from urllib.parse import urlparse
+
+from funcy import walk_values
+from voluptuous import (
+    All,
+    Any,
+    Coerce,
+    Invalid,
+    Lower,
+    Optional,
+    Range,
+    Schema,
+)
+
+Bool = All(
+    Lower,
+    Any("true", "false"),
+    lambda v: v == "true",
+    msg="expected true or false",
+)
+
+
+def supported_cache_type(types):
+    """Checks if link type config option consists only of valid values.
+
+    Args:
+        types (list/string): type(s) of links that dvc should try out.
+    """
+    if types is None:
+        return None
+    if isinstance(types, str):
+        types = [typ.strip() for typ in types.split(",")]
+
+    unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"}
+    if unsupported:
+        raise Invalid(
+            "Unsupported cache type(s): {}".format(", ".join(unsupported))
+        )
+
+    return types
+
+
+def Choices(*choices):
+    """Checks that value belongs to the specified set of values
+
+    Args:
+        *choices: pass allowed values as arguments, or pass a list or
+            tuple as a single argument
+    """
+    return Any(*choices, msg="expected one of {}".format(", ".join(choices)))
+
+
+def ByUrl(mapping):
+    schemas = walk_values(Schema, mapping)
+
+    def validate(data):
+        if "url" not in data:
+            raise Invalid("expected 'url'")
+
+        parsed = urlparse(data["url"])
+        # Windows absolute paths should really have scheme == "" (local)
+        if os.name == "nt" and len(parsed.scheme) == 1 and parsed.netloc == "":
+            return schemas[""](data)
+        if parsed.scheme not in schemas:
+            raise Invalid(f"Unsupported URL type {parsed.scheme}://")
+
+        return schemas[parsed.scheme](data)
+
+    return validate
+
+
+class RelPath(str):
+    pass
+
+
+REMOTE_COMMON = {
+    "url": str,
+    "checksum_jobs": All(Coerce(int), Range(1)),
+    "jobs": All(Coerce(int), Range(1)),
+    Optional("no_traverse"): Bool,  # obsoleted
+    "verify": Bool,
+}
+LOCAL_COMMON = {
+    "type": supported_cache_type,
+    Optional("protected", default=False): Bool,  # obsoleted
+    "shared": All(Lower, Choices("group")),
+    Optional("slow_link_warning", default=True): Bool,
+}
+HTTP_COMMON = {
+    "auth": All(Lower, Choices("basic", "digest", "custom")),
+    "custom_auth_header": str,
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "ssl_verify": Any(Bool, str),
+    "method": str,
+}
+WEBDAV_COMMON = {
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "token": str,
+    "cert_path": str,
+    "key_path": str,
+    "timeout": Coerce(int),
+    "ssl_verify": Any(Bool, str),
+}
+
+SCHEMA = {
+    "core": {
+        "remote": Lower,
+        "checksum_jobs": All(Coerce(int), Range(1)),
+        Optional("interactive", default=False): Bool,
+        Optional("analytics", default=True): Bool,
+        Optional("hardlink_lock", default=False): Bool,
+        Optional("no_scm", default=False): Bool,
+        Optional("autostage", default=False): Bool,
+        Optional("experiments"): Bool,  # obsoleted
+        Optional("check_update", default=True): Bool,
+        "machine": Lower,
+    },
+    "cache": {
+        "local": str,
+        "s3": str,
+        "gs": str,
+        "hdfs": str,
+        "webhdfs": str,
+        "ssh": str,
+        "azure": str,
+        # This is for default local cache
+        "dir": str,
+        **LOCAL_COMMON,
+    },
+    "remote": {
+        str: ByUrl(
+            {
+                "": {**LOCAL_COMMON, **REMOTE_COMMON},
+                "s3": {
+                    "region": str,
+                    "profile": str,
+                    "credentialpath": str,
+                    "configpath": str,
+                    "endpointurl": str,
+                    "access_key_id": str,
+                    "secret_access_key": str,
+                    "session_token": str,
+                    Optional("listobjects", default=False): Bool,  # obsoleted
+                    Optional("use_ssl", default=True): Bool,
+                    "ssl_verify": Any(Bool, str),
+                    "sse": str,
+                    "sse_kms_key_id": str,
+                    "acl": str,
+                    "grant_read": str,
+                    "grant_read_acp": str,
+                    "grant_write_acp": str,
+                    "grant_full_control": str,
+                    "cache_regions": bool,
+                    **REMOTE_COMMON,
+                },
+                "gs": {
+                    "projectname": str,
+                    "credentialpath": str,
+                    **REMOTE_COMMON,
+                },
+                "ssh": {
+                    "type": supported_cache_type,
+                    "port": Coerce(int),
+                    "user": str,
+                    "password": str,
+                    "ask_password": Bool,
+                    "keyfile": str,
+                    "timeout": Coerce(int),
+                    "gss_auth": Bool,
+                    "allow_agent": Bool,
+                    **REMOTE_COMMON,
+                },
+                "hdfs": {"user": str, "kerb_ticket": str, **REMOTE_COMMON},
+                "webhdfs": {
+                    "hdfscli_config": str,
+                    "webhdfs_token": str,
+                    "user": str,
+                    "webhdfs_alias": str,
+                    **REMOTE_COMMON,
+                },
+                "azure": {
+                    "connection_string": str,
+                    "sas_token": str,
+                    "account_name": str,
+                    "account_key": str,
+                    "tenant_id": str,
+                    "client_id": str,
+                    "client_secret": str,
+                    "allow_anonymous_login": Bool,
+                    "exclude_environment_credential": Bool,
+                    "exclude_visual_studio_code_credential": Bool,
+                    "exclude_shared_token_cache_credential": Bool,
+                    "exclude_managed_identity_credential": Bool,
+                    **REMOTE_COMMON,
+                },
+                "oss": {
+                    "oss_key_id": str,
+                    "oss_key_secret": str,
+                    "oss_endpoint": str,
+                    **REMOTE_COMMON,
+                },
+                "gdrive": {
+                    "gdrive_use_service_account": Bool,
+                    "gdrive_client_id": str,
+                    "gdrive_client_secret": str,
+                    "gdrive_user_credentials_file": str,
+                    "gdrive_service_account_user_email": str,
+                    "gdrive_service_account_json_file_path": str,
+                    Optional("gdrive_trash_only", default=False): Bool,
+                    **REMOTE_COMMON,
+                },
+                "http": {**HTTP_COMMON, **REMOTE_COMMON},
+                "https": {**HTTP_COMMON, **REMOTE_COMMON},
+                "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "remote": {str: object},  # Any of the above options are valid
+            }
+        )
+    },
+    "state": {
+        "row_limit": All(Coerce(int), Range(1)),  # obsoleted
+        "row_cleanup_quota": All(Coerce(int), Range(0, 100)),  # obsoleted
+    },
+    "machine": {
+        str: {
+            "cloud": All(Lower, Choices("aws", "azure")),
+            "instance_type": Lower,
+            "startup_script": str,
+        },
+    },
+    # section for experimental features
+    "feature": {
+        Optional("machine", default=False): Bool,
+        # enabled by default. It's of no use, kept for backward compatibility.
+        Optional("parametrization", default=True): Bool,
+    },
+    "plots": {"html_template": str},
+}

dvc/daemon.py

@@ -12,29 +12,34 @@ from dvc.utils import fix_env, is_binary
 
 logger = logging.getLogger(__name__)
 
-CREATE_NEW_PROCESS_GROUP = 0x00000200
-DETACHED_PROCESS = 0x00000008
+CREATE_NO_WINDOW = 0x08000000  # only available since Python 3.7 in subprocess.
 
 
 def _popen(cmd, **kwargs):
     prefix = [sys.executable]
     if not is_binary():
-        prefix += [sys.argv[0]]
-
+        main_entrypoint = os.path.join(
+            os.path.abspath(os.path.dirname(__file__)), "__main__.py"
+        )
+        prefix += [main_entrypoint]
     return Popen(prefix + cmd, close_fds=True, shell=False, **kwargs)
 
 
 def _spawn_windows(cmd, env):
-    from subprocess import STARTF_USESHOWWINDOW, STARTUPINFO
+    from subprocess import (
+        CREATE_NEW_PROCESS_GROUP,
+        STARTF_USESHOWWINDOW,
+        STARTUPINFO,
+    )
 
-    creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
+    # https://stackoverflow.com/a/7006424
+    # https://bugs.python.org/issue41619
+    creationflags = CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW
 
     startupinfo = STARTUPINFO()
     startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 
-    _popen(
-        cmd, env=env, creationflags=creationflags, startupinfo=startupinfo,
-    ).communicate()
+    _popen(cmd, env=env, creationflags=creationflags, startupinfo=startupinfo)
 
 
 def _spawn_posix(cmd, env):

dvc/dagascii.py

@@ -3,52 +3,14 @@
 import logging
 import math
 import os
-import pydoc
-import sys
 
 from grandalf.graphs import Edge, Graph, Vertex
 from grandalf.layouts import SugiyamaLayout
 from grandalf.routing import EdgeViewer, route_with_lines
 
-from dvc.env import DVC_PAGER
-from dvc.utils import format_link
-
 logger = logging.getLogger(__name__)
 
 
-DEFAULT_PAGER = "less"
-DEFAULT_PAGER_FORMATTED = "{} --chop-long-lines --clear-screen".format(
-    DEFAULT_PAGER
-)
-
-
-def make_pager(cmd):
-    def pager(text):
-        return pydoc.tempfilepager(pydoc.plain(text), cmd)
-
-    return pager
-
-
-def find_pager():
-    if not sys.stdout.isatty():
-        return pydoc.plainpager
-
-    env_pager = os.getenv(DVC_PAGER)
-    if env_pager:
-        return make_pager(env_pager)
-
-    if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
-        return make_pager(DEFAULT_PAGER_FORMATTED)
-
-    logger.warning(
-        "Unable to find `less` in the PATH. Check out "
-        "{} for more info.".format(
-            format_link("https://man.dvc.org/pipeline/show")
-        )
-    )
-    return pydoc.plainpager
-
-
 class VertexViewer:
     """Class to define vertex box boundaries that will be accounted for during
     graph building by grandalf.
@@ -201,7 +163,7 @@ class AsciiCanvas:
         self.point(x0 + width, y0 + height, "+")
 
 
-def _build_sugiyama_layout(vertexes, edges):
+def _build_sugiyama_layout(vertices, edges):
     #
     # Just a reminder about naming conventions:
     # +------------X
@@ -212,17 +174,17 @@ def _build_sugiyama_layout(vertexes, edges):
     # Y
     #
 
-    vertexes = {v: Vertex(f" {v} ") for v in vertexes}
+    vertices = {v: Vertex(f" {v} ") for v in vertices}
     # NOTE: reverting edges to correctly orientate the graph
-    edges = [Edge(vertexes[e], vertexes[s]) for s, e in edges]
-    vertexes = vertexes.values()
-    graph = Graph(vertexes, edges)
+    edges = [Edge(vertices[e], vertices[s]) for s, e in edges]
+    vertices = vertices.values()
+    graph = Graph(vertices, edges)
 
-    for vertex in vertexes:
+    for vertex in vertices:
         vertex.view = VertexViewer(vertex.data)
 
     # NOTE: determine min box length to create the best layout
-    minw = min(v.view.w for v in vertexes)
+    minw = min(v.view.w for v in vertices)
 
     for edge in edges:
         edge.view = EdgeViewer()
@@ -242,11 +204,11 @@ def _build_sugiyama_layout(vertexes, edges):
     return sug
 
 
-def draw(vertexes, edges):
+def draw(vertices, edges):
     """Build a DAG and draw it in ASCII.
 
     Args:
-        vertexes (list): list of graph vertexes.
+        vertices (list): list of graph vertices.
         edges (list): list of graph edges.
     """
     # pylint: disable=too-many-locals
@@ -255,7 +217,7 @@ def draw(vertexes, edges):
     Xs = []  # pylint: disable=invalid-name
     Ys = []  # pylint: disable=invalid-name
 
-    sug = _build_sugiyama_layout(vertexes, edges)
+    sug = _build_sugiyama_layout(vertices, edges)
 
     for vertex in sug.g.sV:
         # NOTE: moving boxes w/2 to the left

dvc/data_cloud.py

@@ -1,9 +1,13 @@
 """Manages dvc remotes that user can use with push/pull/status commands."""
 
 import logging
+from typing import TYPE_CHECKING, Iterable, Optional
 
-from dvc.config import NoRemoteError
-from dvc.remote import get_remote
+from dvc.objects.db import get_index
+
+if TYPE_CHECKING:
+    from dvc.hash_info import HashInfo
+    from dvc.objects.db.base import ObjectDB
 
 logger = logging.getLogger(__name__)
 
@@ -22,12 +26,18 @@ class DataCloud:
     def __init__(self, repo):
         self.repo = repo
 
-    def get_remote(self, name=None, command="<command>"):
+    def get_remote_odb(
+        self,
+        name: Optional[str] = None,
+        command: str = "<command>",
+    ) -> "ObjectDB":
+        from dvc.config import NoRemoteError
+
         if not name:
             name = self.repo.config["core"].get("remote")
 
         if name:
-            return self._init_remote(name)
+            return self._init_odb(name)
 
         if bool(self.repo.config["remote"]):
             error_msg = (
@@ -44,77 +54,107 @@ class DataCloud:
 
         raise NoRemoteError(error_msg)
 
-    def _init_remote(self, name):
-        return get_remote(self.repo, name=name)
+    def _init_odb(self, name):
+        from dvc.fs import get_cloud_fs
+        from dvc.objects.db import get_odb
+
+        cls, config, path_info = get_cloud_fs(self.repo, name=name)
+        config["tmp_dir"] = self.repo.tmp_dir
+        return get_odb(cls(**config), path_info, **config)
 
     def push(
-        self, cache, jobs=None, remote=None, show_checksums=False,
+        self,
+        objs: Iterable["HashInfo"],
+        jobs: Optional[int] = None,
+        remote: Optional[str] = None,
+        odb: Optional["ObjectDB"] = None,
     ):
         """Push data items in a cloud-agnostic way.
 
         Args:
-            cache (NamedCache): named checksums to push to the cloud.
-            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.BaseRemote): optional remote to push to.
+            objs: objects to push to the cloud.
+            jobs: number of jobs that can be running simultaneously.
+            remote: optional name of remote to push to.
                 By default remote from core.remote config option is used.
-            show_checksums (bool): show checksums instead of file names in
-                information messages.
+            odb: optional ODB to push to. Overrides remote.
         """
-        remote = self.get_remote(remote, "push")
+        from dvc.objects.transfer import transfer
 
-        return self.repo.cache.local.push(
-            cache, jobs=jobs, remote=remote, show_checksums=show_checksums,
+        if not odb:
+            odb = self.get_remote_odb(remote, "push")
+        return transfer(
+            self.repo.odb.local,
+            odb,
+            objs,
+            jobs=jobs,
+            dest_index=get_index(odb),
+            cache_odb=self.repo.odb.local,
         )
 
     def pull(
-        self, cache, jobs=None, remote=None, show_checksums=False,
+        self,
+        objs: Iterable["HashInfo"],
+        jobs: Optional[int] = None,
+        remote: Optional[str] = None,
+        odb: Optional["ObjectDB"] = None,
     ):
         """Pull data items in a cloud-agnostic way.
 
         Args:
-            cache (NamedCache): named checksums to pull from the cloud.
-            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.BaseRemote): optional remote to pull from.
+            objs: objects to pull from the cloud.
+            jobs: number of jobs that can be running simultaneously.
+            remote: optional name of remote to pull from.
                 By default remote from core.remote config option is used.
-            show_checksums (bool): show checksums instead of file names in
-                information messages.
+            odb: optional ODB to pull from. Overrides remote.
         """
-        remote = self.get_remote(remote, "pull")
+        from dvc.objects.transfer import transfer
 
-        return self.repo.cache.local.pull(
-            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
+        if not odb:
+            odb = self.get_remote_odb(remote, "pull")
+        return transfer(
+            odb,
+            self.repo.odb.local,
+            objs,
+            jobs=jobs,
+            src_index=get_index(odb),
+            cache_odb=self.repo.odb.local,
+            verify=odb.verify,
         )
 
     def status(
         self,
-        cache,
-        jobs=None,
-        remote=None,
-        show_checksums=False,
-        log_missing=True,
+        objs: Iterable["HashInfo"],
+        jobs: Optional[int] = None,
+        remote: Optional[str] = None,
+        odb: Optional["ObjectDB"] = None,
+        log_missing: bool = True,
     ):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
-            cache (NamedCache): named checksums to check status for.
-            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.BaseRemote): optional remote to compare
+            objs: objects to check status for.
+            jobs: number of jobs that can be running simultaneously.
+            remote: optional remote to compare
                 cache to. By default remote from core.remote config option
                 is used.
-            show_checksums (bool): show checksums instead of file names in
-                information messages.
-            log_missing (bool): log warning messages if file doesn't exist
+            odb: optional ODB to check status from. Overrides remote.
+            log_missing: log warning messages if file doesn't exist
                 neither in cache, neither in cloud.
         """
-        remote = self.get_remote(remote, "status")
-        return self.repo.cache.local.status(
-            cache,
+        from dvc.objects.status import compare_status
+
+        if not odb:
+            odb = self.get_remote_odb(remote, "status")
+        return compare_status(
+            self.repo.odb.local,
+            odb,
+            objs,
             jobs=jobs,
-            remote=remote,
-            show_checksums=show_checksums,
             log_missing=log_missing,
+            dest_index=get_index(odb),
+            cache_odb=self.repo.odb.local,
         )
 
     def get_url_for(self, remote, checksum):
-        remote = self.get_remote(remote)
-        return str(remote.tree.hash_to_path_info(checksum))
+        remote_odb = self.get_remote_odb(remote)
+        return str(remote_odb.hash_to_path_info(checksum))

dvc/dependency/__init__.py

@@ -1,62 +1,24 @@
 from collections import defaultdict
-from urllib.parse import urlparse
+from typing import Any, Mapping
 
-import dvc.output as output
-from dvc.dependency.azure import AzureDependency
-from dvc.dependency.gs import GSDependency
-from dvc.dependency.hdfs import HDFSDependency
-from dvc.dependency.http import HTTPDependency
-from dvc.dependency.https import HTTPSDependency
-from dvc.dependency.local import LocalDependency
-from dvc.dependency.param import ParamsDependency
-from dvc.dependency.s3 import S3Dependency
-from dvc.dependency.ssh import SSHDependency
-from dvc.output.base import BaseOutput
-from dvc.scheme import Schemes
+from dvc.output import ARTIFACT_SCHEMA, Output
 
-from ..tree import get_cloud_tree
+from .base import Dependency
+from .param import ParamsDependency
 from .repo import RepoDependency
 
-DEPS = [
-    AzureDependency,
-    GSDependency,
-    HDFSDependency,
-    HTTPDependency,
-    HTTPSDependency,
-    S3Dependency,
-    SSHDependency,
-    # NOTE: LocalDependency is the default choice
-]
-
-DEP_MAP = {
-    Schemes.LOCAL: LocalDependency,
-    Schemes.SSH: SSHDependency,
-    Schemes.S3: S3Dependency,
-    Schemes.AZURE: AzureDependency,
-    Schemes.GS: GSDependency,
-    Schemes.HDFS: HDFSDependency,
-    Schemes.HTTP: HTTPDependency,
-    Schemes.HTTPS: HTTPSDependency,
-}
-
-
 # NOTE: schema for dependencies is basically the same as for outputs, but
 # without output-specific entries like 'cache' (whether or not output is
 # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
-# output is a metric file and how to parse it, see `-M` flag for `dvc run`).
-SCHEMA = output.SCHEMA.copy()
-del SCHEMA[BaseOutput.PARAM_CACHE]
-del SCHEMA[BaseOutput.PARAM_METRIC]
-SCHEMA.update(RepoDependency.REPO_SCHEMA)
-SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
+# output is a metrics file and how to parse it, see `-M` flag for `dvc run`).
+SCHEMA: Mapping[str, Any] = {
+    **ARTIFACT_SCHEMA,
+    **RepoDependency.REPO_SCHEMA,
+    **ParamsDependency.PARAM_SCHEMA,
+}
 
 
 def _get(stage, p, info):
-    parsed = urlparse(p) if p else None
-    if parsed and parsed.scheme == "remote":
-        tree = get_cloud_tree(stage.repo, name=parsed.netloc)
-        return DEP_MAP[tree.scheme](stage, p, info, tree=tree)
-
     if info and info.get(RepoDependency.PARAM_REPO):
         repo = info.pop(RepoDependency.PARAM_REPO)
         return RepoDependency(repo, stage, p, info)
@@ -65,16 +27,13 @@ def _get(stage, p, info):
         params = info.pop(ParamsDependency.PARAM_PARAMS)
         return ParamsDependency(stage, p, params)
 
-    for d in DEPS:
-        if d.supported(p):
-            return d(stage, p, info)
-    return LocalDependency(stage, p, info)
+    return Dependency(stage, p, info)
 
 
 def loadd_from(stage, d_list):
     ret = []
     for d in d_list:
-        p = d.pop(BaseOutput.PARAM_PATH, None)
+        p = d.pop(Output.PARAM_PATH, None)
         ret.append(_get(stage, p, d))
     return ret
 

dvc/dependency/azure.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..tree.azure import AzureTree
-
-
-class AzureDependency(BaseDependency, BaseOutput):
-    TREE_CLS = AzureTree

dvc/dependency/base.py

@@ -1,4 +1,7 @@
+from typing import Type
+
 from dvc.exceptions import DvcException
+from dvc.output import Output
 
 
 class DependencyDoesNotExistError(DvcException):
@@ -15,15 +18,17 @@ class DependencyIsNotFileOrDirError(DvcException):
 
 class DependencyIsStageFileError(DvcException):
     def __init__(self, path):
-        super().__init__(f"Stage file '{path}' cannot be a dependency.")
+        super().__init__(f"DVC file '{path}' cannot be a dependency.")
 
 
-class BaseDependency:
+class Dependency(Output):
     IS_DEPENDENCY = True
 
-    DoesNotExistError = DependencyDoesNotExistError
-    IsNotFileOrDirError = DependencyIsNotFileOrDirError
-    IsStageFileError = DependencyIsStageFileError
+    DoesNotExistError = DependencyDoesNotExistError  # type: Type[DvcException]
+    IsNotFileOrDirError = (
+        DependencyIsNotFileOrDirError
+    )  # type: Type[DvcException]
+    IsStageFileError = DependencyIsStageFileError  # type: Type[DvcException]
 
     def update(self, rev=None):
         pass

dvc/dependency/gs.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.gs import GSOutput
-
-
-class GSDependency(BaseDependency, GSOutput):
-    pass

dvc/dependency/hdfs.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.hdfs import HDFSOutput
-
-
-class HDFSDependency(BaseDependency, HDFSOutput):
-    pass

dvc/dependency/http.py

@@ -1,8 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.base import BaseOutput
-
-from ..tree.http import HTTPTree
-
-
-class HTTPDependency(BaseDependency, BaseOutput):
-    TREE_CLS = HTTPTree

dvc/dependency/https.py

@@ -1,6 +0,0 @@
-from ..tree.https import HTTPSTree
-from .http import HTTPDependency
-
-
-class HTTPSDependency(HTTPDependency):
-    TREE_CLS = HTTPSTree

dvc/dependency/local.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.local import LocalOutput
-
-
-class LocalDependency(BaseDependency, LocalOutput):
-    pass

dvc/dependency/param.py

@@ -4,11 +4,12 @@ from collections import defaultdict
 import dpath.util
 from voluptuous import Any
 
-from dvc.dependency.local import LocalDependency
 from dvc.exceptions import DvcException
 from dvc.hash_info import HashInfo
 from dvc.utils.serialize import LOADERS, ParseError
 
+from .base import Dependency
+
 
 class MissingParamsError(DvcException):
     pass
@@ -18,7 +19,7 @@ class BadParamFileError(DvcException):
     pass
 
 
-class ParamsDependency(LocalDependency):
+class ParamsDependency(Dependency):
     PARAM_PARAMS = "params"
     PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
     DEFAULT_PARAMS_FILE = "params.yaml"
@@ -84,19 +85,34 @@ class ParamsDependency(LocalDependency):
     def status(self):
         return self.workspace_status()
 
-    def read_params(self):
+    def _read(self):
         if not self.exists:
             return {}
 
         suffix = self.path_info.suffix.lower()
         loader = LOADERS[suffix]
         try:
-            config = loader(self.path_info, tree=self.repo.tree)
+            return loader(self.path_info, fs=self.repo.fs)
         except ParseError as exc:
             raise BadParamFileError(
                 f"Unable to read parameters from '{self}'"
             ) from exc
 
+    def read_params_d(self, **kwargs):
+        config = self._read()
+
+        ret = {}
+        for param in self.params:
+            dpath.util.merge(
+                ret,
+                dpath.util.search(config, param, separator="."),
+                separator=".",
+            )
+        return ret
+
+    def read_params(self):
+        config = self._read()
+
         ret = {}
         for param in self.params:
             try:
@@ -112,7 +128,7 @@ class ParamsDependency(LocalDependency):
         if missing_params:
             raise MissingParamsError(
                 "Parameters '{}' are missing from '{}'.".format(
-                    ", ".join(missing_params), self,
+                    ", ".join(missing_params), self
                 )
             )
 

dvc/dependency/repo.py

@@ -1,9 +1,21 @@
+import os
+from collections import defaultdict
+from copy import copy
+from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple
+
 from voluptuous import Required
 
-from .local import LocalDependency
+from dvc.path_info import PathInfo
+
+from .base import Dependency
+
+if TYPE_CHECKING:
+    from dvc.hash_info import HashInfo
+    from dvc.objects.db.base import ObjectDB
+    from dvc.objects.file import HashFile
 
 
-class RepoDependency(LocalDependency):
+class RepoDependency(Dependency):
     PARAM_REPO = "repo"
     PARAM_URL = "url"
     PARAM_REV = "rev"
@@ -19,40 +31,22 @@ class RepoDependency(LocalDependency):
 
     def __init__(self, def_repo, stage, *args, **kwargs):
         self.def_repo = def_repo
+        self._staged_objs: Dict[str, "HashFile"] = {}
         super().__init__(stage, *args, **kwargs)
 
-    def _parse_path(self, tree, path):
+    def _parse_path(self, fs, path_info):
         return None
 
     @property
     def is_in_repo(self):
         return False
 
-    @property
-    def repo_pair(self):
-        d = self.def_repo
-        rev = d.get(self.PARAM_REV_LOCK) or d.get(self.PARAM_REV)
-        return d[self.PARAM_URL], rev
-
     def __str__(self):
-        return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
-
-    def _make_repo(self, *, locked=True, **kwargs):
-        from dvc.external_repo import external_repo
-
-        d = self.def_repo
-        rev = (d.get("rev_lock") if locked else None) or d.get("rev")
-        return external_repo(d["url"], rev=rev, **kwargs)
-
-    def _get_hash(self, locked=True):
-        # we want stream but not fetch, so DVC out directories are
-        # walked, but dir contents is not fetched
-        with self._make_repo(locked=locked, fetch=False, stream=True) as repo:
-            return repo.get_checksum(self.def_path)
+        return f"{self.def_path} ({self.def_repo[self.PARAM_URL]})"
 
     def workspace_status(self):
-        current = self._get_hash(locked=True)
-        updated = self._get_hash(locked=False)
+        current = self.get_obj(locked=True).hash_info
+        updated = self.get_obj(locked=False).hash_info
 
         if current != updated:
             return {str(self): "update available"}
@@ -68,21 +62,25 @@ class RepoDependency(LocalDependency):
     def dumpd(self):
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
-    def download(self, to):
-        cache = self.repo.cache.local
+    def download(self, to, jobs=None):
+        from dvc.checkout import checkout
 
-        with self._make_repo(cache_dir=cache.cache_dir) as repo:
-            if self.def_repo.get(self.PARAM_REV_LOCK) is None:
-                self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
+        for odb, objs in self.get_used_objs().items():
+            self.repo.cloud.pull(objs, jobs=jobs, odb=odb)
 
-            _, _, cache_infos = repo.fetch_external([self.def_path])
-
-        cache.checkout(to.path_info, cache_infos[0])
+        obj = self.get_obj()
+        checkout(
+            to.path_info,
+            to.fs,
+            obj,
+            self.repo.odb.local,
+            dvcignore=None,
+            state=self.repo.state,
+        )
 
     def update(self, rev=None):
         if rev:
             self.def_repo[self.PARAM_REV] = rev
-
         with self._make_repo(locked=False) as repo:
             self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
 
@@ -91,3 +89,124 @@ class RepoDependency(LocalDependency):
         # origin project url and rev_lock, and it makes RepoDependency
         # immutable, hence its impossible for checksum to change.
         return False
+
+    def get_used_objs(
+        self, **kwargs
+    ) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]:
+        used, _ = self._get_used_and_obj(**kwargs)
+        return used
+
+    def _get_used_and_obj(
+        self, obj_only=False, **kwargs
+    ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "HashFile"]:
+        from dvc.config import NoRemoteError
+        from dvc.exceptions import NoOutputOrStageError, PathMissingError
+        from dvc.objects.stage import stage
+        from dvc.objects.tree import Tree
+
+        local_odb = self.repo.odb.local
+        locked = kwargs.pop("locked", True)
+        with self._make_repo(
+            locked=locked, cache_dir=local_odb.cache_dir
+        ) as repo:
+            used_obj_ids = defaultdict(set)
+            rev = repo.get_rev()
+            if locked and self.def_repo.get(self.PARAM_REV_LOCK) is None:
+                self.def_repo[self.PARAM_REV_LOCK] = rev
+
+            path_info = PathInfo(repo.root_dir) / str(self.def_path)
+            if not obj_only:
+                try:
+                    for odb, obj_ids in repo.used_objs(
+                        [os.fspath(path_info)],
+                        force=True,
+                        jobs=kwargs.get("jobs"),
+                        recursive=True,
+                    ).items():
+                        if odb is None:
+                            odb = repo.cloud.get_remote_odb()
+                            odb.read_only = True
+                        self._check_circular_import(odb, obj_ids)
+                        used_obj_ids[odb].update(obj_ids)
+                except (NoRemoteError, NoOutputOrStageError):
+                    pass
+
+            try:
+                staging, staged_obj = stage(
+                    local_odb,
+                    path_info,
+                    repo.repo_fs,
+                    local_odb.fs.PARAM_CHECKSUM,
+                )
+            except FileNotFoundError as exc:
+                raise PathMissingError(
+                    self.def_path, self.def_repo[self.PARAM_URL]
+                ) from exc
+            staging = copy(staging)
+            staging.read_only = True
+
+            self._staged_objs[rev] = staged_obj
+            used_obj_ids[staging].add(staged_obj.hash_info)
+            if isinstance(staged_obj, Tree):
+                used_obj_ids[staging].update(
+                    entry.hash_info for _, entry in staged_obj
+                )
+            return used_obj_ids, staged_obj
+
+    def _check_circular_import(self, odb, obj_ids):
+        from dvc.exceptions import CircularImportError
+        from dvc.fs.repo import RepoFileSystem
+        from dvc.objects.db.reference import ReferenceObjectDB
+        from dvc.objects.tree import Tree
+
+        if not isinstance(odb, ReferenceObjectDB):
+            return
+
+        def iter_objs():
+            for hash_info in obj_ids:
+                if hash_info.isdir:
+                    tree = Tree.load(odb, hash_info)
+                    yield from (odb.get(entry.hash_info) for _, entry in tree)
+                else:
+                    yield odb.get(hash_info)
+
+        checked_urls = set()
+        for obj in iter_objs():
+            if not isinstance(obj.fs, RepoFileSystem):
+                continue
+            if (
+                obj.fs.repo_url in checked_urls
+                or obj.fs.root_dir in checked_urls
+            ):
+                continue
+            self_url = self.repo.url or self.repo.root_dir
+            if (
+                obj.fs.repo_url is not None
+                and obj.fs.repo_url == self_url
+                or obj.fs.root_dir == self.repo.root_dir
+            ):
+                raise CircularImportError(self, obj.fs.repo_url, self_url)
+            checked_urls.update([obj.fs.repo_url, obj.fs.root_dir])
+
+    def get_obj(self, filter_info=None, **kwargs):
+        locked = kwargs.get("locked", True)
+        rev = self._get_rev(locked=locked)
+        if rev in self._staged_objs:
+            return self._staged_objs[rev]
+        _, obj = self._get_used_and_obj(
+            obj_only=True, filter_info=filter_info, **kwargs
+        )
+        return obj
+
+    def _make_repo(self, locked=True, **kwargs):
+        from dvc.external_repo import external_repo
+
+        d = self.def_repo
+        rev = self._get_rev(locked=locked)
+        return external_repo(d[self.PARAM_URL], rev=rev, **kwargs)
+
+    def _get_rev(self, locked=True):
+        d = self.def_repo
+        return (d.get(self.PARAM_REV_LOCK) if locked else None) or d.get(
+            self.PARAM_REV
+        )

dvc/dependency/s3.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.s3 import S3Output
-
-
-class S3Dependency(BaseDependency, S3Output):
-    pass

dvc/dependency/ssh.py

@@ -1,6 +0,0 @@
-from dvc.dependency.base import BaseDependency
-from dvc.output.ssh import SSHOutput
-
-
-class SSHDependency(BaseDependency, SSHOutput):
-    pass

dvc/dvcfile.py

@@ -2,10 +2,12 @@ import collections
 import contextlib
 import logging
 import os
+from typing import TYPE_CHECKING, Any, Union
 
 from voluptuous import MultipleInvalid
 
 from dvc.exceptions import DvcException
+from dvc.parsing.versions import LOCKFILE_VERSION, SCHEMA_KWD
 from dvc.stage import serialize
 from dvc.stage.exceptions import (
     StageFileBadNameError,
@@ -13,17 +15,19 @@ from dvc.stage.exceptions import (
     StageFileFormatError,
     StageFileIsNotDvcFileError,
 )
-from dvc.stage.loader import SingleStageLoader, StageLoader
+from dvc.types import AnyPath
 from dvc.utils import relpath
 from dvc.utils.collections import apply_diff
 from dvc.utils.serialize import (
     dump_yaml,
-    load_yaml,
     modify_yaml,
     parse_yaml,
     parse_yaml_for_update,
 )
 
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
 logger = logging.getLogger(__name__)
 
 DVC_FILE = "Dvcfile"
@@ -32,10 +36,23 @@ PIPELINE_FILE = "dvc.yaml"
 PIPELINE_LOCK = "dvc.lock"
 
 
+class FileIsGitIgnored(DvcException):
+    def __init__(self, path, pipeline_file=False):
+        super().__init__(
+            "{}'{}' is git-ignored.".format(
+                "bad DVC file name " if pipeline_file else "", path
+            )
+        )
+
+
 class LockfileCorruptedError(DvcException):
     pass
 
 
+class ParametrizedDumpError(DvcException):
+    pass
+
+
 def is_valid_filename(path):
     return path.endswith(DVC_FILE_SUFFIX) or os.path.basename(path) in [
         DVC_FILE,
@@ -45,19 +62,38 @@ def is_valid_filename(path):
 
 def is_dvc_file(path):
     return os.path.isfile(path) and (
-        is_valid_filename(path) or os.path.basename(path) == PIPELINE_LOCK
+        is_valid_filename(path) or is_lock_file(path)
     )
 
 
-def check_dvc_filename(path):
+def is_lock_file(path):
+    return os.path.basename(path) == PIPELINE_LOCK
+
+
+def is_git_ignored(repo, path):
+    from dvc.fs.local import LocalFileSystem
+    from dvc.scm.base import NoSCMError
+
+    try:
+        return isinstance(repo.fs, LocalFileSystem) and repo.scm.is_ignored(
+            path
+        )
+    except NoSCMError:
+        return False
+
+
+def check_dvcfile_path(repo, path):
     if not is_valid_filename(path):
         raise StageFileBadNameError(
-            "bad DVC-file name '{}'. DVC-files should be named "
-            "'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
-                relpath(path), os.path.basename(path)
+            "bad DVC file name '{}'. DVC files should be named "
+            "'{}' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
+                relpath(path), PIPELINE_FILE, os.path.basename(path)
             )
         )
 
+    if is_git_ignored(repo, path):
+        raise FileIsGitIgnored(relpath(path), True)
+
 
 class FileMixin:
     SCHEMA = None
@@ -88,31 +124,47 @@ class FileMixin:
         return relpath(self.path)
 
     def exists(self):
-        return self.repo.tree.exists(self.path)
+        is_ignored = self.repo.dvcignore.is_ignored_file(self.path)
+        return self.repo.fs.exists(self.path) and not is_ignored
+
+    def _is_git_ignored(self):
+        return is_git_ignored(self.repo, self.path)
+
+    def _verify_filename(self):
+        if self.verify:
+            check_dvcfile_path(self.repo, self.path)
+
+    def _check_gitignored(self):
+        if self._is_git_ignored():
+            raise FileIsGitIgnored(self.path)
 
     def _load(self):
         # it raises the proper exceptions by priority:
         # 1. when the file doesn't exists
-        # 2. filename is not a DVC-file
+        # 2. filename is not a DVC file
         # 3. path doesn't represent a regular file
+        # 4. when the file is git ignored
         if not self.exists():
-            raise StageFileDoesNotExistError(self.path)
-        if self.verify:
-            check_dvc_filename(self.path)
-        if not self.repo.tree.isfile(self.path):
+            dvc_ignored = self.repo.dvcignore.is_ignored_file(self.path)
+            raise StageFileDoesNotExistError(
+                self.path, dvc_ignored=dvc_ignored
+            )
+
+        self._verify_filename()
+        if not self.repo.fs.isfile(self.path):
             raise StageFileIsNotDvcFileError(self.path)
 
-        with self.repo.tree.open(self.path) as fd:
+        self._check_gitignored()
+        with self.repo.fs.open(self.path, encoding="utf-8") as fd:
             stage_text = fd.read()
         d = parse_yaml(stage_text, self.path)
-        self.validate(d, self.relpath)
-        return d, stage_text
+        return self.validate(d, self.relpath), stage_text
 
     @classmethod
     def validate(cls, d, fname=None):
         assert isinstance(cls.SCHEMA, collections.abc.Callable)
         try:
-            cls.SCHEMA(d)  # pylint: disable=not-callable
+            return cls.SCHEMA(d)  # pylint: disable=not-callable
         except MultipleInvalid as exc:
             raise StageFileFormatError(f"'{fname}' format error: {exc}")
 
@@ -129,16 +181,17 @@ class FileMixin:
 
 class SingleStageFile(FileMixin):
     from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
+    from dvc.stage.loader import SingleStageLoader as LOADER
 
     @property
     def stage(self):
         data, raw = self._load()
-        return SingleStageLoader.load_stage(self, data, raw)
+        return self.LOADER.load_stage(self, data, raw)
 
     @property
     def stages(self):
         data, raw = self._load()
-        return SingleStageLoader(self, data, raw)
+        return self.LOADER(self, data, raw)
 
     def dump(self, stage, **kwargs):
         """Dumps given stage appropriately in the dvcfile."""
@@ -146,10 +199,8 @@ class SingleStageFile(FileMixin):
 
         assert not isinstance(stage, PipelineStage)
         if self.verify:
-            check_dvc_filename(self.path)
-        logger.debug(
-            "Saving information to '{file}'.".format(file=relpath(self.path))
-        )
+            check_dvcfile_path(self.repo, self.path)
+        logger.debug(f"Saving information to '{relpath(self.path)}'.")
         dump_yaml(self.path, serialize.to_single_stage_file(stage))
         self.repo.scm.track_file(self.relpath)
 
@@ -169,6 +220,7 @@ class PipelineFile(FileMixin):
     """Abstraction for pipelines file, .yaml + .lock combined."""
 
     from dvc.schema import COMPILED_MULTI_STAGE_SCHEMA as SCHEMA
+    from dvc.stage.loader import StageLoader as LOADER
 
     @property
     def _lockfile(self):
@@ -182,7 +234,7 @@ class PipelineFile(FileMixin):
 
         assert isinstance(stage, PipelineStage)
         if self.verify:
-            check_dvc_filename(self.path)
+            check_dvcfile_path(self.repo, self.path)
 
         if update_pipeline and not stage.is_data_source:
             self._dump_pipeline_file(stage)
@@ -193,10 +245,16 @@ class PipelineFile(FileMixin):
     def _dump_lockfile(self, stage):
         self._lockfile.dump(stage)
 
+    @staticmethod
+    def _check_if_parametrized(stage):
+        if stage.raw_data.parametrized:
+            raise ParametrizedDumpError(f"cannot dump a parametrized {stage}")
+
     def _dump_pipeline_file(self, stage):
+        self._check_if_parametrized(stage)
         stage_data = serialize.to_pipeline_file(stage)
 
-        with modify_yaml(self.path, tree=self.repo.tree) as data:
+        with modify_yaml(self.path, fs=self.repo.fs) as data:
             if not data:
                 logger.info("Creating '%s'", self.relpath)
 
@@ -209,8 +267,6 @@ class PipelineFile(FileMixin):
 
             if existing_entry:
                 orig_stage_data = data["stages"][stage.name]
-                if "meta" in orig_stage_data:
-                    stage_data[stage.name]["meta"] = orig_stage_data["meta"]
                 apply_diff(stage_data[stage.name], orig_stage_data)
             else:
                 data["stages"].update(stage_data)
@@ -227,7 +283,7 @@ class PipelineFile(FileMixin):
     def stages(self):
         data, _ = self._load()
         lockfile_data = self._lockfile.load()
-        return StageLoader(self, data.get("stages", {}), lockfile_data)
+        return self.LOADER(self, data, lockfile_data)
 
     def remove(self, force=False):
         if not force:
@@ -242,7 +298,7 @@ class PipelineFile(FileMixin):
         if not self.exists():
             return
 
-        with open(self.path, "r") as f:
+        with open(self.path, encoding="utf-8") as f:
             d = parse_yaml_for_update(f.read(), self.path)
 
         self.validate(d, self.path)
@@ -261,35 +317,90 @@ class PipelineFile(FileMixin):
         raise NotImplementedError
 
 
+def get_lockfile_schema(d):
+    from dvc.schema import (
+        COMPILED_LOCKFILE_V1_SCHEMA,
+        COMPILED_LOCKFILE_V2_SCHEMA,
+    )
+
+    schema = {
+        LOCKFILE_VERSION.V1: COMPILED_LOCKFILE_V1_SCHEMA,
+        LOCKFILE_VERSION.V2: COMPILED_LOCKFILE_V2_SCHEMA,
+    }
+
+    version = LOCKFILE_VERSION.from_dict(d)
+    return schema[version]
+
+
+def migrate_lock_v1_to_v2(d, version_info):
+    stages = {k: v for k, v in d.items()}
+
+    for key in stages:
+        d.pop(key)
+
+    # forcing order, meta should always be at the top
+    d.update(version_info)
+    d["stages"] = stages
+
+
 class Lockfile(FileMixin):
-    from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
-
-    def load(self):
-        if not self.exists():
-            return {}
-
-        data = load_yaml(self.path, tree=self.repo.tree)
+    @classmethod
+    def validate(cls, d, fname=None):
+        schema = get_lockfile_schema(d)
         try:
-            self.validate(data, fname=self.relpath)
-        except StageFileFormatError:
+            return schema(d)
+        except MultipleInvalid as exc:
+            raise StageFileFormatError(f"'{fname}' format error: {exc}")
+
+    def _verify_filename(self):
+        pass  # lockfile path is hardcoded, so no need to verify here
+
+    def _load(self):
+        try:
+            return super()._load()
+        except StageFileDoesNotExistError:
+            # we still need to account for git-ignored dvc.lock file
+            # even though it may not exist or have been .dvcignored
+            self._check_gitignored()
+            return {}, ""
+        except StageFileFormatError as exc:
             raise LockfileCorruptedError(
                 f"Lockfile '{self.relpath}' is corrupted."
-            )
-        return data
+            ) from exc
+
+    def load(self):
+        d, _ = self._load()
+        return d
+
+    @property
+    def latest_version_info(self):
+        version = LOCKFILE_VERSION.V2.value  # pylint:disable=no-member
+        return {SCHEMA_KWD: version}
 
     def dump(self, stage, **kwargs):
         stage_data = serialize.to_lockfile(stage)
 
-        with modify_yaml(self.path, tree=self.repo.tree) as data:
-            if not data:
-                logger.info("Generating lock file '%s'", self.relpath)
+        with modify_yaml(self.path, fs=self.repo.fs) as data:
+            version = LOCKFILE_VERSION.from_dict(data)
+            if version == LOCKFILE_VERSION.V1:
+                logger.info(
+                    "Migrating lock file '%s' from v1 to v2", self.relpath
+                )
+                migrate_lock_v1_to_v2(data, self.latest_version_info)
+            else:
+                if not data:
+                    data.update(self.latest_version_info)
+                    # order is important, meta should always be at the top
+                    logger.info("Generating lock file '%s'", self.relpath)
 
-            modified = data.get(stage.name, {}) != stage_data.get(
+            data["stages"] = data.get("stages", {})
+            modified = data["stages"].get(stage.name, {}) != stage_data.get(
                 stage.name, {}
             )
             if modified:
                 logger.info("Updating lock file '%s'", self.relpath)
-            data.update(stage_data)
+
+            data["stages"].update(stage_data)
 
         if modified:
             self.repo.scm.track_file(self.relpath)
@@ -298,17 +409,19 @@ class Lockfile(FileMixin):
         if not self.exists():
             return
 
-        with open(self.path) as f:
+        with open(self.path, encoding="utf-8") as f:
             d = parse_yaml_for_update(f.read(), self.path)
         self.validate(d, self.path)
 
-        if stage.name not in d:
+        version = LOCKFILE_VERSION.from_dict(d)
+        data = d if version == LOCKFILE_VERSION.V1 else d.get("stages", {})
+        if stage.name not in data:
             return
 
         logger.debug("Removing '%s' from '%s'", stage.name, self.path)
-        del d[stage.name]
+        del data[stage.name]
 
-        if d:
+        if data:
             dump_yaml(self.path, d)
         else:
             self.remove()
@@ -318,12 +431,19 @@ class Lockfile(FileMixin):
 
 
 class Dvcfile:
-    def __new__(cls, repo, path, **kwargs):
+    def __new__(cls, repo: "Repo", path: AnyPath, **kwargs: Any):
         assert path
         assert repo
 
-        _, ext = os.path.splitext(path)
-        if ext in [".yaml", ".yml"]:
-            return PipelineFile(repo, path, **kwargs)
-        # fallback to single stage file for better error messages
-        return SingleStageFile(repo, path, **kwargs)
+        return make_dvcfile(repo, path, **kwargs)
+
+
+DVCFile = Union["PipelineFile", "SingleStageFile"]
+
+
+def make_dvcfile(repo: "Repo", path: AnyPath, **kwargs: Any) -> DVCFile:
+    _, ext = os.path.splitext(str(path))
+    if ext in [".yaml", ".yml"]:
+        return PipelineFile(repo, path, **kwargs)
+    # fallback to single stage file for better error messages
+    return SingleStageFile(repo, path, **kwargs)

dvc/env.py

@@ -1,2 +1,12 @@
+DVC_CHECKPOINT = "DVC_CHECKPOINT"
 DVC_DAEMON = "DVC_DAEMON"
 DVC_PAGER = "DVC_PAGER"
+DVC_ROOT = "DVC_ROOT"
+DVCLIVE_PATH = "DVCLIVE_PATH"
+DVCLIVE_SUMMARY = "DVCLIVE_SUMMARY"
+DVCLIVE_HTML = "DVCLIVE_HTML"
+DVCLIVE_RESUME = "DVCLIVE_RESUME"
+DVC_IGNORE_ISATTY = "DVC_IGNORE_ISATTY"
+DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE"
+DVC_EXP_AUTO_PUSH = "DVC_EXP_AUTO_PUSH"
+DVC_NO_ANALYTICS = "DVC_NO_ANALYTICS"

dvc/exceptions.py

@@ -1,7 +1,4 @@
 """Exceptions raised by the dvc."""
-from funcy import first
-
-from dvc.utils import error_link, format_link, relpath
 
 
 class DvcException(Exception):
@@ -9,6 +6,7 @@ class DvcException(Exception):
 
     def __init__(self, msg, *args):
         assert msg
+        self.msg = msg
         super().__init__(msg, *args)
 
 
@@ -26,6 +24,8 @@ class OutputDuplicationError(DvcException):
     """
 
     def __init__(self, output, stages):
+        from funcy import first
+
         assert isinstance(output, str)
         assert all(hasattr(stage, "relpath") for stage in stages)
         if len(stages) == 1:
@@ -34,7 +34,7 @@ class OutputDuplicationError(DvcException):
             )
         else:
             msg = "output '{}' is already specified in stages:\n{}".format(
-                output, "\n".join(f"\t- {s.addressing}" for s in stages),
+                output, "\n".join(f"\t- {s.addressing}" for s in stages)
             )
         super().__init__(msg)
         self.stages = stages
@@ -42,17 +42,19 @@ class OutputDuplicationError(DvcException):
 
 
 class OutputNotFoundError(DvcException):
-    """Thrown if a file/directory not found in repository pipelines.
+    """Thrown if a file/directory is not found as an output in any pipeline.
 
     Args:
         output (unicode): path to the file/directory.
     """
 
     def __init__(self, output, repo=None):
+        from dvc.utils import relpath
+
         self.output = output
         self.repo = repo
         super().__init__(
-            "Unable to find DVC-file with output '{path}'".format(
+            "Unable to find DVC file with output '{path}'".format(
                 path=relpath(self.output)
             )
         )
@@ -105,7 +107,7 @@ class ArgumentDuplicationError(DvcException):
 
 
 class MoveNotDataSourceError(DvcException):
-    """Thrown if attempted to move a file/directory that is not an output
+    """Thrown when trying to move a file/directory that is not an output
     in a data source stage.
 
     Args:
@@ -164,24 +166,12 @@ class BadMetricError(DvcException):
     def __init__(self, paths):
         super().__init__(
             "the following metrics do not exist, "
-            "are not metric files or are malformed: {paths}".format(
+            "are not metrics files or are malformed: {paths}".format(
                 paths=", ".join(f"'{path}'" for path in paths)
             )
         )
 
 
-class NoMetricsError(DvcException):
-    pass
-
-
-class NoPlotsError(DvcException):
-    def __init__(self):
-        super().__init__(
-            "no plots in this repository. Use `--plots/--plots-no-cache` "
-            "options for `dvc run` to mark stage outputs as plots."
-        )
-
-
 class RecursiveAddingWhileUsingFilename(DvcException):
     def __init__(self):
         super().__init__(
@@ -228,6 +218,8 @@ class DvcIgnoreInCollectedDirError(DvcException):
 
 class GitHookAlreadyExistsError(DvcException):
     def __init__(self, hook_name):
+        from dvc.utils import format_link
+
         super().__init__(
             "Hook '{}' already exists. Please refer to {} for more "
             "info.".format(
@@ -236,29 +228,34 @@ class GitHookAlreadyExistsError(DvcException):
         )
 
 
-class DownloadError(DvcException):
+class FileTransferError(DvcException):
+    _METHOD = "transfer"
+
     def __init__(self, amount):
         self.amount = amount
 
-        super().__init__(f"{amount} files failed to download")
+        super().__init__(f"{amount} files failed to {self._METHOD}")
 
 
-class UploadError(DvcException):
-    def __init__(self, amount):
-        self.amount = amount
+class DownloadError(FileTransferError):
+    _METHOD = "download"
 
-        super().__init__(f"{amount} files failed to upload")
+
+class UploadError(FileTransferError):
+    _METHOD = "upload"
 
 
 class CheckoutError(DvcException):
     def __init__(self, target_infos, stats=None):
+        from dvc.utils import error_link
+
         self.target_infos = target_infos
         self.stats = stats
         targets = [str(t) for t in target_infos]
         m = (
             "Checkout failed for following targets:\n{}\nIs your "
             "cache up to date?\n{}".format(
-                "\n".join(targets), error_link("missing-files"),
+                "\n".join(targets), error_link("missing-files")
             )
         )
         super().__init__(m)
@@ -277,6 +274,8 @@ class NoRemoteInExternalRepoError(DvcException):
 
 class NoOutputInExternalRepoError(DvcException):
     def __init__(self, path, external_repo_path, external_repo_url):
+        from dvc.utils import relpath
+
         super().__init__(
             "Output '{}' not found in target repository '{}'".format(
                 relpath(path, external_repo_path), external_repo_url
@@ -307,6 +306,8 @@ class PathMissingError(DvcException):
 
 class RemoteCacheRequiredError(DvcException):
     def __init__(self, path_info):
+        from dvc.utils import format_link
+
         super().__init__(
             (
                 "Current operation was unsuccessful because '{}' requires "
@@ -338,3 +339,28 @@ class NoOutputOrStageError(DvcException):
 
 class MergeError(DvcException):
     pass
+
+
+class CacheLinkError(DvcException):
+    from dvc.utils import format_link
+
+    SUPPORT_LINK = "See {} for more information.".format(
+        format_link(
+            "https://dvc.org/doc/user-guide/troubleshooting#cache-types"
+        )
+    )
+
+    def __init__(self, path_infos):
+        msg = "No possible cache link types for '{}'. {}".format(
+            ", ".join([str(path) for path in path_infos]), self.SUPPORT_LINK
+        )
+        super().__init__(msg)
+        self.path_infos = path_infos
+
+
+class CircularImportError(DvcException):
+    def __init__(self, dep, a, b):
+        super().__init__(
+            f"'{dep}' contains invalid circular import. "
+            f"DVC repo '{a}' already imports from '{b}'."
+        )

dvc/external_repo.py

@@ -3,40 +3,31 @@ import os
 import tempfile
 import threading
 from contextlib import contextmanager
-from distutils.dir_util import copy_tree
-from typing import Iterable
+from typing import Dict
 
-from funcy import cached_property, reraise, retry, wrap_with
+from funcy import retry, wrap_with
 
-from dvc.cache import Cache
-from dvc.config import NoRemoteError, NotDvcRepoError
 from dvc.exceptions import (
-    DvcException,
     FileMissingError,
     NoOutputInExternalRepoError,
     NoRemoteInExternalRepoError,
+    NotDvcRepoError,
     OutputNotFoundError,
     PathMissingError,
 )
-from dvc.path_info import PathInfo
 from dvc.repo import Repo
-from dvc.scm.base import CloneError
-from dvc.scm.git import Git
-from dvc.state import StateNoop
-from dvc.tree.local import LocalTree
-from dvc.tree.repo import RepoTree
 from dvc.utils import relpath
-from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
 
-class IsADVCRepoError(DvcException):
-    """Raised when it is not expected to be a dvc repo."""
-
-
 @contextmanager
-def external_repo(url, rev=None, for_write=False, **kwargs):
+def external_repo(
+    url, rev=None, for_write=False, cache_dir=None, cache_types=None, **kwargs
+):
+    from dvc.config import NoRemoteError
+    from dvc.scm.git import Git
+
     logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
     # Local HEAD points to the tip of whatever branch we first cloned from
@@ -44,19 +35,31 @@ def external_repo(url, rev=None, for_write=False, **kwargs):
     # the tip of the default branch
     rev = rev or "refs/remotes/origin/HEAD"
 
+    cache_config = {
+        "cache": {"dir": cache_dir or _get_cache_dir(url), "type": cache_types}
+    }
+
+    config = _get_remote_config(url) if os.path.isdir(url) else {}
+    config.update(cache_config)
+
     root_dir = path if for_write else os.path.realpath(path)
-    conf = dict(
+    repo_kwargs = dict(
         root_dir=root_dir,
         url=url,
         scm=None if for_write else Git(root_dir),
         rev=None if for_write else rev,
-        for_write=for_write,
+        config=config,
+        repo_factory=erepo_factory(url, cache_config),
         **kwargs,
     )
-    try:
-        repo = ExternalRepo(**conf)
-    except NotDvcRepoError:
-        repo = ExternalGitRepo(**conf)
+
+    if "subrepos" not in repo_kwargs:
+        repo_kwargs["subrepos"] = True
+
+    if "uninitialized" not in repo_kwargs:
+        repo_kwargs["uninitialized"] = True
+
+    repo = Repo(**repo_kwargs)
 
     try:
         yield repo
@@ -76,8 +79,29 @@ def external_repo(url, rev=None, for_write=False, **kwargs):
             _remove(path)
 
 
-CLONES = {}
-CACHE_DIRS = {}
+def erepo_factory(url, cache_config, *args, **kwargs):
+    def make_repo(path, **_kwargs):
+        _config = cache_config.copy()
+        if os.path.isdir(url):
+            rel = os.path.relpath(path, _kwargs["scm"].root_dir)
+            repo_path = os.path.join(url, rel)
+            _config.update(_get_remote_config(repo_path))
+        return Repo(path, config=_config, **_kwargs)
+
+    return make_repo
+
+
+CLONES: Dict[str, str] = {}
+CACHE_DIRS: Dict[str, str] = {}
+
+
+@wrap_with(threading.Lock())
+def _get_cache_dir(url):
+    try:
+        cache_dir = CACHE_DIRS[url]
+    except KeyError:
+        cache_dir = CACHE_DIRS[url] = tempfile.mkdtemp("dvc-cache")
+    return cache_dir
 
 
 def clean_repos():
@@ -90,256 +114,28 @@ def clean_repos():
         _remove(path)
 
 
-class BaseExternalRepo:
-    # pylint: disable=no-member
+def _get_remote_config(url):
+    try:
+        repo = Repo(url)
+    except NotDvcRepoError:
+        return {}
 
-    def __str__(self):
-        return self.url
+    try:
+        name = repo.config["core"].get("remote")
+        if not name:
+            # Fill the empty upstream entry with a new remote pointing to the
+            # original repo's cache location.
+            name = "auto-generated-upstream"
+            return {
+                "core": {"remote": name},
+                "remote": {name: {"url": repo.odb.local.cache_dir}},
+            }
 
-    @cached_property
-    def repo_tree(self):
-        return self._get_tree_for(
-            self, subrepos=not self.for_write, repo_factory=self.make_repo
-        )
-
-    def get_rev(self):
-        assert self.scm
-        if isinstance(self.tree, LocalTree):
-            return self.scm.get_rev()
-        return self.tree.rev
-
-    def _fetch_to_cache(self, path_info, repo, callback):
-        # don't support subrepo traversal as it might fail due to difference
-        # in remotes
-        tree = self._get_tree_for(repo)
-        cache = repo.cache.local
-
-        hash_info = tree.get_hash(path_info, download_callback=callback)
-        cache.save(
-            path_info,
-            tree,
-            hash_info,
-            save_link=False,
-            download_callback=callback,
-        )
-        return hash_info
-
-    def fetch_external(self, paths: Iterable, **kwargs):
-        """Fetch specified external repo paths into cache.
-
-        Returns 3-tuple in the form
-            (downloaded, failed, list(cache_infos))
-        where cache_infos can be used as checkout targets for the
-        fetched paths.
-        """
-        download_results = []
-        failed = 0
-        root = PathInfo(self.root_dir)
-
-        paths = [root / path for path in paths]
-
-        def download_update(result):
-            download_results.append(result)
-
-        hash_infos = []
-        for path in paths:
-            with reraise(FileNotFoundError, PathMissingError(path, self.url)):
-                metadata = self.repo_tree.metadata(path)
-
-            self._check_repo(path, metadata.repo)
-            repo = metadata.repo
-            hash_info = self._fetch_to_cache(path, repo, download_update)
-            hash_infos.append(hash_info)
-
-        return sum(download_results), failed, hash_infos
-
-    def _check_repo(self, path_info, repo):
-        if not repo:
-            return
-
-        repo_path = PathInfo(repo.root_dir)
-        if path_info == repo_path and isinstance(repo, Repo):
-            message = "Cannot fetch a complete DVC repository"
-            if repo.root_dir != self.root_dir:
-                rel = relpath(repo.root_dir, self.root_dir)
-                message += f" '{rel}'"
-            raise IsADVCRepoError(message)
-
-    def get_external(self, path, dest):
-        """Convenience wrapper for fetch_external and checkout."""
-        path_info = PathInfo(self.root_dir) / path
-        with reraise(FileNotFoundError, PathMissingError(path, self.url)):
-            metadata = self.repo_tree.metadata(path_info)
-
-        self._check_repo(path_info, metadata.repo)
-        if metadata.output_exists:
-            repo = metadata.repo
-            cache = repo.cache.local
-            # fetch DVC and git files to tmpdir cache, then checkout
-            save_info = self._fetch_to_cache(path_info, repo, None)
-            cache.checkout(PathInfo(dest), save_info)
-        else:
-            # git-only folder, just copy files directly to dest
-            tree = self._get_tree_for(metadata.repo)  # ignore subrepos
-            tree.copytree(path_info, dest)
-
-    def _get_tree_for(self, repo, **kwargs):
-        """
-        Provides a combined tree of a single repo with dvc + git/local tree.
-        """
-        kw = {**self.tree_confs, **kwargs}
-        if "fetch" not in kw:
-            kw["fetch"] = True
-        return RepoTree(repo, **kw)
-
-    def get_checksum(self, path):
-        path_info = PathInfo(self.root_dir) / path
-        with reraise(FileNotFoundError, PathMissingError(path, self.url)):
-            metadata = self.repo_tree.metadata(path_info)
-
-        # skip subrepos to check for
-        tree = self._get_tree_for(metadata.repo)
-        return tree.get_hash(path_info)
-
-    @staticmethod
-    def _fix_local_remote(orig_repo, src_repo, remote_name):
-        # If a remote URL is relative to the source repo,
-        # it will have changed upon config load and made
-        # relative to this new repo. Restore the old one here.
-        new_remote = orig_repo.config["remote"][remote_name]
-        old_remote = src_repo.config["remote"][remote_name]
-        if new_remote["url"] != old_remote["url"]:
-            new_remote["url"] = old_remote["url"]
-
-    @staticmethod
-    def _add_upstream(orig_repo, src_repo):
-        # Fill the empty upstream entry with a new remote pointing to the
-        # original repo's cache location.
-        cache_dir = src_repo.cache.local.cache_dir
-        orig_repo.config["remote"]["auto-generated-upstream"] = {
-            "url": cache_dir
-        }
-        orig_repo.config["core"]["remote"] = "auto-generated-upstream"
-
-    def make_repo(self, path):
-        repo = Repo(path, scm=self.scm, rev=self.get_rev())
-
-        self._setup_cache(repo)
-        self._fix_upstream(repo)
-
-        return repo
-
-    def _setup_cache(self, repo):
-        repo.cache.local.cache_dir = self.cache_dir
-        if self.cache_types:
-            repo.cache.local.cache_types = self.cache_types
-
-    def _fix_upstream(self, repo):
-        if not os.path.isdir(self.url):
-            return
-
-        try:
-            rel_path = os.path.relpath(repo.root_dir, self.root_dir)
-            src_repo = Repo(PathInfo(self.url) / rel_path)
-        except NotDvcRepoError:
-            return
-
-        try:
-            remote_name = repo.config["core"].get("remote")
-            if remote_name:
-                self._fix_local_remote(repo, src_repo, remote_name)
-            else:
-                self._add_upstream(repo, src_repo)
-        finally:
-            src_repo.close()
-
-    @wrap_with(threading.Lock())
-    def _get_cache_dir(self):
-        try:
-            cache_dir = CACHE_DIRS[self.url]
-        except KeyError:
-            cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
-        return cache_dir
-
-
-class ExternalRepo(BaseExternalRepo, Repo):
-    def __init__(
-        self,
-        root_dir,
-        url,
-        scm=None,
-        rev=None,
-        for_write=False,
-        cache_dir=None,
-        cache_types=None,
-        **kwargs,
-    ):
-        super().__init__(root_dir, scm=scm, rev=rev)
-
-        self.url = url
-        self.for_write = for_write
-        self.cache_dir = cache_dir or self._get_cache_dir()
-        self.cache_types = cache_types
-
-        self._setup_cache(self)
-        self._fix_upstream(self)
-        self.tree_confs = kwargs
-
-
-class ExternalGitRepo(BaseExternalRepo):
-    def __init__(
-        self,
-        root_dir,
-        url,
-        scm=None,
-        rev=None,
-        for_write=False,
-        cache_dir=None,
-        cache_types=None,
-        **kwargs,
-    ):
-        self.root_dir = os.path.realpath(root_dir)
-        self.scm = scm
-
-        self.url = url
-        self.for_write = for_write
-        self.cache_dir = cache_dir or self._get_cache_dir()
-        self.cache_types = cache_types
-
-        self.rev = rev
-        self.tree_confs = kwargs
-
-        self.config = {"cache": {"dir": self.cache_dir}}
-        self.cache = Cache(self)
-        if cache_types:
-            self.cache.local.cache_types = cache_types
-
-        self.state = StateNoop()
-
-    @cached_property
-    def tree(self):
-        if self.scm:
-            return self.scm.get_tree(self.rev)
-        return LocalTree(self, {"url": self.root_dir})
-
-    def close(self):
-        if self.scm:
-            self.scm.close()
-
-    def find_out_by_relpath(self, path):
-        raise OutputNotFoundError(path, self)
-
-    @contextmanager
-    def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
-        """Opens a specified resource as a file object."""
-        path_info = PathInfo(self.root_dir) / path
-        try:
-            with self.repo_tree.open(
-                path_info, mode=mode, encoding=encoding, **kwargs
-            ) as fobj:
-                yield fobj
-        except FileNotFoundError as exc:
-            raise PathMissingError(path, self.url) from exc
+        # Use original remote to make sure that we are using correct url,
+        # credential paths, etc if they are relative to the config location.
+        return {"remote": {name: repo.config["remote"][name]}}
+    finally:
+        repo.close()
 
 
 def _cached_clone(url, rev, for_write=False):
@@ -349,6 +145,8 @@ def _cached_clone(url, rev, for_write=False):
     revision checked out. If for_write is set prevents reusing this dir via
     cache.
     """
+    from shutil import copytree
+
     # even if we have already cloned this repo, we may need to
     # fetch/fast-forward to get specified rev
     clone_path, shallow = _clone_default_branch(url, rev, for_write=for_write)
@@ -359,7 +157,7 @@ def _cached_clone(url, rev, for_write=False):
     # Copy to a new dir to keep the clone clean
     repo_path = tempfile.mkdtemp("dvc-erepo")
     logger.debug("erepo: making a copy of %s clone", url)
-    copy_tree(clone_path, repo_path)
+    copytree(clone_path, repo_path)
 
     # Check out the specified revision
     if for_write:
@@ -375,6 +173,8 @@ def _clone_default_branch(url, rev, for_write=False):
 
     The cloned is reactualized with git pull unless rev is a known sha.
     """
+    from dvc.scm.git import Git
+
     clone_path, shallow = CLONES.get(url, (None, False))
 
     git = None
@@ -402,6 +202,8 @@ def _clone_default_branch(url, rev, for_write=False):
             clone_path = tempfile.mkdtemp("dvc-clone")
             if not for_write and rev and not Git.is_sha(rev):
                 # If rev is a tag or branch name try shallow clone first
+                from dvc.scm.base import CloneError
+
                 try:
                     git = Git.clone(url, clone_path, shallow_branch=rev)
                     shallow = True
@@ -422,19 +224,21 @@ def _clone_default_branch(url, rev, for_write=False):
 
 
 def _unshallow(git):
-    if git.repo.head.is_detached:
+    if git.gitpython.repo.head.is_detached:
         # If this is a detached head (i.e. we shallow cloned a tag) switch to
         # the default branch
-        origin_refs = git.repo.remotes["origin"].refs
+        origin_refs = git.gitpython.repo.remotes["origin"].refs
         ref = origin_refs["HEAD"].reference
         branch_name = ref.name.split("/")[-1]
-        branch = git.repo.create_head(branch_name, ref)
+        branch = git.gitpython.repo.create_head(branch_name, ref)
         branch.set_tracking_branch(ref)
         branch.checkout()
     git.pull(unshallow=True)
 
 
 def _git_checkout(repo_path, rev):
+    from dvc.scm.git import Git
+
     logger.debug("erepo: git checkout %s@%s", repo_path, rev)
     git = Git(repo_path)
     try:
@@ -444,6 +248,8 @@ def _git_checkout(repo_path, rev):
 
 
 def _remove(path):
+    from dvc.utils.fs import remove
+
     if os.name == "nt":
         # git.exe may hang for a while not permitting to remove temp dir
         os_retry = retry(5, errors=OSError, timeout=0.1)

dvc/fs/__init__.py

@@ -0,0 +1,122 @@
+import posixpath
+from urllib.parse import urlparse
+
+from ..scheme import Schemes
+from . import utils  # noqa: F401
+from .azure import AzureFileSystem
+from .gdrive import GDriveFileSystem
+from .gs import GSFileSystem
+from .hdfs import HDFSFileSystem
+from .http import HTTPFileSystem
+from .https import HTTPSFileSystem
+from .local import LocalFileSystem
+from .oss import OSSFileSystem
+from .s3 import S3FileSystem
+from .ssh import SSHFileSystem
+from .webdav import WebDAVFileSystem, WebDAVSFileSystem
+from .webhdfs import WebHDFSFileSystem
+
+FS_MAP = {
+    Schemes.AZURE: AzureFileSystem,
+    Schemes.GDRIVE: GDriveFileSystem,
+    Schemes.GS: GSFileSystem,
+    Schemes.HDFS: HDFSFileSystem,
+    Schemes.WEBHDFS: WebHDFSFileSystem,
+    Schemes.HTTP: HTTPFileSystem,
+    Schemes.HTTPS: HTTPSFileSystem,
+    Schemes.S3: S3FileSystem,
+    Schemes.SSH: SSHFileSystem,
+    Schemes.OSS: OSSFileSystem,
+    Schemes.WEBDAV: WebDAVFileSystem,
+    Schemes.WEBDAVS: WebDAVSFileSystem,
+    # NOTE: LocalFileSystem is the default
+}
+
+
+def get_fs_cls(remote_conf, scheme=None):
+    if not scheme:
+        scheme = urlparse(remote_conf["url"]).scheme
+    return FS_MAP.get(scheme, LocalFileSystem)
+
+
+def get_fs_config(config, **kwargs):
+    name = kwargs.get("name")
+    if name:
+        try:
+            remote_conf = config["remote"][name.lower()]
+        except KeyError:
+            from dvc.config import RemoteNotFoundError
+
+            raise RemoteNotFoundError(f"remote '{name}' doesn't exist")
+    else:
+        remote_conf = kwargs
+    return _resolve_remote_refs(config, remote_conf)
+
+
+def _resolve_remote_refs(config, remote_conf):
+    # Support for cross referenced remotes.
+    # This will merge the settings, shadowing base ref with remote_conf.
+    # For example, having:
+    #
+    #       dvc remote add server ssh://localhost
+    #       dvc remote modify server user root
+    #       dvc remote modify server ask_password true
+    #
+    #       dvc remote add images remote://server/tmp/pictures
+    #       dvc remote modify images user alice
+    #       dvc remote modify images ask_password false
+    #       dvc remote modify images password asdf1234
+    #
+    # Results on a config dictionary like:
+    #
+    #       {
+    #           "url": "ssh://localhost/tmp/pictures",
+    #           "user": "alice",
+    #           "password": "asdf1234",
+    #           "ask_password": False,
+    #       }
+    parsed = urlparse(remote_conf["url"])
+    if parsed.scheme != "remote":
+        return remote_conf
+
+    base = get_fs_config(config, name=parsed.netloc)
+    url = posixpath.join(base["url"], parsed.path.lstrip("/"))
+    return {**base, **remote_conf, "url": url}
+
+
+def get_cloud_fs(repo, **kwargs):
+    from dvc.config import ConfigError
+    from dvc.config_schema import SCHEMA, Invalid
+
+    repo_config = repo.config if repo else {}
+    core_config = repo_config.get("core", {})
+
+    remote_conf = get_fs_config(repo_config, **kwargs)
+    try:
+        remote_conf = SCHEMA["remote"][str](remote_conf)
+    except Invalid as exc:
+        raise ConfigError(str(exc)) from None
+
+    if "jobs" not in remote_conf:
+        jobs = core_config.get("jobs")
+        if jobs:
+            remote_conf["jobs"] = jobs
+
+    if "checksum_jobs" not in remote_conf:
+        checksum_jobs = core_config.get("checksum_jobs")
+        if checksum_jobs:
+            remote_conf["checksum_jobs"] = checksum_jobs
+
+    cls = get_fs_cls(remote_conf)
+
+    if cls == GDriveFileSystem and repo:
+        remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir
+
+    url = remote_conf.pop("url")
+    path_info = cls.PATH_CLS(
+        cls._strip_protocol(url)  # pylint:disable=protected-access
+    )
+
+    extras = cls._get_kwargs_from_urls(url)  # pylint:disable=protected-access
+    conf = {**extras, **remote_conf}  # remote config takes priority
+    return cls, conf, path_info

dvc/fs/_metadata.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, List
 
-from dvc.output import BaseOutput
+from dvc.output import Output
 from dvc.path_info import PathInfo
 
 if TYPE_CHECKING:
@@ -33,7 +33,7 @@ class Metadata:
     # optional fields
     isdir: bool = False  # is it a directory?
     is_exec: bool = False  # is it an executable?
-    outs: List[BaseOutput] = field(default_factory=list)  # list of outputs
+    outs: List[Output] = field(default_factory=list)  # list of outputs
 
     def __post_init__(self):
         self.output_exists = bool(self.outs)

dvc/fs/azure.py

@@ -0,0 +1,161 @@
+import logging
+import os
+import threading
+
+from fsspec.asyn import fsspec_loop
+from fsspec.utils import infer_storage_options
+from funcy import cached_property, memoize, wrap_prop
+
+from dvc.exceptions import DvcException
+from dvc.path_info import CloudURLInfo
+from dvc.scheme import Schemes
+from dvc.utils import format_link
+
+from .fsspec_wrapper import ObjectFSWrapper
+
+logger = logging.getLogger(__name__)
+_DEFAULT_CREDS_STEPS = (
+    "https://azuresdkdocs.blob.core.windows.net/$web/python/"
+    "azure-identity/1.4.0/azure.identity.html#azure.identity"
+    ".DefaultAzureCredential"
+)
+
+
+class AzureAuthError(DvcException):
+    pass
+
+
+@memoize
+def _az_config():
+    # NOTE: ideally we would've used get_default_cli().config from
+    # azure.cli.core, but azure-cli-core has a lot of conflicts with other
+    # dependencies. So instead we are just use knack directly
+    from knack.config import CLIConfig
+
+    config_dir = os.getenv(
+        "AZURE_CONFIG_DIR", os.path.expanduser(os.path.join("~", ".azure"))
+    )
+    return CLIConfig(config_dir=config_dir, config_env_var_prefix="AZURE")
+
+
+# pylint:disable=abstract-method
+class AzureFileSystem(ObjectFSWrapper):
+    scheme = Schemes.AZURE
+    PATH_CLS = CloudURLInfo
+    PARAM_CHECKSUM = "etag"
+    DETAIL_FIELDS = frozenset(("etag", "size"))
+    REQUIRES = {
+        "adlfs": "adlfs",
+        "knack": "knack",
+        "azure-identity": "azure.identity",
+    }
+
+    @classmethod
+    def _strip_protocol(cls, path: str):
+        bucket = infer_storage_options(path).get("host")
+        if bucket:
+            return path
+
+        bucket = _az_config().get("storage", "container_name", None)
+        return f"azure://{bucket}"
+
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath):
+        ops = infer_storage_options(urlpath)
+        if "host" in ops:
+            return {"bucket": ops["host"]}
+        return {}
+
+    def _prepare_credentials(self, **config):
+        from azure.identity.aio import DefaultAzureCredential
+
+        # Disable spam from failed cred types for DefaultAzureCredential
+        logging.getLogger("azure.identity.aio").setLevel(logging.ERROR)
+
+        login_info = {}
+        login_info["connection_string"] = config.get(
+            "connection_string",
+            _az_config().get("storage", "connection_string", None),
+        )
+        login_info["account_name"] = config.get(
+            "account_name", _az_config().get("storage", "account", None)
+        )
+        login_info["account_key"] = config.get(
+            "account_key", _az_config().get("storage", "key", None)
+        )
+        login_info["sas_token"] = config.get(
+            "sas_token", _az_config().get("storage", "sas_token", None)
+        )
+        login_info["tenant_id"] = config.get("tenant_id")
+        login_info["client_id"] = config.get("client_id")
+        login_info["client_secret"] = config.get("client_secret")
+
+        if not (login_info["account_name"] or login_info["connection_string"]):
+            raise AzureAuthError(
+                "Authentication to Azure Blob Storage requires either "
+                "account_name or connection_string.\nLearn more about "
+                "configuration settings at "
+                + format_link("https://man.dvc.org/remote/modify")
+            )
+
+        any_secondary = any(
+            value for key, value in login_info.items() if key != "account_name"
+        )
+        if (
+            login_info["account_name"]
+            and not any_secondary
+            and not config.get("allow_anonymous_login", False)
+        ):
+            with fsspec_loop():
+                login_info["credential"] = DefaultAzureCredential(
+                    exclude_interactive_browser_credential=False,
+                    exclude_environment_credential=config.get(
+                        "exclude_environment_credential", False
+                    ),
+                    exclude_visual_studio_code_credential=config.get(
+                        "exclude_visual_studio_code_credential", False
+                    ),
+                    exclude_shared_token_cache_credential=config.get(
+                        "exclude_shared_token_cache_credential", False
+                    ),
+                    exclude_managed_identity_credential=config.get(
+                        "exclude_managed_identity_credential", False
+                    ),
+                )
+
+        for login_method, required_keys in [  # noqa
+            ("connection string", ["connection_string"]),
+            (
+                "AD service principal",
+                ["tenant_id", "client_id", "client_secret"],
+            ),
+            ("account key", ["account_name", "account_key"]),
+            ("SAS token", ["account_name", "sas_token"]),
+            (
+                f"default credentials ({_DEFAULT_CREDS_STEPS})",
+                ["account_name", "credential"],
+            ),
+            ("anonymous login", ["account_name"]),
+        ]:
+            if all(login_info.get(key) is not None for key in required_keys):
+                break
+        else:
+            login_method = None
+
+        self.login_method = login_method
+        return login_info
+
+    @wrap_prop(threading.Lock())
+    @cached_property
+    def fs(self):
+        from adlfs import AzureBlobFileSystem
+        from azure.core.exceptions import AzureError
+
+        try:
+            return AzureBlobFileSystem(**self.fs_args)
+        except (ValueError, AzureError) as e:
+            raise AzureAuthError(
+                f"Authentication to Azure Blob Storage via {self.login_method}"
+                " failed.\nLearn more about configuration settings at"
+                f" {format_link('https://man.dvc.org/remote/modify')}"
+            ) from e

dvc/fs/base.py

@@ -0,0 +1,350 @@
+import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from functools import partial, partialmethod
+from multiprocessing import cpu_count
+from typing import Any, ClassVar, Dict, FrozenSet, Optional
+
+from dvc.exceptions import DvcException
+from dvc.path_info import URLInfo
+from dvc.progress import Tqdm
+from dvc.utils import tmp_fname
+from dvc.utils.fs import makedirs, move
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteCmdError(DvcException):
+    def __init__(self, remote, cmd, ret, err):
+        super().__init__(
+            "{remote} command '{cmd}' finished with non-zero return code"
+            " {ret}': {err}".format(remote=remote, cmd=cmd, ret=ret, err=err)
+        )
+
+
+class RemoteActionNotImplemented(DvcException):
+    def __init__(self, action, scheme):
+        m = f"{action} is not supported for {scheme} remotes"
+        super().__init__(m)
+
+
+class RemoteMissingDepsError(DvcException):
+    pass
+
+
+class BaseFileSystem:
+    sep = "/"
+
+    scheme = "base"
+    REQUIRES: ClassVar[Dict[str, str]] = {}
+    PATH_CLS = URLInfo  # type: Any
+    _JOBS = 4 * cpu_count()
+
+    CHECKSUM_DIR_SUFFIX = ".dir"
+    HASH_JOBS = max(1, min(4, cpu_count() // 2))
+    LIST_OBJECT_PAGE_SIZE = 1000
+    TRAVERSE_WEIGHT_MULTIPLIER = 5
+    TRAVERSE_PREFIX_LEN = 3
+    TRAVERSE_THRESHOLD_SIZE = 500000
+    CAN_TRAVERSE = True
+
+    # Needed for some providers, and http open()
+    CHUNK_SIZE = 64 * 1024 * 1024  # 64 MiB
+
+    PARAM_CHECKSUM: ClassVar[Optional[str]] = None
+    DETAIL_FIELDS: FrozenSet[str] = frozenset()
+
+    def __init__(self, **kwargs):
+        self._check_requires(**kwargs)
+
+        self.jobs = kwargs.get("jobs") or self._JOBS
+        self.hash_jobs = kwargs.get("checksum_jobs") or self.HASH_JOBS
+        self._config = kwargs
+
+    @property
+    def config(self):
+        return self._config
+
+    @classmethod
+    def _strip_protocol(cls, path: str):
+        return path
+
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath):  # pylint:disable=unused-argument
+        return {}
+
+    @classmethod
+    def get_missing_deps(cls):
+        import importlib
+
+        missing = []
+        for package, module in cls.REQUIRES.items():
+            try:
+                importlib.import_module(module)
+            except ImportError:
+                missing.append(package)
+
+        return missing
+
+    def _check_requires(self, **kwargs):
+        from ..scheme import Schemes
+        from ..utils import format_link
+        from ..utils.pkg import PKG
+
+        missing = self.get_missing_deps()
+        if not missing:
+            return
+
+        url = kwargs.get("url", f"{self.scheme}://")
+
+        scheme = self.scheme
+        if scheme == Schemes.WEBDAVS:
+            scheme = Schemes.WEBDAV
+
+        by_pkg = {
+            "pip": f"pip install 'dvc[{scheme}]'",
+            "conda": f"conda install -c conda-forge dvc-{scheme}",
+        }
+
+        cmd = by_pkg.get(PKG)
+        if cmd:
+            link = format_link("https://dvc.org/doc/install")
+            hint = (
+                f"To install dvc with those dependencies, run:\n"
+                "\n"
+                f"\t{cmd}\n"
+                "\n"
+                f"See {link} for more info."
+            )
+        else:
+            link = format_link("https://github.com/iterative/dvc/issues")
+            hint = f"Please report this bug to {link}. Thank you!"
+
+        raise RemoteMissingDepsError(
+            f"URL '{url}' is supported but requires these missing "
+            f"dependencies: {missing}. {hint}"
+        )
+
+    def checksum(self, path_info) -> str:
+        raise NotImplementedError
+
+    def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
+        raise RemoteActionNotImplemented("open", self.scheme)
+
+    def exists(self, path_info) -> bool:
+        raise NotImplementedError
+
+    # pylint: disable=unused-argument
+
+    def isdir(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return False
+
+    def isfile(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between a directory and a file.
+        """
+        return True
+
+    def isexec(self, path_info):
+        """Optional: Overwrite only if the remote has a way to distinguish
+        between executable and non-executable file.
+        """
+        return False
+
+    def iscopy(self, path_info):
+        """Check if this file is an independent copy."""
+        return False  # We can't be sure by default
+
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
+        """Return a generator with (root, dirs, files)."""
+        raise NotImplementedError
+
+    def walk_files(self, path_info, **kwargs):
+        """Return a generator with `PathInfo`s to all the files.
+
+        Optional kwargs:
+            prefix (bool): If true `path_info` will be treated as a prefix
+                rather than directory path.
+        """
+        raise NotImplementedError
+
+    def ls(self, path_info, detail=False):
+        raise RemoteActionNotImplemented("ls", self.scheme)
+
+    def find(self, path_info, detail=False, prefix=None):
+        raise RemoteActionNotImplemented("find", self.scheme)
+
+    def is_empty(self, path_info):
+        return False
+
+    def info(self, path_info):
+        raise NotImplementedError
+
+    def getsize(self, path_info):
+        return self.info(path_info).get("size")
+
+    def remove(self, path_info):
+        raise RemoteActionNotImplemented("remove", self.scheme)
+
+    def makedirs(self, path_info, **kwargs):
+        """Optional: Implement only if the remote needs to create
+        directories before copying/linking/moving data
+        """
+
+    def move(self, from_info, to_info):
+        self.copy(from_info, to_info)
+        self.remove(from_info)
+
+    def copy(self, from_info, to_info):
+        raise RemoteActionNotImplemented("copy", self.scheme)
+
+    def symlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("symlink", self.scheme)
+
+    def hardlink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("hardlink", self.scheme)
+
+    def reflink(self, from_info, to_info):
+        raise RemoteActionNotImplemented("reflink", self.scheme)
+
+    # pylint: enable=unused-argument
+
+    @classmethod
+    def is_dir_hash(cls, hash_):
+        if not hash_:
+            return False
+        return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
+
+    def upload(
+        self,
+        from_info,
+        to_info,
+        total=None,
+        desc=None,
+        no_progress_bar=False,
+        **pbar_args,
+    ):
+        is_file_obj = hasattr(from_info, "read")
+        method = "upload_fobj" if is_file_obj else "_upload"
+        if not hasattr(self, method):
+            raise RemoteActionNotImplemented(method, self.scheme)
+
+        if to_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        logger.debug("Uploading '%s' to '%s'", from_info, to_info)
+        if is_file_obj:
+            with Tqdm.wrapattr(
+                from_info,
+                "read",
+                disable=no_progress_bar,
+                bytes=True,
+                total=total,
+                desc=desc,
+                **pbar_args,
+            ) as wrapped:
+                # `size` is used to provide hints to the WebdavFileSystem
+                # for legacy servers.
+                # pylint: disable=no-member
+                return self.upload_fobj(wrapped, to_info, size=total)
+
+        if from_info.scheme != "local":
+            raise NotImplementedError
+
+        name = desc or from_info.name
+        return self._upload(  # noqa, pylint: disable=no-member
+            from_info.fspath,
+            to_info,
+            name=name,
+            no_progress_bar=no_progress_bar,
+            **pbar_args,
+        )
+
+    def download(
+        self,
+        from_info,
+        to_info,
+        name=None,
+        no_progress_bar=False,
+        jobs=None,
+        _only_file=False,
+        **kwargs,
+    ):
+        if not hasattr(self, "_download"):
+            raise RemoteActionNotImplemented("download", self.scheme)
+
+        if from_info.scheme != self.scheme:
+            raise NotImplementedError
+
+        if to_info.scheme == self.scheme != "local":
+            self.copy(from_info, to_info)
+            return 0
+
+        if to_info.scheme != "local":
+            raise NotImplementedError
+
+        if not _only_file and self.isdir(from_info):
+            return self._download_dir(
+                from_info, to_info, name, no_progress_bar, jobs, **kwargs
+            )
+        return self._download_file(from_info, to_info, name, no_progress_bar)
+
+    download_file = partialmethod(download, _only_file=True)
+
+    def _download_dir(
+        self, from_info, to_info, name, no_progress_bar, jobs, **kwargs
+    ):
+        from_infos = list(self.walk_files(from_info, **kwargs))
+        if not from_infos:
+            makedirs(to_info, exist_ok=True)
+            return None
+        to_infos = (
+            to_info / info.relative_to(from_info) for info in from_infos
+        )
+
+        with Tqdm(
+            total=len(from_infos),
+            desc="Downloading directory",
+            unit="Files",
+            disable=no_progress_bar,
+        ) as pbar:
+            download_files = pbar.wrap_fn(
+                partial(self._download_file, name=name, no_progress_bar=True)
+            )
+            max_workers = jobs or self.jobs
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                futures = [
+                    executor.submit(download_files, from_info, to_info)
+                    for from_info, to_info in zip(from_infos, to_infos)
+                ]
+
+                # NOTE: unlike pulling/fetching cache, where we need to
+                # download everything we can, not raising an error here might
+                # turn very ugly, as the user might think that he has
+                # downloaded a complete directory, while having a partial one,
+                # which might cause unexpected results in his pipeline.
+                for future in as_completed(futures):
+                    # NOTE: executor won't let us raise until all futures that
+                    # it has are finished, so we need to cancel them ourselves
+                    # before re-raising.
+                    exc = future.exception()
+                    if exc:
+                        for entry in futures:
+                            entry.cancel()
+                        raise exc
+
+    def _download_file(self, from_info, to_info, name, no_progress_bar):
+        makedirs(to_info.parent, exist_ok=True)
+
+        logger.debug("Downloading '%s' to '%s'", from_info, to_info)
+        name = name or to_info.name
+
+        tmp_file = tmp_fname(to_info)
+
+        self._download(  # noqa, pylint: disable=no-member
+            from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
+        )
+
+        move(tmp_file, to_info)

dvc/fs/dvc.py

@@ -0,0 +1,270 @@
+import logging
+import os
+import typing
+
+from dvc.exceptions import OutputNotFoundError
+from dvc.path_info import PathInfo
+from dvc.utils import relpath
+
+from ._metadata import Metadata
+from .base import BaseFileSystem
+
+if typing.TYPE_CHECKING:
+    from dvc.output import Output
+
+
+logger = logging.getLogger(__name__)
+
+
+class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
+    """DVC repo fs.
+
+    Args:
+        repo: DVC repo.
+    """
+
+    sep = os.sep
+
+    scheme = "local"
+    PARAM_CHECKSUM = "md5"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.repo = kwargs["repo"]
+
+    @property
+    def config(self):
+        raise NotImplementedError
+
+    def _find_outs(self, path, *args, **kwargs):
+        outs = self.repo.find_outs_by_path(path, *args, **kwargs)
+
+        def _is_cached(out):
+            return out.use_cache
+
+        outs = list(filter(_is_cached, outs))
+        if not outs:
+            raise OutputNotFoundError(path, self.repo)
+
+        return outs
+
+    def _get_granular_hash(
+        self, path_info: PathInfo, out: "Output", remote=None
+    ):
+        assert isinstance(path_info, PathInfo)
+        # NOTE: use string paths here for performance reasons
+        key = tuple(relpath(path_info, out.path_info).split(os.sep))
+        out.get_dir_cache(remote=remote)
+        if out.obj is None:
+            raise FileNotFoundError
+        obj = out.obj.trie.get(key)
+        if obj:
+            return obj.hash_info
+        raise FileNotFoundError
+
+    def _get_fs_path(self, path: PathInfo, remote=None):
+        try:
+            outs = self._find_outs(path, strict=False)
+        except OutputNotFoundError as exc:
+            raise FileNotFoundError from exc
+
+        if len(outs) != 1 or (
+            outs[0].is_dir_checksum and path == outs[0].path_info
+        ):
+            raise IsADirectoryError
+
+        out = outs[0]
+
+        if not out.hash_info:
+            raise FileNotFoundError
+
+        if out.changed_cache(filter_info=path):
+            from dvc.config import NoRemoteError
+
+            try:
+                remote_odb = self.repo.cloud.get_remote_odb(remote)
+            except NoRemoteError as exc:
+                raise FileNotFoundError from exc
+            if out.is_dir_checksum:
+                checksum = self._get_granular_hash(path, out).value
+            else:
+                checksum = out.hash_info.value
+            remote_info = remote_odb.hash_to_path_info(checksum)
+            return remote_odb.fs, remote_info
+
+        if out.is_dir_checksum:
+            checksum = self._get_granular_hash(path, out).value
+            cache_path = out.odb.hash_to_path_info(checksum).url
+        else:
+            cache_path = out.cache_path
+        return out.odb.fs, cache_path
+
+    def open(  # type: ignore
+        self, path: PathInfo, mode="r", encoding=None, **kwargs
+    ):  # pylint: disable=arguments-renamed
+        fs, fspath = self._get_fs_path(path, **kwargs)
+        return fs.open(fspath, mode=mode, encoding=encoding)
+
+    def exists(self, path):  # pylint: disable=arguments-renamed
+        try:
+            self.metadata(path)
+            return True
+        except FileNotFoundError:
+            return False
+
+    def isdir(self, path):  # pylint: disable=arguments-renamed
+        try:
+            meta = self.metadata(path)
+            return meta.isdir
+        except FileNotFoundError:
+            return False
+
+    def check_isdir(self, path_info, outs):
+        if len(outs) != 1:
+            return True
+
+        out = outs[0]
+        if not out.is_dir_checksum:
+            return out.path_info != path_info
+        if out.path_info == path_info:
+            return True
+
+        try:
+            self._get_granular_hash(path_info, out)
+            return False
+        except FileNotFoundError:
+            return True
+
+    def isfile(self, path):  # pylint: disable=arguments-renamed
+        try:
+            meta = self.metadata(path)
+            return meta.isfile
+        except FileNotFoundError:
+            return False
+
+    def _fetch_dir(self, out, **kwargs):
+        # pull dir cache if needed
+        out.get_dir_cache(**kwargs)
+
+        if not out.obj:
+            raise FileNotFoundError
+
+    def _add_dir(self, trie, out, **kwargs):
+        self._fetch_dir(out, **kwargs)
+
+        base = out.path_info.parts
+        for key, _ in out.obj:  # noqa: B301
+            trie[base + key] = None
+
+    def _walk(self, root, trie, topdown=True, **kwargs):
+        dirs = set()
+        files = []
+
+        out = trie.get(root.parts)
+        if out and out.is_dir_checksum:
+            self._add_dir(trie, out, **kwargs)
+
+        root_len = len(root.parts)
+        for key, out in trie.iteritems(prefix=root.parts):  # noqa: B301
+            if key == root.parts:
+                continue
+
+            name = key[root_len]
+            if len(key) > root_len + 1 or (out and out.is_dir_checksum):
+                dirs.add(name)
+                continue
+
+            files.append(name)
+
+        assert topdown
+        dirs = list(dirs)
+        yield root.fspath, dirs, files
+
+        for dname in dirs:
+            yield from self._walk(root / dname, trie)
+
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
+        from pygtrie import Trie
+
+        assert topdown
+        root = PathInfo(os.path.abspath(top))
+        try:
+            meta = self.metadata(root)
+        except FileNotFoundError:
+            if onerror is not None:
+                onerror(FileNotFoundError(top))
+            return
+
+        if not meta.isdir:
+            if onerror is not None:
+                onerror(NotADirectoryError(top))
+            return
+
+        trie = Trie()
+        for out in meta.outs:
+            trie[out.path_info.parts] = out
+
+            if out.is_dir_checksum and root.isin_or_eq(out.path_info):
+                self._add_dir(trie, out, **kwargs)
+
+        yield from self._walk(root, trie, topdown=topdown, **kwargs)
+
+    def walk_files(self, path_info, **kwargs):
+        for root, _, files in self.walk(path_info):
+            for fname in files:
+                # NOTE: os.path.join is ~5.5 times slower
+                yield PathInfo(f"{root}{os.sep}{fname}")
+
+    def isdvc(self, path, recursive=False, strict=True):
+        try:
+            meta = self.metadata(path)
+        except FileNotFoundError:
+            return False
+
+        recurse = recursive or not strict
+        return meta.output_exists if recurse else meta.is_output
+
+    def isexec(self, path_info):  # pylint: disable=unused-argument
+        return False
+
+    def metadata(self, path_info):
+        path_info = PathInfo(os.path.abspath(path_info))
+
+        try:
+            outs = self._find_outs(path_info, strict=False, recursive=True)
+        except OutputNotFoundError as exc:
+            raise FileNotFoundError from exc
+
+        meta = Metadata(path_info=path_info, outs=outs, repo=self.repo)
+        meta.isdir = meta.isdir or self.check_isdir(meta.path_info, meta.outs)
+        return meta
+
+    def info(self, path_info):
+        meta = self.metadata(path_info)
+        ret = {"type": "directory" if meta.isdir else "file"}
+        if meta.is_output and len(meta.outs) == 1 and meta.outs[0].hash_info:
+            hash_info = meta.outs[0].hash_info
+            ret["size"] = hash_info.size
+            ret[hash_info.name] = hash_info.value
+        elif meta.part_of_output:
+            (out,) = meta.outs
+            key = path_info.relative_to(out.path_info).parts
+            obj = out.obj.trie.get(key)
+            if obj:
+                ret["size"] = obj.size
+                ret[obj.hash_info.name] = obj.hash_info.value
+
+        return ret
+
+    def _download(self, from_info, to_file, **kwargs):
+        fs, path = self._get_fs_path(from_info)
+        fs._download(  # pylint: disable=protected-access
+            path, to_file, **kwargs
+        )
+
+    def checksum(self, path_info):
+        info = self.info(path_info)
+        md5 = info.get("md5")
+        if md5:
+            return md5
+        raise NotImplementedError

dvc/fs/fsspec_wrapper.py

@@ -0,0 +1,297 @@
+import os
+import shutil
+from functools import lru_cache
+
+from funcy import cached_property
+
+from dvc.progress import Tqdm
+
+from .base import BaseFileSystem
+from .local import LocalFileSystem
+
+
+# pylint: disable=no-member
+class FSSpecWrapper(BaseFileSystem):
+    TRAVERSE_PREFIX_LEN = 2
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.fs_args = {"skip_instance_cache": True}
+        self.fs_args.update(self._prepare_credentials(**kwargs))
+
+    @cached_property
+    def fs(self):
+        raise NotImplementedError
+
+    def _with_bucket(self, path):
+        return str(path)
+
+    def _strip_bucket(self, entry):
+        return entry
+
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath):
+        from fsspec.utils import infer_storage_options
+
+        options = infer_storage_options(urlpath)
+        options.pop("path", None)
+        options.pop("protocol", None)
+        return options
+
+    def _strip_buckets(self, entries, detail=False):
+        for entry in entries:
+            if detail:
+                entry = self._entry_hook(entry.copy())
+                entry["name"] = self._strip_bucket(entry["name"])
+            else:
+                entry = self._strip_bucket(entry)
+            yield entry
+
+    def _entry_hook(self, entry):
+        """Simple hook method to be overridden when wanted to process
+        entries within info() and ls(detail=True) calls"""
+        return entry
+
+    def _prepare_credentials(
+        self, **config
+    ):  # pylint: disable=unused-argument
+        """Prepare the arguments for authentication to the
+        host filesystem"""
+        return {}
+
+    def _isdir(self, path_info):
+        return self.fs.isdir(self._with_bucket(path_info))
+
+    def isdir(self, path_info):
+        try:
+            return self._isdir(path_info)
+        except FileNotFoundError:
+            return False
+
+    def isfile(self, path_info):
+        try:
+            return not self._isdir(path_info)
+        except FileNotFoundError:
+            return False
+
+    def is_empty(self, path_info):
+        entry = self.info(path_info)
+        if entry["type"] == "directory":
+            return not self.fs.ls(self._with_bucket(path_info))
+        return entry["size"] == 0
+
+    def open(
+        self, path_info, mode="r", **kwargs
+    ):  # pylint: disable=arguments-differ
+        return self.fs.open(self._with_bucket(path_info), mode=mode)
+
+    def checksum(self, path_info):
+        return self.fs.checksum(self._with_bucket(path_info))
+
+    def copy(self, from_info, to_info):
+        self.makedirs(to_info.parent)
+        self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))
+
+    def exists(self, path_info) -> bool:
+        return self.fs.exists(self._with_bucket(path_info))
+
+    def ls(self, path_info, detail=False):
+        path = self._with_bucket(path_info)
+        files = self.fs.ls(path, detail=detail)
+        yield from self._strip_buckets(files, detail=detail)
+
+    # pylint: disable=unused-argument
+    def find(self, path_info, detail=False, prefix=None):
+        path = self._with_bucket(path_info)
+        files = self.fs.find(path, detail=detail)
+        if detail:
+            files = files.values()
+
+        yield from self._strip_buckets(files, detail=detail)
+
+    def walk_files(self, path_info, **kwargs):
+        for file in self.find(path_info, **kwargs):
+            yield path_info.replace(path=file)
+
+    def move(self, from_info, to_info):
+        self.fs.move(self._with_bucket(from_info), self._with_bucket(to_info))
+
+    def remove(self, path_info):
+        self.fs.rm_file(self._with_bucket(path_info))
+
+    def info(self, path_info):
+        info = self.fs.info(self._with_bucket(path_info)).copy()
+        info = self._entry_hook(info)
+        info["name"] = self._strip_bucket(info["name"])
+        return info
+
+    def makedirs(self, path_info, **kwargs):
+        self.fs.makedirs(
+            self._with_bucket(path_info), exist_ok=kwargs.pop("exist_ok", True)
+        )
+
+    def upload_fobj(self, fobj, to_info, **kwargs):
+        self.makedirs(to_info.parent)
+        with self.open(to_info, "wb") as fdest:
+            shutil.copyfileobj(fobj, fdest, length=fdest.blocksize)
+
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **kwargs
+    ):
+        self.makedirs(to_info.parent)
+        size = os.path.getsize(from_file)
+        with open(from_file, "rb") as fobj:
+            with Tqdm.wrapattr(
+                fobj,
+                "read",
+                disable=no_progress_bar,
+                bytes=True,
+                total=size,
+                desc=name,
+            ) as wrapped:
+                self.upload_fobj(wrapped, to_info, size=size)
+        self.fs.invalidate_cache(self._with_bucket(to_info.parent))
+
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **pbar_args
+    ):
+        total = self.getsize(self._with_bucket(from_info))
+        with self.open(from_info, "rb") as fobj:
+            with Tqdm.wrapattr(
+                fobj,
+                "read",
+                desc=name,
+                disable=no_progress_bar,
+                bytes=True,
+                total=total,
+                **pbar_args,
+            ) as wrapped:
+                with open(to_file, "wb") as fdest:
+                    shutil.copyfileobj(wrapped, fdest, length=fobj.blocksize)
+
+
+# pylint: disable=abstract-method
+class ObjectFSWrapper(FSSpecWrapper):
+    TRAVERSE_PREFIX_LEN = 3
+
+    @lru_cache(512)
+    def _with_bucket(self, path):
+        if isinstance(path, self.PATH_CLS):
+            return f"{path.bucket}/{path.path}"
+        return path
+
+    def _strip_bucket(self, entry):
+        try:
+            bucket, path = entry.split("/", 1)
+        except ValueError:
+            # If there is no path attached, only returns
+            # the bucket (top-level).
+            bucket, path = entry, None
+        return path or bucket
+
+    def makedirs(self, path_info, **kwargs):
+        # For object storages make this method a no-op. The original
+        # fs.makedirs() method will only check if the bucket exists
+        # and create if it doesn't though we don't want to support
+        # that behavior, and the check will cost some time so we'll
+        # simply ignore all mkdir()/makedirs() calls.
+        return None
+
+    def _isdir(self, path_info):
+        # Directory in object storages are interpreted differently
+        # among different fsspec providers, so this logic is a temporary
+        # measure for us to adapt as of now. It checks whether it is a
+        # directory (as in a prefix with contents) or whether it is an empty
+        # file where it's name ends with a forward slash
+
+        entry = self.info(path_info)
+        return entry["type"] == "directory" or (
+            entry["size"] == 0
+            and entry["type"] == "file"
+            and entry["name"].endswith("/")
+        )
+
+    def find(self, path_info, detail=False, prefix=None):
+        if prefix:
+            path = self._with_bucket(path_info.parent)
+            files = self.fs.find(
+                path, detail=detail, prefix=path_info.parts[-1]
+            )
+        else:
+            path = self._with_bucket(path_info)
+            files = self.fs.find(path, detail=detail)
+
+        if detail:
+            files = list(files.values())
+
+        # When calling find() on a file, it returns the same file in a list.
+        # For object-based storages, the same behavior applies to empty
+        # directories since they are represented as files. This condition
+        # checks whether we should yield an empty list (if it is an empty
+        # directory) or just yield the file itself.
+        if len(files) == 1 and files[0] == path and self.isdir(path_info):
+            return None
+
+        yield from self._strip_buckets(files, detail=detail)
+
+
+# pylint: disable=arguments-differ
+class NoDirectoriesMixin:
+    def isdir(self, *args, **kwargs):
+        return False
+
+    def isfile(self, *args, **kwargs):
+        return True
+
+    def find(self, *args, **kwargs):
+        raise NotImplementedError
+
+    def walk(self, *args, **kwargs):
+        raise NotImplementedError
+
+    def walk_files(self, *args, **kwargs):
+        raise NotImplementedError
+
+    def ls(self, *args, **kwargs):
+        raise NotImplementedError
+
+
+_LOCAL_FS = LocalFileSystem()
+
+
+class CallbackMixin:
+    """Use the native ``get_file()``/``put_file()`` APIs
+    if the target filesystem supports callbacks."""
+
+    def _upload(
+        self, from_file, to_info, name=None, no_progress_bar=False, **pbar_args
+    ):
+        with Tqdm(
+            desc=name,
+            disable=no_progress_bar,
+            bytes=True,
+            total=-1,
+            **pbar_args,
+        ) as pbar:
+            self.fs.put_file(
+                os.fspath(from_file),
+                self._with_bucket(to_info),
+                callback=pbar.as_callback(_LOCAL_FS, from_file),
+            )
+        self.fs.invalidate_cache(self._with_bucket(to_info.parent))
+
+    def _download(
+        self, from_info, to_file, name=None, no_progress_bar=False, **pbar_args
+    ):
+        with Tqdm(
+            desc=name,
+            disable=no_progress_bar,
+            bytes=True,
+            total=-1,
+            **pbar_args,
+        ) as pbar:
+            self.fs.get_file(
+                self._with_bucket(from_info),
+                os.fspath(to_file),
+                callback=pbar.as_callback(self, from_info),
+            )

dvc/fs/gdrive.py

@@ -0,0 +1,304 @@
+import logging
+import os
+import posixpath
+import re
+import threading
+from urllib.parse import urlparse
+
+from funcy import cached_property, retry, wrap_prop
+
+from dvc.exceptions import DvcException
+from dvc.path_info import CloudURLInfo
+from dvc.scheme import Schemes
+from dvc.utils import format_link, tmp_fname
+
+from .fsspec_wrapper import CallbackMixin, FSSpecWrapper
+
+logger = logging.getLogger(__name__)
+FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"
+
+
+class GDriveAuthError(DvcException):
+    def __init__(self, cred_location):
+
+        if cred_location:
+            message = (
+                "GDrive remote auth failed with credentials in '{}'.\n"
+                "Backup first, remove or fix them, and run DVC again.\n"
+                "It should do auth again and refresh the credentials.\n\n"
+                "Details:".format(cred_location)
+            )
+        else:
+            message = "Failed to authenticate GDrive remote"
+
+        super().__init__(message)
+
+
+def _gdrive_retry(func):
+    def should_retry(exc):
+        from pydrive2.files import ApiRequestError
+
+        if not isinstance(exc, ApiRequestError):
+            return False
+
+        error_code = exc.error.get("code", 0)
+        result = False
+        if 500 <= error_code < 600:
+            result = True
+
+        if error_code == 403:
+            result = exc.GetField("reason") in [
+                "userRateLimitExceeded",
+                "rateLimitExceeded",
+            ]
+        if result:
+            logger.debug(f"Retrying GDrive API call, error: {exc}.")
+
+        return result
+
+    # 16 tries, start at 0.5s, multiply by golden ratio, cap at 20s
+    return retry(
+        16,
+        timeout=lambda a: min(0.5 * 1.618 ** a, 20),
+        filter_errors=should_retry,
+    )(func)
+
+
+class GDriveURLInfo(CloudURLInfo):
+    def __init__(self, url):
+        super().__init__(url)
+
+        # GDrive URL host part is case sensitive,
+        # we are restoring it here.
+        p = urlparse(url)
+        self.host = p.netloc
+        assert self.netloc == self.host
+
+        # Normalize path. Important since we have a cache (path to ID)
+        # and don't want to deal with different variations of path in it.
+        self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
+
+
+class GDriveFileSystem(
+    CallbackMixin, FSSpecWrapper
+):  # pylint:disable=abstract-method
+    scheme = Schemes.GDRIVE
+    PATH_CLS = GDriveURLInfo
+    PARAM_CHECKSUM = "checksum"
+    REQUIRES = {"pydrive2": "pydrive2"}
+    # Always prefer traverse for GDrive since API usage quotas are a concern.
+    TRAVERSE_WEIGHT_MULTIPLIER = 1
+    TRAVERSE_PREFIX_LEN = 2
+
+    GDRIVE_CREDENTIALS_DATA = "GDRIVE_CREDENTIALS_DATA"
+    DEFAULT_USER_CREDENTIALS_FILE = "gdrive-user-credentials.json"
+
+    DEFAULT_GDRIVE_CLIENT_ID = "710796635688-iivsgbgsb6uv1fap6635dhvuei09o66c.apps.googleusercontent.com"  # noqa: E501
+    DEFAULT_GDRIVE_CLIENT_SECRET = "a1Fz59uTpVNeG_VGuSKDLJXv"
+
+    def __init__(self, **config):
+        super().__init__(**config)
+
+        self.path_info = self.PATH_CLS(config["url"])
+
+        if not self.path_info.bucket:
+            raise DvcException(
+                "Empty GDrive URL '{}'. Learn more at {}".format(
+                    config["url"],
+                    format_link("https://man.dvc.org/remote/add"),
+                )
+            )
+
+        self._bucket = self.path_info.bucket
+        self._path = self.path_info.path
+        self._trash_only = config.get("gdrive_trash_only")
+        self._use_service_account = config.get("gdrive_use_service_account")
+        self._service_account_user_email = config.get(
+            "gdrive_service_account_user_email"
+        )
+        self._service_account_json_file_path = config.get(
+            "gdrive_service_account_json_file_path"
+        )
+        self._client_id = config.get("gdrive_client_id")
+        self._client_secret = config.get("gdrive_client_secret")
+        self._validate_config()
+
+        tmp_dir = config["gdrive_credentials_tmp_dir"]
+        assert tmp_dir
+
+        self._gdrive_service_credentials_path = tmp_fname(
+            os.path.join(tmp_dir, "")
+        )
+        self._gdrive_user_credentials_path = (
+            tmp_fname(os.path.join(tmp_dir, ""))
+            if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
+            else config.get(
+                "gdrive_user_credentials_file",
+                os.path.join(tmp_dir, self.DEFAULT_USER_CREDENTIALS_FILE),
+            )
+        )
+
+    @staticmethod
+    def _get_kwargs_from_urls(urlpath):
+        return {"url": urlpath}
+
+    def _validate_config(self):
+        # Validate Service Account configuration
+        if (
+            self._use_service_account
+            and not self._service_account_json_file_path
+        ):
+            raise DvcException(
+                "To use service account, set "
+                "`gdrive_service_account_json_file_path`, and optionally"
+                "`gdrive_service_account_user_email` in DVC config\n"
+                "Learn more at {}".format(
+                    format_link("https://man.dvc.org/remote/modify")
+                )
+            )
+
+        # Validate OAuth 2.0 Client ID configuration
+        if not self._use_service_account:
+            if bool(self._client_id) != bool(self._client_secret):
+                raise DvcException(
+                    "Please specify GDrive's client ID and secret in "
+                    "DVC config or omit both to use the defaults.\n"
+                    "Learn more at {}".format(
+                        format_link("https://man.dvc.org/remote/modify")
+                    )
+                )
+
+    @cached_property
+    def credentials_location(self):
+        """
+        Helper to determine where will GDrive remote read credentials from.
+        Useful for tests, exception messages, etc. Returns either env variable
+        name if it's set or actual path to the credentials file.
+        """
+        if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
+            return GDriveFileSystem.GDRIVE_CREDENTIALS_DATA
+        if os.path.exists(self._gdrive_user_credentials_path):
+            return self._gdrive_user_credentials_path
+        return None
+
+    @staticmethod
+    def _validate_credentials(auth, settings):
+        """
+        Detects discrepancy in DVC config and cached credentials file.
+        Usually happens when a second remote is added and it is using
+        the same credentials default file. Or when someones decides to change
+        DVC config client id or secret but forgets to remove the cached
+        credentials file.
+        """
+        if not os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
+            if (
+                settings["client_config"]["client_id"]
+                != auth.credentials.client_id
+                or settings["client_config"]["client_secret"]
+                != auth.credentials.client_secret
+            ):
+                logger.warning(
+                    "Client ID and secret configured do not match the "
+                    "actual ones used\nto access the remote. Do you "
+                    "use multiple GDrive remotes and forgot to\nset "
+                    "`gdrive_user_credentials_file` for one or more of them? "
+                    "Learn more at\n{}.\n".format(
+                        format_link("https://man.dvc.org/remote/modify")
+                    )
+                )
+
+    @wrap_prop(threading.RLock())
+    @cached_property
+    def fs(self):
+        from pydrive2.auth import GoogleAuth
+        from pydrive2.fs import GDriveFileSystem as _GDriveFileSystem
+
+        temporary_save_path = self._gdrive_user_credentials_path
+        is_credentials_temp = os.getenv(
+            GDriveFileSystem.GDRIVE_CREDENTIALS_DATA
+        )
+        if self._use_service_account:
+            temporary_save_path = self._gdrive_service_credentials_path
+
+        if is_credentials_temp:
+            with open(temporary_save_path, "w") as cred_file:
+                cred_file.write(
+                    os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
+                )
+
+        auth_settings = {
+            "client_config_backend": "settings",
+            "save_credentials": True,
+            "save_credentials_backend": "file",
+            "save_credentials_file": self._gdrive_user_credentials_path,
+            "get_refresh_token": True,
+            "oauth_scope": [
+                "https://www.googleapis.com/auth/drive",
+                "https://www.googleapis.com/auth/drive.appdata",
+            ],
+        }
+
+        if self._use_service_account:
+            auth_settings["service_config"] = {
+                "client_user_email": self._service_account_user_email,
+                "client_json_file_path": self._service_account_json_file_path,
+            }
+            if is_credentials_temp:
+                auth_settings["service_config"][
+                    "client_json_file_path"
+                ] = temporary_save_path
+
+        else:
+            auth_settings["client_config"] = {
+                "client_id": self._client_id or self.DEFAULT_GDRIVE_CLIENT_ID,
+                "client_secret": self._client_secret
+                or self.DEFAULT_GDRIVE_CLIENT_SECRET,
+                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                "token_uri": "https://oauth2.googleapis.com/token",
+                "revoke_uri": "https://oauth2.googleapis.com/revoke",
+                "redirect_uri": "",
+            }
+
+        GoogleAuth.DEFAULT_SETTINGS.update(auth_settings)
+
+        # Pass non existent settings path to force DEFAULT_SETTINGS loadings
+        gauth = GoogleAuth(settings_file="")
+
+        try:
+            logger.debug(
+                "GDrive remote auth with config '{}'.".format(
+                    GoogleAuth.DEFAULT_SETTINGS
+                )
+            )
+            if self._use_service_account:
+                gauth.ServiceAuth()
+            else:
+                gauth.CommandLineAuth()
+                GDriveFileSystem._validate_credentials(gauth, auth_settings)
+
+        # Handle AuthenticationError, RefreshError and other auth failures
+        # It's hard to come up with a narrow exception, since PyDrive throws
+        # a lot of different errors - broken credentials file, refresh token
+        # expired, flow failed, etc.
+        except Exception as exc:
+            raise GDriveAuthError(self.credentials_location) from exc
+        finally:
+            if is_credentials_temp:
+                os.remove(temporary_save_path)
+
+        return _GDriveFileSystem(
+            self._with_bucket(self.path_info),
+            gauth,
+            trash_only=self._trash_only,
+        )
+
+    def _with_bucket(self, path):
+        if isinstance(path, self.PATH_CLS):
+            return posixpath.join(path.bucket, path.path)
+
+        return super()._with_bucket(path)
+
+    def upload_fobj(self, fobj, to_info, **kwargs):
+        rpath = self._with_bucket(to_info)
+        self.makedirs(os.path.dirname(rpath))
+        return self.fs.upload_fobj(fobj, rpath, **kwargs)