dvc: bump to 1.10.2

* parametrization: print pretty interpolation errors

* Fix col/loc in exception msg

* Fix tests

.codecov.yml

@@ -1 +1,10 @@
-
+comment: false
+coverage:
+  status:
+    project:
+      default:
+        target: 85
+        threshold: 10%
+    patch:
+      default:
+        threshold: 25%

.coveragerc

@@ -1,10 +1,14 @@
 [run]
+branch = True
+source = dvc
 omit =
     *__main__.py
 
 [report]
 exclude_lines =
     if __name__ == .__main__.:
+    if typing.TYPE_CHECKING:
+    if TYPE_CHECKING:
     pass
     raise NotImplementedError
     # pragma: no cover

.github/release-drafter.yml

@@ -6,6 +6,7 @@ categories:
     labels:
       - "feature"
       - "enhancement"
+      - "ui"
   - title: "💣 Breaking Changes "
     labels:
       - "breaking-change"

.github/workflows/check-patch.yaml

@@ -1,19 +0,0 @@
-name: Lint Python
-
-on: [push, pull_request]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.8
-    - name: Install requirements
-      run: bash ./scripts/ci/install.sh
-    - name: Check Patch
-      run: ./scripts/ci/check_patch.sh

.github/workflows/packages.yaml

@@ -0,0 +1,131 @@
+name: Build packages
+on:
+  push:
+    branches:
+    - master
+  release:
+    types: [released]
+jobs:
+  linux:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.7
+      uses: actions/[email protected]
+      with:
+        python-version: 3.7
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Build
+      run: ./scripts/build_posix.sh
+    - name: Publish deb
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_content_type: binary/octet-stream
+    - name: Publish rpm
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_content_type: binary/octet-stream
+  mac:
+    runs-on: macos-10.15
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.7
+      uses: actions/[email protected]
+      with:
+        python-version: 3.7
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Build
+      run: ./scripts/build_posix.sh
+    - name: Publish
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_name: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_content_type: binary/octet-stream
+  windows:
+    runs-on: windows-2019
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.7
+      uses: actions/[email protected]
+      with:
+        python-version: 3.7
+    - name: Build
+      run: scripts\build_windows.cmd
+      shell: cmd
+    - name: Publish
+      if: github.event_name == 'release'
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      uses: actions/[email protected]
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc-${{ github.event.release.tag_name }}.exe
+        asset_name: dvc-${{ github.event.release.tag_name }}.exe
+        asset_content_type: binary/octet-stream
+  python:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.7
+      uses: actions/[email protected]
+      with:
+        python-version: 3.7
+    - name: Install
+      run: pip install ".[all,tests]"
+    - name: Build packages
+      run: ./scripts/build_package.sh
+    - name: Publish packages to PyPI
+      if: github.event_name == 'release'
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_TOKEN }}
+  snap:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/[email protected]
+    - name: Build the package
+      uses: snapcore/action-build@v1
+      id: snapcraft
+    - name: Test the package
+      run: |
+        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
+        dvc version
+        sudo snap remove dvc
+    - name: Publish to the edge channel
+      uses: snapcore/action-publish@v1
+      if: github.event_name == 'push'
+      with:
+        store_login: ${{ secrets.STORE_LOGIN }}
+        snap: ${{ steps.snapcraft.outputs.snap }}
+        release: edge
+    - name: Publish to the stable channel
+      uses: snapcore/action-publish@v1
+      if: github.event_name == 'release'
+      with:
+        store_login: ${{ secrets.STORE_LOGIN }}
+        snap: ${{ steps.snapcraft.outputs.snap }}
+        release: stable

.github/workflows/tests.yaml

@@ -0,0 +1,86 @@
+name: Tests
+
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+
+env:
+  DVC_TEST: "true"
+  HOMEBREW_NO_AUTO_UPDATE: 1
+  SHELL: /bin/bash
+  GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Install requirements
+      run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
+    - name: Check README
+      run: python setup.py checkdocs
+    - name: Check formatting
+      run: pre-commit run --all-files
+  tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, windows-2019, macos-10.15]
+        pyv: ["3.6", "3.7", "3.8", "3.9"]
+        include:
+          - os: "macos-11.0"
+            pyv: "3.9"
+        exclude:
+          - os: macos-10.15
+            pyv: "3.9"
+
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python
+      uses: actions/[email protected]
+      with:
+        python-version: ${{ matrix.pyv }}
+    - name: install
+      run: |
+        pip install wheel
+        pip install -e ".[all,tests]"
+    - name: setup git
+      run: |
+        git config --global user.email "[email protected]"
+        git config --global user.name "DVC Tester"
+    - name: setup gs creds
+      shell: bash
+      if: env.GS_CREDS_JSON != ''
+      run: |
+        mkdir -p scripts/ci
+        echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
+    - name: run tests
+      run: python -m tests --all --cov-report=xml --cov-report=term --tap-combined
+    - name: upload coverage report
+      uses: codecov/[email protected]
+      with:
+        file: ./coverage.xml
+        fail_ci_if_error: false
+    - name: Post TAP
+      # Running custom fork with an option, that does not fail CI on error
+      uses: skshetry/flaky-service/packages/action@master
+      if: always() && github.ref == 'refs/heads/master'
+      with:
+        file-path: ${{github.workspace}}/testresults.tap
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        os: ${{runner.os}}
+        matrix: ${{toJson(matrix)}}
+        repo-description: ${{github.event.repository.description}}
+        fail_ci_if_error: false

.pylintrc

@@ -265,7 +265,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=pytest.lazy_fixture,logging.TRACE,logger.trace,sys.getwindowsversion
+generated-members=hdfs.*,pytest.lazy_fixture,logging.TRACE,logger.trace,sys.getwindowsversion
 
 # Tells whether missing members accessed in mixin class should be ignored. A
 # mixin class is detected if its name ends with "mixin" (case insensitive).

.restyled.yaml

@@ -1,20 +0,0 @@
----
-restylers:
-  - name: black
-    image: restyled/restyler-black:v19.10b0
-    command:
-    - black
-    arguments: []
-    include:
-    - "**/*.py"
-    interpreters:
-    - python
-  - name: isort
-    image: restyled/restyler-isort:v5.4.2
-    command:
-    - isort
-    arguments: []
-    include:
-    - "**/*.py"
-    interpreters:
-    - python

.travis.yml

@@ -1,162 +0,0 @@
-cache:
-  directories:
-    - $HOME/.cache/pip
-    - $HOME/.pyenv
-dist: xenial
-env:
-  global:
-    - HOMEBREW_NO_AUTO_UPDATE=1
-    - PYENV_ROOT="$HOME/.pyenv"
-    - PATH="$PYENV_ROOT/bin:$PATH"
-    - PIP_CACHE_DIR="$HOME/.cache/pip"  # unify pip cache location for all platforms
-stages:
-  - check
-  - test
-  - build
-# Travis doesn't support testing python on Mac yet, so we need
-# to workaround it by installing it directly with brew.
-#
-# Note that we specify language as ruby-2.4.3 to help us satisfy
-# fpm dependencies, as travis runs el capitan with an old ruby
-# version, that causes fpm to crash.
-#
-# Each job will use the previous job's `stage` tag if not specified.
-jobs:
-  include:
-  # patch formatting checks
-  - name: "Check patch formatting"
-    stage: check
-    os: linux
-    language: python
-    python: 3.7
-    before_install: bash ./scripts/ci/install.sh
-    install:
-    script: ./scripts/ci/check_patch.sh
-  # test jobs
-  - name: "3.7 on Windows"
-    stage: test
-    os: windows           # Windows 10.0.17134 N/A Build 17134
-    language: shell       # 'language: python' is an error on Travis CI Windows
-    env:
-      - PYTHON_VERSION=3.7.5
-      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
-  - name: "3.8 on Windows"
-    os: windows           # Windows 10.0.17134 N/A Build 17134
-    language: shell       # 'language: python' is an error on Travis CI Windows
-    env:
-      - PYTHON_VERSION=3.8.0
-      - PATH=/c/Python38:/c/Python38/Scripts:$PATH
-  - name: "3.6 on Linux"
-    language: python
-    python: 3.6
-  - name: "3.7 on Linux"
-    language: python
-    python: 3.7
-  - name: "3.8 on Linux"
-    language: python
-    python: 3.8
-  - name: "3.7 on Mac"
-    os: osx
-    osx_image: xcode11.3
-  # build jobs
-  - name: "Mac pkg"
-    stage: build
-    os: osx
-    osx_image: xcode9.4
-    language: ruby
-    rvm: 2.4.3
-    install:
-    script: ./scripts/build_posix.sh
-  - name: "Windows pkg"
-    os: windows
-    language: shell
-    install:
-    script: ./scripts/build_windows.cmd
-    env:
-      - PYTHON_VERSION=3.7.5
-      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
-  - name: "Linux pkgs"
-    language: python
-    python: 3.7
-    before_install:
-    install:
-    script: ./scripts/build_posix.sh
-  - name: "PyPI pkgs"
-    language: python
-    python: 3.8
-    script: ./scripts/build_package.sh
-  - name: Snapcraft snap
-    language: python
-    python: 3.8
-    addons:
-     snaps:
-     - name: snapcraft
-       channel: stable
-       confinement: classic
-     - name: review-tools
-       channel: stable
-     - name: lxd
-       channel: stable
-    env:
-    - SNAPCRAFT_IMAGE_INFO: |
-        '{"build_url": "$TRAVIS_JOB_WEB_URL"}'
-    - SNAPCRAFT_BUILD_ENVIRONMENT: lxd
-    - SNAPCRAFT_BUILD_INFO: 1  # https://snapcraft.io/blog/introducing-developer-notifications-for-snap-security-updates
-    install:
-    - sudo usermod --append --groups lxd $USER
-    - sudo /snap/bin/lxd.migrate -yes
-    - sudo /snap/bin/lxd waitready
-    - sudo /snap/bin/lxd init --auto
-    script:
-    - ./scripts/build_snap.sh
-    after_failure:
-    - sudo journalctl -u snapd
-
-before_install:
-  - bash ./scripts/ci/before_install.sh
-  - source env.sh
-  - bash ./scripts/ci/check_python.sh
-install:
-  - bash ./scripts/ci/install.sh
-script:
-  - bash ./scripts/ci/script.sh
-deploy:
-  - provider: releases
-    api_key:
-      secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=
-    file_glob: true
-    file:
-      - dvc*.rpm
-      - dvc*.deb
-      - dvc*.pkg
-      - dvc*.exe
-    skip_cleanup: true
-    on:
-      condition: "$(./scripts/ci/deploy_condition.sh dvc*.rpm dvc*.deb dvc*.pkg dvc*.exe)"
-      tags: true
-      repo: iterative/dvc
-      stage: build
-  - provider: pypi
-    user:
-      secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
-    password:
-      secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
-    skip_existing: true
-    file_glob: true
-    file:
-      - dist/dvc*.whl
-      - dist/dvc*.tar.gz
-    skip_cleanup: true
-    on:
-      condition: "$(./scripts/ci/deploy_condition.sh dist/dvc*.whl dist/dvc*.tar.gz)"
-      tags: true
-      repo: iterative/dvc
-      stage: build
-  - provider: script
-    skip_cleanup: true
-    script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out"
-    on:
-      all_branches: true
-      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL"
-      repo: iterative/dvc
-      stage: build

README.rst

@@ -262,9 +262,9 @@ Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
    :target: https://travis-ci.com/iterative/dvc/branches
    :alt: Release
 
-.. |CI| image:: https://img.shields.io/travis/com/iterative/dvc/master?label=dev&logo=travis
-   :target: https://travis-ci.com/iterative/dvc/builds
-   :alt: Travis dev branch
+.. |CI| image:: https://github.com/iterative/dvc/workflows/Tests/badge.svg?branch=master
+   :target: https://github.com/iterative/dvc/actions
+   :alt: GHA Tests
 
 .. |Maintainability| image:: https://codeclimate.com/github/iterative/dvc/badges/gpa.svg
    :target: https://codeclimate.com/github/iterative/dvc

dvc/api.py

@@ -20,7 +20,7 @@ def get_url(path, repo=None, rev=None, remote=None):
     in a DVC repo. For Git repos, HEAD is used unless a rev argument is
     supplied. The default remote is tried unless a remote argument is supplied.
 
-    Raises OutputNotFoundError if the file is not a dvc-tracked file.
+    Raises OutputNotFoundError if the file is not tracked by DVC.
 
     NOTE: This function does not check for the actual existence of the file or
     directory in the remote storage.
@@ -108,3 +108,35 @@ def _make_repo(repo_url=None, rev=None):
             pass  # fallthrough to external_repo
     with external_repo(url=repo_url, rev=rev) as repo:
         yield repo
+
+
+def make_checkpoint():
+    """
+    Signal DVC to create a checkpoint experiment.
+
+    If the current process is being run from DVC, this function will block
+    until DVC has finished creating the checkpoint. Otherwise, this function
+    will return immediately.
+    """
+    import builtins
+    from time import sleep
+
+    from dvc.env import DVC_CHECKPOINT, DVC_ROOT
+    from dvc.stage.run import CHECKPOINT_SIGNAL_FILE
+
+    if os.getenv(DVC_CHECKPOINT) is None:
+        return
+
+    root_dir = os.getenv(DVC_ROOT, Repo.find_root())
+    signal_file = os.path.join(
+        root_dir, Repo.DVC_DIR, "tmp", CHECKPOINT_SIGNAL_FILE
+    )
+
+    with builtins.open(signal_file, "w") as fobj:
+        # NOTE: force flushing/writing empty file to disk, otherwise when
+        # run in certain contexts (pytest) file may not actually be written
+        fobj.write("")
+        fobj.flush()
+        os.fsync(fobj.fileno())
+    while os.path.exists(signal_file):
+        sleep(1)

dvc/cache/__init__.py

@@ -1,45 +1,31 @@
 """Manages cache of a DVC repo."""
 from collections import defaultdict
 
-from funcy import cached_property
+from ..scheme import Schemes
 
 
-def _make_remote_property(name):
-    """
-    The config file is stored in a way that allows you to have a
-    cache for each remote.
+def get_cloud_cache(tree):
+    from .base import CloudCache
+    from .local import LocalCache
+    from .ssh import SSHCache
 
-    This is needed when specifying external outputs
-    (as they require you to have an external cache location).
+    if tree.scheme == Schemes.LOCAL:
+        return LocalCache(tree)
 
-    Imagine a config file like the following:
+    if tree.scheme == Schemes.SSH:
+        return SSHCache(tree)
 
-            ['remote "dvc-storage"']
-            url = ssh://localhost/tmp
-            ask_password = true
+    return CloudCache(tree)
 
-            [cache]
-            ssh = dvc-storage
 
-    This method creates a cached property, containing cache named `name`:
+def _get_cache(repo, settings):
+    from ..tree import get_cloud_tree
 
-        self.config == {'ssh': 'dvc-storage'}
-        self.ssh  # a RemoteSSH instance
-    """
+    if not settings:
+        return None
 
-    def getter(self):
-        from ..tree import get_cloud_tree
-        from .base import CloudCache
-
-        remote = self.config.get(name)
-        if not remote:
-            return None
-
-        tree = get_cloud_tree(self.repo, name=remote)
-        return CloudCache(tree)
-
-    getter.__name__ = name
-    return cached_property(getter)
+    tree = get_cloud_tree(repo, **settings)
+    return get_cloud_cache(tree)
 
 
 class Cache:
@@ -50,18 +36,25 @@ class Cache:
     """
 
     CACHE_DIR = "cache"
+    CLOUD_SCHEMES = [
+        Schemes.S3,
+        Schemes.GS,
+        Schemes.SSH,
+        Schemes.HDFS,
+        Schemes.WEBHDFS,
+    ]
 
     def __init__(self, repo):
-        from ..tree import get_cloud_tree
-        from .local import LocalCache
-
         self.repo = repo
         self.config = config = repo.config["cache"]
+        self._cache = {}
 
         local = config.get("local")
 
         if local:
             settings = {"name": local}
+        elif "dir" not in config:
+            settings = None
         else:
             from ..config import LOCAL_COMMON
 
@@ -70,14 +63,21 @@ class Cache:
                 if opt in config:
                     settings[str(opt)] = config.get(opt)
 
-        tree = get_cloud_tree(repo, **settings)
-        self.local = LocalCache(tree)
+        self._cache[Schemes.LOCAL] = _get_cache(repo, settings)
 
-    s3 = _make_remote_property("s3")
-    gs = _make_remote_property("gs")
-    ssh = _make_remote_property("ssh")
-    hdfs = _make_remote_property("hdfs")
-    azure = _make_remote_property("azure")
+        for scheme in self.CLOUD_SCHEMES:
+            remote = self.config.get(scheme)
+            settings = {"name": remote} if remote else None
+            self._cache[scheme] = _get_cache(repo, settings)
+
+    def __getattr__(self, name):
+        try:
+            return self._cache[name]
+        except KeyError as exc:
+            raise AttributeError from exc
+
+    def by_scheme(self):
+        yield from self._cache.items()
 
 
 class NamedCacheItem:

dvc/cache/base.py

@@ -1,39 +1,30 @@
+import itertools
 import json
 import logging
+from concurrent.futures import ThreadPoolExecutor
 from copy import copy
-from operator import itemgetter
+from typing import Optional
 
+from funcy import decorator
 from shortuuid import uuid
 
 import dvc.prompt as prompt
+from dvc.dir_info import DirInfo
 from dvc.exceptions import (
+    CacheLinkError,
     CheckoutError,
     ConfirmRemoveError,
     DvcException,
-    MergeError,
 )
-from dvc.hash_info import HashInfo
-from dvc.path_info import WindowsPathInfo
 from dvc.progress import Tqdm
-from dvc.remote.slow_link_detection import slow_link_guard
+from dvc.remote.slow_link_detection import (  # type: ignore[attr-defined]
+    slow_link_guard,
+)
 
 from ..tree.base import RemoteActionNotImplemented
 
 logger = logging.getLogger(__name__)
 
-STATUS_OK = 1
-STATUS_MISSING = 2
-STATUS_NEW = 3
-STATUS_DELETED = 4
-
-STATUS_MAP = {
-    # (local_exists, remote_exists)
-    (True, True): STATUS_OK,
-    (False, False): STATUS_MISSING,
-    (True, False): STATUS_NEW,
-    (False, True): STATUS_DELETED,
-}
-
 
 class DirCacheError(DvcException):
     def __init__(self, hash_):
@@ -42,11 +33,18 @@ class DirCacheError(DvcException):
         )
 
 
+@decorator
+def use_state(call):
+    tree = call._args[0].tree  # pylint: disable=protected-access
+    with tree.state:
+        return call()
+
+
 class CloudCache:
     """Cloud cache class."""
 
     DEFAULT_CACHE_TYPES = ["copy"]
-    CACHE_MODE = None
+    CACHE_MODE: Optional[int] = None
 
     def __init__(self, tree):
         self.tree = tree
@@ -68,7 +66,7 @@ class CloudCache:
         try:
             dir_info = self.load_dir_cache(hash_info)
         except DirCacheError:
-            dir_info = []
+            dir_info = DirInfo()
 
         self._dir_info[hash_info.value] = dir_info
         return dir_info
@@ -87,18 +85,9 @@ class CloudCache:
                 "dir cache file format error '%s' [skipping the file]",
                 path_info,
             )
-            return []
+            d = []
 
-        if self.tree.PATH_CLS == WindowsPathInfo:
-            # only need to convert it for Windows
-            for info in d:
-                # NOTE: here is a BUG, see comment to .as_posix() below
-                relpath = info[self.tree.PARAM_RELPATH]
-                info[self.tree.PARAM_RELPATH] = relpath.replace(
-                    "/", self.tree.PATH_CLS.sep
-                )
-
-        return d
+        return DirInfo.from_list(d)
 
     def changed(self, path_info, hash_info):
         """Checks if data has changed.
@@ -184,7 +173,7 @@ class CloudCache:
                 )
                 del link_types[0]
 
-        raise DvcException("no possible cache types left to try out.")
+        raise CacheLinkError([to_info])
 
     def _do_link(self, from_info, to_info, link_method):
         if self.tree.exists(to_info):
@@ -218,7 +207,7 @@ class CloudCache:
             # we need to update path and cache, since in case of reflink,
             # or copy cache type moving original file results in updates on
             # next executed command, which causes md5 recalculation
-            self.tree.state.save(path_info, hash_info.value)
+            self.tree.state.save(path_info, hash_info)
         else:
             if self.changed_cache(hash_info):
                 with tree.open(path_info, mode="rb") as fobj:
@@ -232,7 +221,7 @@ class CloudCache:
                 if callback:
                     callback(1)
 
-        self.tree.state.save(cache_info, hash_info.value)
+        self.tree.state.save(cache_info, hash_info)
 
     def _cache_is_copy(self, path_info):
         """Checks whether cache uses copies."""
@@ -262,12 +251,9 @@ class CloudCache:
         from dvc.path_info import PathInfo
         from dvc.utils import tmp_fname
 
-        # Sorting the list by path to ensure reproducibility
-        dir_info = sorted(dir_info, key=itemgetter(self.tree.PARAM_RELPATH))
-
         tmp = tempfile.NamedTemporaryFile(delete=False).name
         with open(tmp, "w+") as fobj:
-            json.dump(dir_info, fobj, sort_keys=True)
+            json.dump(dir_info.to_list(), fobj, sort_keys=True)
 
         from_info = PathInfo(tmp)
         to_info = self.tree.path_info / tmp_fname("")
@@ -276,9 +262,11 @@ class CloudCache:
         hash_info = self.tree.get_file_hash(to_info)
         hash_info.value += self.tree.CHECKSUM_DIR_SUFFIX
         hash_info.dir_info = dir_info
+        hash_info.nfiles = dir_info.nfiles
 
         return hash_info, to_info
 
+    @use_state
     def save_dir_info(self, dir_info, hash_info=None):
         if (
             hash_info
@@ -293,7 +281,7 @@ class CloudCache:
             self.tree.makedirs(new_info.parent)
             self.tree.move(tmp_info, new_info, mode=self.CACHE_MODE)
 
-        self.tree.state.save(new_info, hi.value)
+        self.tree.state.save(new_info, hi)
 
         return hi
 
@@ -301,13 +289,11 @@ class CloudCache:
         if not hash_info.dir_info:
             hash_info.dir_info = tree.cache.get_dir_cache(hash_info)
         hi = self.save_dir_info(hash_info.dir_info, hash_info)
-        for entry in Tqdm(
-            hi.dir_info, desc="Saving " + path_info.name, unit="file"
+        for entry_info, entry_hash in Tqdm(
+            hi.dir_info.items(path_info),
+            desc="Saving " + path_info.name,
+            unit="file",
         ):
-            entry_info = path_info / entry[self.tree.PARAM_RELPATH]
-            entry_hash = HashInfo(
-                self.tree.PARAM_CHECKSUM, entry[self.tree.PARAM_CHECKSUM]
-            )
             self._save_file(
                 entry_info, tree, entry_hash, save_link=False, **kwargs
             )
@@ -315,11 +301,12 @@ class CloudCache:
         if save_link:
             self.tree.state.save_link(path_info)
         if self.tree.exists(path_info):
-            self.tree.state.save(path_info, hi.value)
+            self.tree.state.save(path_info, hi)
 
         cache_info = self.tree.hash_to_path_info(hi.value)
-        self.tree.state.save(cache_info, hi.value)
+        self.tree.state.save(cache_info, hi)
 
+    @use_state
     def save(self, path_info, tree, hash_info, save_link=True, **kwargs):
         if path_info.scheme != self.tree.scheme:
             raise RemoteActionNotImplemented(
@@ -391,13 +378,9 @@ class CloudCache:
         if self.changed_cache_file(hash_info):
             return True
 
-        for entry in self.get_dir_cache(hash_info):
-            entry_hash = HashInfo(
-                self.tree.PARAM_CHECKSUM, entry[self.tree.PARAM_CHECKSUM]
-            )
-
+        dir_info = self.get_dir_cache(hash_info)
+        for entry_info, entry_hash in dir_info.items(path_info):
             if path_info and filter_info:
-                entry_info = path_info / entry[self.tree.PARAM_RELPATH]
                 if not entry_info.isin_or_eq(filter_info):
                     continue
 
@@ -449,7 +432,7 @@ class CloudCache:
 
         self.link(cache_info, path_info)
         self.tree.state.save_link(path_info)
-        self.tree.state.save(path_info, hash_info.value)
+        self.tree.state.save(path_info, hash_info)
         if progress_callback:
             progress_callback(str(path_info))
 
@@ -475,21 +458,19 @@ class CloudCache:
 
         logger.debug("Linking directory '%s'.", path_info)
 
-        for entry in dir_info:
-            relative_path = entry[self.tree.PARAM_RELPATH]
-            entry_hash = entry[self.tree.PARAM_CHECKSUM]
-            entry_cache_info = self.tree.hash_to_path_info(entry_hash)
-            entry_info = path_info / relative_path
+        for entry_info, entry_hash_info in dir_info.items(path_info):
+            entry_cache_info = self.tree.hash_to_path_info(
+                entry_hash_info.value
+            )
 
             if filter_info and not entry_info.isin_or_eq(filter_info):
                 continue
 
-            entry_hash_info = HashInfo(self.tree.PARAM_CHECKSUM, entry_hash)
             if relink or self.changed(entry_info, entry_hash_info):
                 modified = True
                 self.safe_remove(entry_info, force=force)
                 self.link(entry_cache_info, entry_info)
-                self.tree.state.save(entry_info, entry_hash)
+                self.tree.state.save(entry_info, entry_hash_info)
             if progress_callback:
                 progress_callback(str(entry_info))
 
@@ -499,7 +480,7 @@ class CloudCache:
         )
 
         self.tree.state.save_link(path_info)
-        self.tree.state.save(path_info, hash_info.value)
+        self.tree.state.save(path_info, hash_info)
 
         # relink is not modified, assume it as nochange
         return added, not added and modified and not relink
@@ -507,15 +488,14 @@ class CloudCache:
     def _remove_redundant_files(self, path_info, dir_info, force):
         existing_files = set(self.tree.walk_files(path_info))
 
-        needed_files = {
-            path_info / entry[self.tree.PARAM_RELPATH] for entry in dir_info
-        }
+        needed_files = {info for info, _ in dir_info.items(path_info)}
         redundant_files = existing_files - needed_files
         for path in redundant_files:
             self.safe_remove(path, force)
 
         return bool(redundant_files)
 
+    @use_state
     def checkout(
         self,
         path_info,
@@ -524,6 +504,7 @@ class CloudCache:
         progress_callback=None,
         relink=False,
         filter_info=None,
+        quiet=False,
     ):
         if path_info.scheme not in ["local", self.tree.scheme]:
             raise NotImplementedError
@@ -531,10 +512,11 @@ class CloudCache:
         failed = None
         skip = False
         if not hash_info:
-            logger.warning(
-                "No file hash info found for '%s'. " "It won't be created.",
-                path_info,
-            )
+            if not quiet:
+                logger.warning(
+                    "No file hash info found for '%s'. It won't be created.",
+                    path_info,
+                )
             self.safe_remove(path_info, force=force)
             failed = path_info
 
@@ -545,11 +527,12 @@ class CloudCache:
         elif self.changed_cache(
             hash_info, path_info=path_info, filter_info=filter_info
         ):
-            logger.warning(
-                "Cache '%s' not found. File '%s' won't be created.",
-                hash_info,
-                path_info,
-            )
+            if not quiet:
+                logger.warning(
+                    "Cache '%s' not found. File '%s' won't be created.",
+                    hash_info,
+                    path_info,
+                )
             self.safe_remove(path_info, force=force)
             failed = path_info
 
@@ -606,69 +589,21 @@ class CloudCache:
             return 1
 
         if not filter_info:
-            return len(self.get_dir_cache(hash_info))
+            return self.get_dir_cache(hash_info).nfiles
 
         return ilen(
-            filter_info.isin_or_eq(path_info / entry[self.tree.PARAM_CHECKSUM])
-            for entry in self.get_dir_cache(hash_info)
+            filter_info.isin_or_eq(path_info / relpath)
+            for relpath, _ in self.get_dir_cache(hash_info).items()
         )
 
-    def _to_dict(self, dir_info):
-        return {
-            entry[self.tree.PARAM_RELPATH]: entry[self.tree.PARAM_CHECKSUM]
-            for entry in dir_info
-        }
-
-    def _from_dict(self, dir_dict):
-        return [
-            {
-                self.tree.PARAM_RELPATH: relpath,
-                self.tree.PARAM_CHECKSUM: checksum,
-            }
-            for relpath, checksum in dir_dict.items()
-        ]
-
-    @staticmethod
-    def _diff(ancestor, other, allow_removed=False):
-        from dictdiffer import diff
-
-        allowed = ["add"]
-        if allow_removed:
-            allowed.append("remove")
-
-        result = list(diff(ancestor, other))
-        for typ, _, _ in result:
-            if typ not in allowed:
-                raise MergeError(
-                    "unable to auto-merge directories with diff that contains "
-                    f"'{typ}'ed files"
-                )
-        return result
-
-    def _merge_dirs(self, ancestor_info, our_info, their_info):
-        from dictdiffer import patch
-
-        ancestor = self._to_dict(ancestor_info)
-        our = self._to_dict(our_info)
-        their = self._to_dict(their_info)
-
-        our_diff = self._diff(ancestor, our)
-        if not our_diff:
-            return self._from_dict(their)
-
-        their_diff = self._diff(ancestor, their)
-        if not their_diff:
-            return self._from_dict(our)
-
-        # make sure there are no conflicting files
-        self._diff(our, their, allow_removed=True)
-
-        merged = patch(our_diff + their_diff, ancestor, in_place=True)
-
-        # Sorting the list by path to ensure reproducibility
-        return sorted(
-            self._from_dict(merged), key=itemgetter(self.tree.PARAM_RELPATH)
-        )
+    def _get_dir_size(self, dir_info):
+        try:
+            return sum(
+                self.tree.getsize(self.tree.hash_to_path_info(hi.value))
+                for _, hi in dir_info.items()
+            )
+        except FileNotFoundError:
+            return None
 
     def merge(self, ancestor_info, our_info, their_info):
         assert our_info
@@ -677,18 +612,326 @@ class CloudCache:
         if ancestor_info:
             ancestor = self.get_dir_cache(ancestor_info)
         else:
-            ancestor = []
+            ancestor = DirInfo()
 
         our = self.get_dir_cache(our_info)
         their = self.get_dir_cache(their_info)
 
-        merged = self._merge_dirs(ancestor, our, their)
-        return self.save_dir_info(merged)
+        merged = our.merge(ancestor, their)
+        hash_info = self.save_dir_info(merged)
+        hash_info.size = self._get_dir_size(merged)
+        return hash_info
 
+    @use_state
     def get_hash(self, tree, path_info):
         hash_info = tree.get_hash(path_info)
         if not hash_info.isdir:
             assert hash_info.name == self.tree.PARAM_CHECKSUM
             return hash_info
 
-        return self.save_dir_info(hash_info.dir_info, hash_info)
+        hi = self.save_dir_info(hash_info.dir_info, hash_info)
+        hi.size = hash_info.size
+        return hi
+
+    def set_dir_info(self, hash_info):
+        assert hash_info.isdir
+
+        hash_info.dir_info = self.get_dir_cache(hash_info)
+        hash_info.nfiles = hash_info.dir_info.nfiles
+
+    def _list_paths(self, prefix=None, progress_callback=None):
+        if prefix:
+            if len(prefix) > 2:
+                path_info = self.tree.path_info / prefix[:2] / prefix[2:]
+            else:
+                path_info = self.tree.path_info / prefix[:2]
+            prefix = True
+        else:
+            path_info = self.tree.path_info
+            prefix = False
+        if progress_callback:
+            for file_info in self.tree.walk_files(path_info, prefix=prefix):
+                progress_callback()
+                yield file_info.path
+        else:
+            yield from self.tree.walk_files(path_info, prefix=prefix)
+
+    def _path_to_hash(self, path):
+        parts = self.tree.PATH_CLS(path).parts[-2:]
+
+        if not (len(parts) == 2 and parts[0] and len(parts[0]) == 2):
+            raise ValueError(f"Bad cache file path '{path}'")
+
+        return "".join(parts)
+
+    def list_hashes(self, prefix=None, progress_callback=None):
+        """Iterate over hashes in this tree.
+
+        If `prefix` is specified, only hashes which begin with `prefix`
+        will be returned.
+        """
+        for path in self._list_paths(prefix, progress_callback):
+            try:
+                yield self._path_to_hash(path)
+            except ValueError:
+                logger.debug(
+                    "'%s' doesn't look like a cache file, skipping", path
+                )
+
+    def _hashes_with_limit(self, limit, prefix=None, progress_callback=None):
+        count = 0
+        for hash_ in self.list_hashes(prefix, progress_callback):
+            yield hash_
+            count += 1
+            if count > limit:
+                logger.debug(
+                    "`list_hashes()` returned max '{}' hashes, "
+                    "skipping remaining results".format(limit)
+                )
+                return
+
+    def _max_estimation_size(self, hashes):
+        # Max remote size allowed for us to use traverse method
+        return max(
+            self.tree.TRAVERSE_THRESHOLD_SIZE,
+            len(hashes)
+            / self.tree.TRAVERSE_WEIGHT_MULTIPLIER
+            * self.tree.LIST_OBJECT_PAGE_SIZE,
+        )
+
+    def _estimate_remote_size(self, hashes=None, name=None):
+        """Estimate tree size based on number of entries beginning with
+        "00..." prefix.
+        """
+        prefix = "0" * self.tree.TRAVERSE_PREFIX_LEN
+        total_prefixes = pow(16, self.tree.TRAVERSE_PREFIX_LEN)
+        if hashes:
+            max_hashes = self._max_estimation_size(hashes)
+        else:
+            max_hashes = None
+
+        with Tqdm(
+            desc="Estimating size of "
+            + (f"cache in '{name}'" if name else "remote cache"),
+            unit="file",
+        ) as pbar:
+
+            def update(n=1):
+                pbar.update(n * total_prefixes)
+
+            if max_hashes:
+                hashes = self._hashes_with_limit(
+                    max_hashes / total_prefixes, prefix, update
+                )
+            else:
+                hashes = self.list_hashes(prefix, update)
+
+            remote_hashes = set(hashes)
+            if remote_hashes:
+                remote_size = total_prefixes * len(remote_hashes)
+            else:
+                remote_size = total_prefixes
+            logger.debug(f"Estimated remote size: {remote_size} files")
+        return remote_size, remote_hashes
+
+    def list_hashes_traverse(
+        self, remote_size, remote_hashes, jobs=None, name=None
+    ):
+        """Iterate over all hashes found in this tree.
+        Hashes are fetched in parallel according to prefix, except in
+        cases where the remote size is very small.
+
+        All hashes from the remote (including any from the size
+        estimation step passed via the `remote_hashes` argument) will be
+        returned.
+
+        NOTE: For large remotes the list of hashes will be very
+        big(e.g. 100M entries, md5 for each is 32 bytes, so ~3200Mb list)
+        and we don't really need all of it at the same time, so it makes
+        sense to use a generator to gradually iterate over it, without
+        keeping all of it in memory.
+        """
+        num_pages = remote_size / self.tree.LIST_OBJECT_PAGE_SIZE
+        if num_pages < 256 / self.tree.JOBS:
+            # Fetching prefixes in parallel requires at least 255 more
+            # requests, for small enough remotes it will be faster to fetch
+            # entire cache without splitting it into prefixes.
+            #
+            # NOTE: this ends up re-fetching hashes that were already
+            # fetched during remote size estimation
+            traverse_prefixes = [None]
+            initial = 0
+        else:
+            yield from remote_hashes
+            initial = len(remote_hashes)
+            traverse_prefixes = [f"{i:02x}" for i in range(1, 256)]
+            if self.tree.TRAVERSE_PREFIX_LEN > 2:
+                traverse_prefixes += [
+                    "{0:0{1}x}".format(i, self.tree.TRAVERSE_PREFIX_LEN)
+                    for i in range(
+                        1, pow(16, self.tree.TRAVERSE_PREFIX_LEN - 2)
+                    )
+                ]
+        with Tqdm(
+            desc="Querying "
+            + (f"cache in '{name}'" if name else "remote cache"),
+            total=remote_size,
+            initial=initial,
+            unit="file",
+        ) as pbar:
+
+            def list_with_update(prefix):
+                return list(
+                    self.list_hashes(
+                        prefix=prefix, progress_callback=pbar.update
+                    )
+                )
+
+            with ThreadPoolExecutor(
+                max_workers=jobs or self.tree.JOBS
+            ) as executor:
+                in_remote = executor.map(list_with_update, traverse_prefixes,)
+                yield from itertools.chain.from_iterable(in_remote)
+
+    def all(self, jobs=None, name=None):
+        """Iterate over all hashes in this tree.
+
+        Hashes will be fetched in parallel threads according to prefix
+        (except for small remotes) and a progress bar will be displayed.
+        """
+        logger.debug(
+            "Fetching all hashes from '{}'".format(
+                name if name else "remote cache"
+            )
+        )
+
+        if not self.tree.CAN_TRAVERSE:
+            return self.list_hashes()
+
+        remote_size, remote_hashes = self._estimate_remote_size(name=name)
+        return self.list_hashes_traverse(
+            remote_size, remote_hashes, jobs, name
+        )
+
+    def _remove_unpacked_dir(self, hash_):
+        pass
+
+    def gc(self, used, jobs=None):
+        removed = False
+        # hashes must be sorted to ensure we always remove .dir files first
+        for hash_ in sorted(
+            self.all(jobs, str(self.tree.path_info)),
+            key=self.tree.is_dir_hash,
+            reverse=True,
+        ):
+            if hash_ in used:
+                continue
+            path_info = self.tree.hash_to_path_info(hash_)
+            if self.tree.is_dir_hash(hash_):
+                # backward compatibility
+                # pylint: disable=protected-access
+                self._remove_unpacked_dir(hash_)
+            self.tree.remove(path_info)
+            removed = True
+
+        return removed
+
+    def list_hashes_exists(self, hashes, jobs=None, name=None):
+        """Return list of the specified hashes which exist in this tree.
+        Hashes will be queried individually.
+        """
+        logger.debug(
+            "Querying {} hashes via object_exists".format(len(hashes))
+        )
+        with Tqdm(
+            desc="Querying "
+            + ("cache in " + name if name else "remote cache"),
+            total=len(hashes),
+            unit="file",
+        ) as pbar:
+
+            def exists_with_progress(path_info):
+                ret = self.tree.exists(path_info)
+                pbar.update_msg(str(path_info))
+                return ret
+
+            with ThreadPoolExecutor(
+                max_workers=jobs or self.tree.JOBS
+            ) as executor:
+                path_infos = map(self.tree.hash_to_path_info, hashes)
+                in_remote = executor.map(exists_with_progress, path_infos)
+                ret = list(itertools.compress(hashes, in_remote))
+                return ret
+
+    def hashes_exist(self, hashes, jobs=None, name=None):
+        """Check if the given hashes are stored in the remote.
+
+        There are two ways of performing this check:
+
+        - Traverse method: Get a list of all the files in the remote
+            (traversing the cache directory) and compare it with
+            the given hashes. Cache entries will be retrieved in parallel
+            threads according to prefix (i.e. entries starting with, "00...",
+            "01...", and so on) and a progress bar will be displayed.
+
+        - Exists method: For each given hash, run the `exists`
+            method and filter the hashes that aren't on the remote.
+            This is done in parallel threads.
+            It also shows a progress bar when performing the check.
+
+        The reason for such an odd logic is that most of the remotes
+        take much shorter time to just retrieve everything they have under
+        a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can
+        check if particular file exists much quicker, use their own
+        implementation of hashes_exist (see ssh, local).
+
+        Which method to use will be automatically determined after estimating
+        the size of the remote cache, and comparing the estimated size with
+        len(hashes). To estimate the size of the remote cache, we fetch
+        a small subset of cache entries (i.e. entries starting with "00...").
+        Based on the number of entries in that subset, the size of the full
+        cache can be estimated, since the cache is evenly distributed according
+        to hash.
+
+        Returns:
+            A list with hashes that were found in the remote
+        """
+        # Remotes which do not use traverse prefix should override
+        # hashes_exist() (see ssh, local)
+        assert self.tree.TRAVERSE_PREFIX_LEN >= 2
+
+        hashes = set(hashes)
+        if len(hashes) == 1 or not self.tree.CAN_TRAVERSE:
+            remote_hashes = self.list_hashes_exists(hashes, jobs, name)
+            return remote_hashes
+
+        # Max remote size allowed for us to use traverse method
+        remote_size, remote_hashes = self._estimate_remote_size(hashes, name)
+
+        traverse_pages = remote_size / self.tree.LIST_OBJECT_PAGE_SIZE
+        # For sufficiently large remotes, traverse must be weighted to account
+        # for performance overhead from large lists/sets.
+        # From testing with S3, for remotes with 1M+ files, object_exists is
+        # faster until len(hashes) is at least 10k~100k
+        if remote_size > self.tree.TRAVERSE_THRESHOLD_SIZE:
+            traverse_weight = (
+                traverse_pages * self.tree.TRAVERSE_WEIGHT_MULTIPLIER
+            )
+        else:
+            traverse_weight = traverse_pages
+        if len(hashes) < traverse_weight:
+            logger.debug(
+                "Large remote ('{}' hashes < '{}' traverse weight), "
+                "using object_exists for remaining hashes".format(
+                    len(hashes), traverse_weight
+                )
+            )
+            return list(hashes & remote_hashes) + self.list_hashes_exists(
+                hashes - remote_hashes, jobs, name
+            )
+
+        logger.debug("Querying '{}' hashes via traverse".format(len(hashes)))
+        remote_hashes = set(
+            self.list_hashes_traverse(remote_size, remote_hashes, jobs, name)
+        )
+        return list(hashes & set(remote_hashes))

dvc/cache/local.py

@@ -1,53 +1,23 @@
-import errno
 import logging
 import os
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from functools import partial, wraps
 
-from funcy import cached_property, concat
+from funcy import cached_property
 
-from dvc.exceptions import DownloadError, UploadError
 from dvc.hash_info import HashInfo
 from dvc.path_info import PathInfo
 from dvc.progress import Tqdm
 
-from ..remote.base import index_locked
 from ..tree.local import LocalTree
-from .base import (
-    STATUS_DELETED,
-    STATUS_MAP,
-    STATUS_MISSING,
-    STATUS_NEW,
-    CloudCache,
-)
+from ..utils.fs import walk_files
+from .base import CloudCache
 
 logger = logging.getLogger(__name__)
 
 
-def _log_exceptions(func, operation):
-    @wraps(func)
-    def wrapper(from_info, to_info, *args, **kwargs):
-        try:
-            func(from_info, to_info, *args, **kwargs)
-            return 0
-        except Exception as exc:  # pylint: disable=broad-except
-            # NOTE: this means we ran out of file descriptors and there is no
-            # reason to try to proceed, as we will hit this error anyways.
-            # pylint: disable=no-member
-            if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
-                raise
-
-            logger.exception(
-                "failed to %s '%s' to '%s'", operation, from_info, to_info
-            )
-            return 1
-
-    return wrapper
-
-
 class LocalCache(CloudCache):
     DEFAULT_CACHE_TYPES = ["reflink", "copy"]
     CACHE_MODE = LocalTree.CACHE_MODE
+    UNPACKED_DIR_SUFFIX = ".unpacked"
 
     def __init__(self, tree):
         super().__init__(tree)
@@ -61,10 +31,6 @@ class LocalCache(CloudCache):
     def cache_dir(self, value):
         self.tree.path_info = PathInfo(value) if value else None
 
-    @classmethod
-    def supported(cls, config):  # pylint: disable=unused-argument
-        return True
-
     @cached_property
     def cache_path(self):
         return os.path.abspath(self.cache_dir)
@@ -102,371 +68,24 @@ class LocalCache(CloudCache):
 
         super()._verify_link(path_info, link_type)
 
-    @index_locked
-    def status(
-        self,
-        named_cache,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-        log_missing=True,
-    ):
-        # Return flattened dict containing all status info
-        dir_status, file_status, _ = self._status(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=download,
-            log_missing=log_missing,
-        )
-        return dict(dir_status, **file_status)
-
-    def _status(
-        self,
-        named_cache,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-        log_missing=True,
-    ):
-        """Return a tuple of (dir_status_info, file_status_info, dir_contents).
-
-        dir_status_info contains status for .dir files, file_status_info
-        contains status for all other files, and dir_contents is a dict of
-        {dir_hash: set(file_hash, ...)} which can be used to map
-        a .dir file to its file contents.
-        """
-        logger.debug(
-            f"Preparing to collect status from {remote.tree.path_info}"
-        )
-        md5s = set(named_cache.scheme_keys(self.tree.scheme))
-
-        logger.debug("Collecting information from local cache...")
-        local_exists = frozenset(
-            self.hashes_exist(md5s, jobs=jobs, name=self.cache_dir)
-        )
-
-        # This is a performance optimization. We can safely assume that,
-        # if the resources that we want to fetch are already cached,
-        # there's no need to check the remote storage for the existence of
-        # those files.
-        if download and local_exists == md5s:
-            remote_exists = local_exists
+    def _list_paths(self, prefix=None, progress_callback=None):
+        assert self.tree.path_info is not None
+        if prefix:
+            path_info = self.tree.path_info / prefix[:2]
+            if not self.tree.exists(path_info):
+                return
         else:
-            logger.debug("Collecting information from remote cache...")
-            remote_exists = set()
-            dir_md5s = set(named_cache.dir_keys(self.tree.scheme))
-            if dir_md5s:
-                remote_exists.update(
-                    self._indexed_dir_hashes(named_cache, remote, dir_md5s)
-                )
-                md5s.difference_update(remote_exists)
-            if md5s:
-                remote_exists.update(
-                    remote.hashes_exist(
-                        md5s, jobs=jobs, name=str(remote.tree.path_info)
-                    )
-                )
-        return self._make_status(
-            named_cache,
-            show_checksums,
-            local_exists,
-            remote_exists,
-            log_missing,
-        )
-
-    def _make_status(
-        self,
-        named_cache,
-        show_checksums,
-        local_exists,
-        remote_exists,
-        log_missing,
-    ):
-        def make_names(hash_, names):
-            return {"name": hash_ if show_checksums else " ".join(names)}
-
-        dir_status = {}
-        file_status = {}
-        dir_contents = {}
-        for hash_, item in named_cache[self.tree.scheme].items():
-            if item.children:
-                dir_status[hash_] = make_names(hash_, item.names)
-                dir_contents[hash_] = set()
-                for child_hash, child in item.children.items():
-                    file_status[child_hash] = make_names(
-                        child_hash, child.names
-                    )
-                    dir_contents[hash_].add(child_hash)
-            else:
-                file_status[hash_] = make_names(hash_, item.names)
-
-        self._fill_statuses(dir_status, local_exists, remote_exists)
-        self._fill_statuses(file_status, local_exists, remote_exists)
-
-        if log_missing:
-            self._log_missing_caches(dict(dir_status, **file_status))
-
-        return dir_status, file_status, dir_contents
-
-    def _indexed_dir_hashes(self, named_cache, remote, dir_md5s):
-        # Validate our index by verifying all indexed .dir hashes
-        # still exist on the remote
-        indexed_dirs = set(remote.index.dir_hashes())
-        indexed_dir_exists = set()
-        if indexed_dirs:
-            indexed_dir_exists.update(
-                remote.tree.list_hashes_exists(indexed_dirs)
-            )
-            missing_dirs = indexed_dirs.difference(indexed_dir_exists)
-            if missing_dirs:
-                logger.debug(
-                    "Remote cache missing indexed .dir hashes '{}', "
-                    "clearing remote index".format(", ".join(missing_dirs))
-                )
-                remote.index.clear()
-
-        # Check if non-indexed (new) dir hashes exist on remote
-        dir_exists = dir_md5s.intersection(indexed_dir_exists)
-        dir_exists.update(
-            remote.tree.list_hashes_exists(dir_md5s - dir_exists)
-        )
-
-        # If .dir hash exists on the remote, assume directory contents
-        # still exists on the remote
-        for dir_hash in dir_exists:
-            file_hashes = list(
-                named_cache.child_keys(self.tree.scheme, dir_hash)
-            )
-            if dir_hash not in remote.index:
-                logger.debug(
-                    "Indexing new .dir '{}' with '{}' nested files".format(
-                        dir_hash, len(file_hashes)
-                    )
-                )
-                remote.index.update([dir_hash], file_hashes)
-            yield dir_hash
-            yield from file_hashes
-
-    @staticmethod
-    def _fill_statuses(hash_info_dir, local_exists, remote_exists):
-        # Using sets because they are way faster for lookups
-        local = set(local_exists)
-        remote = set(remote_exists)
-
-        for md5, info in hash_info_dir.items():
-            status = STATUS_MAP[(md5 in local, md5 in remote)]
-            info["status"] = status
-
-    def _get_plans(self, download, remote, status_info, status):
-        cache = []
-        path_infos = []
-        names = []
-        hashes = []
-        missing = []
-        for md5, info in Tqdm(
-            status_info.items(), desc="Analysing status", unit="file"
-        ):
-            if info["status"] == status:
-                cache.append(self.tree.hash_to_path_info(md5))
-                path_infos.append(remote.tree.hash_to_path_info(md5))
-                names.append(info["name"])
-                hashes.append(md5)
-            elif info["status"] == STATUS_MISSING:
-                missing.append(md5)
-
-        if download:
-            to_infos = cache
-            from_infos = path_infos
+            path_info = self.tree.path_info
+        # NOTE: use utils.fs walk_files since tree.walk_files will not follow
+        # symlinks
+        if progress_callback:
+            for path in walk_files(path_info):
+                progress_callback()
+                yield path
         else:
-            to_infos = path_infos
-            from_infos = cache
+            yield from walk_files(path_info)
 
-        return (from_infos, to_infos, names, hashes), missing
-
-    def _process(
-        self,
-        named_cache,
-        remote,
-        jobs=None,
-        show_checksums=False,
-        download=False,
-    ):
-        logger.debug(
-            "Preparing to {} '{}'".format(
-                "download data from" if download else "upload data to",
-                remote.tree.path_info,
-            )
-        )
-
-        if download:
-            func = partial(
-                _log_exceptions(remote.tree.download, "download"),
-                dir_mode=self.tree.dir_mode,
-                file_mode=self.tree.file_mode,
-            )
-            status = STATUS_DELETED
-            desc = "Downloading"
-        else:
-            func = _log_exceptions(remote.tree.upload, "upload")
-            status = STATUS_NEW
-            desc = "Uploading"
-
-        if jobs is None:
-            jobs = remote.tree.JOBS
-
-        dir_status, file_status, dir_contents = self._status(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=download,
-        )
-
-        dir_plans, _ = self._get_plans(download, remote, dir_status, status)
-        file_plans, missing_files = self._get_plans(
-            download, remote, file_status, status
-        )
-
-        total = len(dir_plans[0]) + len(file_plans[0])
-        if total == 0:
-            return 0
-
-        with Tqdm(total=total, unit="file", desc=desc) as pbar:
-            func = pbar.wrap_fn(func)
-            with ThreadPoolExecutor(max_workers=jobs) as executor:
-                if download:
-                    from_infos, to_infos, names, _ = (
-                        d + f for d, f in zip(dir_plans, file_plans)
-                    )
-                    fails = sum(
-                        executor.map(func, from_infos, to_infos, names)
-                    )
-                else:
-                    # for uploads, push files first, and any .dir files last
-
-                    file_futures = {}
-                    for from_info, to_info, name, hash_ in zip(*file_plans):
-                        file_futures[hash_] = executor.submit(
-                            func, from_info, to_info, name
-                        )
-                    dir_futures = {}
-                    for from_info, to_info, name, dir_hash in zip(*dir_plans):
-                        # if for some reason a file contained in this dir is
-                        # missing both locally and in the remote, we want to
-                        # push whatever file content we have, but should not
-                        # push .dir file
-                        for file_hash in missing_files:
-                            if file_hash in dir_contents[dir_hash]:
-                                logger.debug(
-                                    "directory '%s' contains missing files,"
-                                    "skipping .dir file upload",
-                                    name,
-                                )
-                                break
-                        else:
-                            wait_futures = {
-                                future
-                                for file_hash, future in file_futures.items()
-                                if file_hash in dir_contents[dir_hash]
-                            }
-                            dir_futures[dir_hash] = executor.submit(
-                                self._dir_upload,
-                                func,
-                                wait_futures,
-                                from_info,
-                                to_info,
-                                name,
-                            )
-                    fails = sum(
-                        future.result()
-                        for future in concat(
-                            file_futures.values(), dir_futures.values()
-                        )
-                    )
-
-        if fails:
-            if download:
-                remote.index.clear()
-                raise DownloadError(fails)
-            raise UploadError(fails)
-
-        if not download:
-            # index successfully pushed dirs
-            for dir_hash, future in dir_futures.items():
-                if future.result() == 0:
-                    file_hashes = dir_contents[dir_hash]
-                    logger.debug(
-                        "Indexing pushed dir '{}' with "
-                        "'{}' nested files".format(dir_hash, len(file_hashes))
-                    )
-                    remote.index.update([dir_hash], file_hashes)
-
-        return len(dir_plans[0]) + len(file_plans[0])
-
-    @staticmethod
-    def _dir_upload(func, futures, from_info, to_info, name):
-        for future in as_completed(futures):
-            if future.result():
-                # do not upload this .dir file if any file in this
-                # directory failed to upload
-                logger.debug(
-                    "failed to upload full contents of '{}', "
-                    "aborting .dir file upload".format(name)
-                )
-                logger.error(f"failed to upload '{from_info}' to '{to_info}'")
-                return 1
-        return func(from_info, to_info, name)
-
-    @index_locked
-    def push(self, named_cache, remote, jobs=None, show_checksums=False):
-        return self._process(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=False,
-        )
-
-    @index_locked
-    def pull(self, named_cache, remote, jobs=None, show_checksums=False):
-        ret = self._process(
-            named_cache,
-            remote,
-            jobs=jobs,
-            show_checksums=show_checksums,
-            download=True,
-        )
-
-        if not remote.tree.verify:
-            for checksum in named_cache.scheme_keys("local"):
-                cache_file = self.tree.hash_to_path_info(checksum)
-                if self.tree.exists(cache_file):
-                    # We can safely save here, as existing corrupted files will
-                    # be removed upon status, while files corrupted during
-                    # download will not be moved from tmp_file
-                    # (see `BaseTree.download()`)
-                    self.tree.state.save(cache_file, checksum)
-
-        return ret
-
-    @staticmethod
-    def _log_missing_caches(hash_info_dict):
-        missing_caches = [
-            (md5, info)
-            for md5, info in hash_info_dict.items()
-            if info["status"] == STATUS_MISSING
-        ]
-        if missing_caches:
-            missing_desc = "\n".join(
-                "name: {}, md5: {}".format(info["name"], md5)
-                for md5, info in missing_caches
-            )
-            msg = (
-                "Some of the cache files do not exist neither locally "
-                "nor on remote. Missing cache files:\n{}".format(missing_desc)
-            )
-            logger.warning(msg)
+    def _remove_unpacked_dir(self, hash_):
+        info = self.tree.hash_to_path_info(hash_)
+        path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
+        self.tree.remove(path_info)

dvc/cache/ssh.py

@@ -1,17 +1,18 @@
 import errno
 import itertools
 import logging
+import posixpath
 from concurrent.futures import ThreadPoolExecutor
 
 from dvc.progress import Tqdm
 from dvc.utils import to_chunks
 
-from .base import Remote
+from .base import CloudCache
 
 logger = logging.getLogger(__name__)
 
 
-class SSHRemote(Remote):
+class SSHCache(CloudCache):
     def batch_exists(self, path_infos, callback):
         def _exists(chunk_and_channel):
             chunk, channel = chunk_and_channel
@@ -47,7 +48,7 @@ class SSHRemote(Remote):
         remote/base.
         """
         if not self.tree.CAN_TRAVERSE:
-            return list(set(hashes) & set(self.tree.all()))
+            return list(set(hashes) & set(self.all()))
 
         # possibly prompt for credentials before "Querying" progress output
         self.tree.ensure_credentials()
@@ -71,3 +72,19 @@ class SSHRemote(Remote):
                 in_remote = itertools.chain.from_iterable(results)
                 ret = list(itertools.compress(hashes, in_remote))
                 return ret
+
+    def _list_paths(self, prefix=None, progress_callback=None):
+        if prefix:
+            root = posixpath.join(self.tree.path_info.path, prefix[:2])
+        else:
+            root = self.tree.path_info.path
+        with self.tree.ssh(self.tree.path_info) as ssh:
+            if prefix and not ssh.exists(root):
+                return
+            # If we simply return an iterator then with above closes instantly
+            if progress_callback:
+                for path in ssh.walk_files(root):
+                    progress_callback()
+                    yield path
+            else:
+                yield from ssh.walk_files(root)

dvc/cli.py

@@ -1,6 +1,7 @@
 """DVC command line interface"""
 import argparse
 import logging
+import os
 import sys
 
 from .command import (
@@ -145,6 +146,10 @@ def get_parent_parser():
     )
     parent_parser.add_argument("--cprofile-dump", help=argparse.SUPPRESS)
 
+    parent_parser.add_argument(
+        "--pdb", action="store_true", default=False, help=argparse.SUPPRESS,
+    )
+
     log_level_group = parent_parser.add_mutually_exclusive_group()
     log_level_group.add_argument(
         "-q", "--quiet", action="count", default=0, help="Be quiet."
@@ -191,6 +196,14 @@ def get_main_parser():
         help="Show program's version.",
     )
 
+    parser.add_argument(
+        "--cd",
+        default=os.path.curdir,
+        metavar="<path>",
+        help="Change to directory before executing.",
+        type=str,
+    )
+
     # Sub commands
     subparsers = parser.add_subparsers(
         title="Available Commands",

dvc/command/add.py

@@ -20,6 +20,8 @@ class CmdAdd(CmdBase):
                 no_commit=self.args.no_commit,
                 fname=self.args.file,
                 external=self.args.external,
+                glob=self.args.glob,
+                desc=self.args.desc,
             )
 
         except DvcException:
@@ -57,11 +59,26 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Allow targets that are outside of the DVC repository.",
     )
+    parser.add_argument(
+        "--glob",
+        action="store_true",
+        default=False,
+        help="Allows targets containing shell-style wildcards.",
+    )
     parser.add_argument(
         "--file",
         help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
     )
+    parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the data (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
     parser.add_argument(
         "targets", nargs="+", help="Input files/directories to add.",
     ).complete = completion.FILE

dvc/command/base.py

@@ -1,4 +1,5 @@
 import logging
+import os
 from abc import ABC, abstractmethod
 
 logger = logging.getLogger(__name__)
@@ -28,16 +29,21 @@ def append_doc_link(help_message, path):
 
 
 class CmdBase(ABC):
+    UNINITIALIZED = False
+
     def __init__(self, args):
         from dvc.repo import Repo
         from dvc.updater import Updater
 
-        self.repo = Repo()
+        os.chdir(args.cd)
+
+        self.repo = Repo(uninitialized=self.UNINITIALIZED)
         self.config = self.repo.config
         self.args = args
-        hardlink_lock = self.config["core"].get("hardlink_lock", False)
-        updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
-        updater.check()
+        if self.repo.tmp_dir:
+            hardlink_lock = self.config["core"].get("hardlink_lock", False)
+            updater = Updater(self.repo.tmp_dir, hardlink_lock=hardlink_lock)
+            updater.check()
 
     @property
     def default_targets(self):
@@ -55,3 +61,5 @@ class CmdBase(ABC):
 class CmdBaseNoRepo(CmdBase):
     def __init__(self, args):  # pylint: disable=super-init-not-called
         self.args = args
+
+        os.chdir(args.cd)

dvc/command/cache.py

@@ -1,12 +1,18 @@
 import argparse
+import logging
 
 from dvc.command import completion
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
 
+logger = logging.getLogger(__name__)
+
 
 class CmdCacheDir(CmdConfig):
     def run(self):
+        if self.args.value is None and not self.args.unset:
+            logger.info(self.config["cache"]["dir"])
+            return 0
         with self.config.edit(level=self.args.level) as edit:
             edit["cache"]["dir"] = self.args.value
         return 0
@@ -55,6 +61,8 @@ def add_parser(subparsers, parent_parser):
         "value",
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "
-        "config file location.",
+        "config file location. If no path is provided, it returns the "
+        "current cache directory.",
+        nargs="?",
     ).complete = completion.DIR
     cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/dag.py

@@ -2,7 +2,7 @@ import argparse
 import logging
 
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.exceptions import DvcException
+from dvc.stage import Stage
 
 logger = logging.getLogger(__name__)
 
@@ -30,58 +30,83 @@ def _show_dot(G):
     return dot_file.getvalue()
 
 
-def _build(G, target=None, full=False):
+def _collect_targets(repo, target, outs):
+    if not target:
+        return []
+
+    pairs = repo.collect_granular(target)
+    if not outs:
+        return [stage.addressing for stage, _ in pairs]
+
+    targets = []
+    for stage, info in pairs:
+        if not info:
+            targets.extend([str(out) for out in stage.outs])
+            continue
+
+        for out in repo.outs_trie.itervalues(prefix=info.parts):  # noqa: B301
+            targets.extend(str(out))
+
+    return targets
+
+
+def _transform(repo, outs):
     import networkx as nx
 
-    from dvc.repo.graph import get_pipeline, get_pipelines
+    def _relabel(node) -> str:
+        return node.addressing if isinstance(node, Stage) else str(node)
 
-    if target:
-        H = get_pipeline(get_pipelines(G), target)
-        if not full:
-            descendants = nx.descendants(G, target)
+    G = repo.outs_graph if outs else repo.graph
+    return nx.relabel_nodes(G, _relabel, copy=True)
+
+
+def _filter(G, targets, full):
+    import networkx as nx
+
+    if not targets:
+        return G
+
+    H = G.copy()
+    if not full:
+        descendants = set()
+        for target in targets:
+            descendants.update(nx.descendants(G, target))
             descendants.add(target)
-            H.remove_nodes_from(set(G.nodes()) - descendants)
-    else:
-        H = G
+        H.remove_nodes_from(set(G.nodes()) - descendants)
 
-    def _relabel(stage):
-        return stage.addressing
+    undirected = H.to_undirected()
+    connected = set()
+    for target in targets:
+        connected.update(nx.node_connected_component(undirected, target))
 
-    return nx.relabel_nodes(H, _relabel, copy=False)
+    H.remove_nodes_from(set(H.nodes()) - connected)
+
+    return H
+
+
+def _build(repo, target=None, full=False, outs=False):
+    targets = _collect_targets(repo, target, outs)
+    G = _transform(repo, outs)
+    return _filter(G, targets, full)
 
 
 class CmdDAG(CmdBase):
     def run(self):
-        try:
-            target = None
-            if self.args.target:
-                stages = self.repo.collect(self.args.target)
-                if len(stages) > 1:
-                    logger.error(
-                        f"'{self.args.target}' contains more than one stage "
-                        "{stages}, please specify one stage"
-                    )
-                    return 1
-                target = stages[0]
+        G = _build(
+            self.repo,
+            target=self.args.target,
+            full=self.args.full,
+            outs=self.args.outs,
+        )
 
-            G = _build(self.repo.graph, target=target, full=self.args.full,)
+        if self.args.dot:
+            logger.info(_show_dot(G))
+        else:
+            from dvc.utils.pager import pager
 
-            if self.args.dot:
-                logger.info(_show_dot(G))
-            else:
-                from dvc.utils.pager import pager
+            pager(_show_ascii(G))
 
-                pager(_show_ascii(G))
-
-            return 0
-        except DvcException:
-            msg = "failed to show "
-            if self.args.target:
-                msg += f"a pipeline for '{target}'"
-            else:
-                msg += "pipelines"
-            logger.exception(msg)
-            return 1
+        return 0
 
 
 def add_parser(subparsers, parent_parser):
@@ -108,6 +133,13 @@ def add_parser(subparsers, parent_parser):
             "showing DAG consisting only of ancestors."
         ),
     )
+    dag_parser.add_argument(
+        "-o",
+        "--outs",
+        action="store_true",
+        default=False,
+        help="Print output files instead of stages.",
+    )
     dag_parser.add_argument(
         "target",
         nargs="?",

dvc/command/experiments.py

@@ -1,14 +1,21 @@
 import argparse
 import io
 import logging
+import os
 from collections import OrderedDict
-from datetime import date
+from collections.abc import Mapping
+from datetime import date, datetime
 from itertools import groupby
 from typing import Iterable, Optional
 
+import dvc.prompt as prompt
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.command.metrics import DEFAULT_PRECISION
+from dvc.command.repro import CmdRepro
+from dvc.command.repro import add_arguments as add_repro_arguments
 from dvc.exceptions import DvcException, InvalidArgumentError
+from dvc.repo.experiments import Experiments
+from dvc.scm.git import Git
 from dvc.utils.flatten import flatten
 
 logger = logging.getLogger(__name__)
@@ -44,13 +51,13 @@ def _filter_names(
                 update_func({match: None for match in matches})
 
     if include:
-        ret = OrderedDict()
+        ret: OrderedDict = OrderedDict()
         _filter(include, ret.update)
     else:
         ret = OrderedDict({name: None for name in names})
 
     if exclude:
-        _filter(exclude, ret.difference_update)
+        _filter(exclude, ret.difference_update)  # type: ignore[attr-defined]
 
     return [".".join(name) for name in ret]
 
@@ -109,19 +116,47 @@ def _collect_rows(
         reverse = sort_order == "desc"
         experiments = _sort_exp(experiments, sort_by, sort_type, reverse)
 
+    new_checkpoint = True
     for i, (rev, exp) in enumerate(experiments.items()):
         row = []
         style = None
         queued = "*" if exp.get("queued", False) else ""
 
+        tip = exp.get("checkpoint_tip")
+        parent = ""
         if rev == "baseline":
-            name = exp.get("name", base_rev)
+            if Git.is_sha(base_rev):
+                name_rev = base_rev[:7]
+            else:
+                name_rev = base_rev
+            name = exp.get("name", name_rev)
             row.append(f"{name}")
             style = "bold"
-        elif i < len(experiments) - 1:
-            row.append(f"├── {queued}{rev[:7]}")
         else:
-            row.append(f"└── {queued}{rev[:7]}")
+            if tip:
+                parent_rev = exp.get("checkpoint_parent", "")
+                parent_exp = experiments.get(parent_rev, {})
+                parent_tip = parent_exp.get("checkpoint_tip")
+                if tip == parent_tip:
+                    if new_checkpoint:
+                        tree = "│ ╓"
+                    else:
+                        tree = "│ ╟"
+                    new_checkpoint = False
+                else:
+                    if parent_rev == base_rev:
+                        tree = "├─╨"
+                    else:
+                        tree = "│ ╟"
+                        parent = f" ({parent_rev[:7]})"
+                    new_checkpoint = True
+            else:
+                if i < len(experiments) - 1:
+                    tree = "├──"
+                else:
+                    tree = "└──"
+                new_checkpoint = True
+            row.append(f"{tree} {queued}{rev[:7]}{parent}")
 
         if not no_timestamp:
             row.append(_format_time(exp.get("timestamp")))
@@ -135,13 +170,12 @@ def _collect_rows(
 
 
 def _sort_exp(experiments, sort_by, typ, reverse):
-    if "baseline" in experiments:
-        ret = OrderedDict({"baseline": experiments.pop("baseline")})
-    else:
-        ret = OrderedDict()
-
     def _sort(item):
-        _, exp = item
+        rev, exp = item
+        tip = exp.get("checkpoint_tip")
+        if tip and tip != rev:
+            # Sort checkpoint experiments by tip commit
+            return _sort((tip, experiments[tip]))
         for fname, item in exp.get(typ, {}).items():
             if isinstance(item, dict):
                 item = flatten(item)
@@ -152,6 +186,10 @@ def _sort_exp(experiments, sort_by, typ, reverse):
                 return (val is None, val)
         return (True, None)
 
+    ret = OrderedDict()
+    if "baseline" in experiments:
+        ret["baseline"] = experiments.pop("baseline")
+
     ret.update(sorted(experiments.items(), key=_sort, reverse=reverse))
     return ret
 
@@ -166,12 +204,19 @@ def _format_time(timestamp):
     return timestamp.strftime(fmt)
 
 
-def _extend_row(row, names, items, precision):
-    def _round(val):
-        if isinstance(val, float):
-            return round(val, precision)
+def _format_field(val, precision=DEFAULT_PRECISION):
+    if isinstance(val, float):
+        fmt = f"{{:.{precision}g}}"
+        return fmt.format(val)
+    elif isinstance(val, Mapping):
+        return {k: _format_field(v) for k, v in val.items()}
+    elif isinstance(val, list):
+        return [_format_field(x) for x in val]
+    return str(val)
 
-        return val
+
+def _extend_row(row, names, items, precision):
+    from rich.text import Text
 
     if not items:
         row.extend(["-"] * len(names))
@@ -185,7 +230,13 @@ def _extend_row(row, names, items, precision):
         for name in names:
             if name in item:
                 value = item[name]
-                text = str(_round(value)) if value is not None else "-"
+                if value is None:
+                    text = "-"
+                else:
+                    # wrap field data in rich.Text, otherwise rich may
+                    # interpret unescaped braces from list/dict types as rich
+                    # markup tags
+                    text = Text(str(_format_field(value, precision)))
                 row.append(text)
             else:
                 row.append("-")
@@ -205,8 +256,6 @@ def _parse_list(param_list):
 def _show_experiments(all_experiments, console, **kwargs):
     from rich.table import Table
 
-    from dvc.scm.git import Git
-
     include_metrics = _parse_list(kwargs.pop("include_metrics", []))
     exclude_metrics = _parse_list(kwargs.pop("exclude_metrics", []))
     include_params = _parse_list(kwargs.pop("include_params", []))
@@ -230,9 +279,6 @@ def _show_experiments(all_experiments, console, **kwargs):
         table.add_column(name, justify="left")
 
     for base_rev, experiments in all_experiments.items():
-        if Git.is_sha(base_rev):
-            base_rev = base_rev[:7]
-
         for row, _, in _collect_rows(
             base_rev, experiments, metric_names, param_names, **kwargs,
         ):
@@ -241,6 +287,12 @@ def _show_experiments(all_experiments, console, **kwargs):
     console.print(table)
 
 
+def _format_json(item):
+    if isinstance(item, (date, datetime)):
+        return item.isoformat()
+    raise TypeError
+
+
 class CmdExperimentsShow(CmdBase):
     def run(self):
         from rich.console import Console
@@ -258,6 +310,12 @@ class CmdExperimentsShow(CmdBase):
                 sha_only=self.args.sha,
             )
 
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(all_experiments, default=_format_json))
+                return 0
+
             if self.args.no_pager:
                 console = Console()
             else:
@@ -267,6 +325,11 @@ class CmdExperimentsShow(CmdBase):
                     file=io.StringIO(), force_terminal=True, width=9999
                 )
 
+            if self.args.precision is None:
+                precision = DEFAULT_PRECISION
+            else:
+                precision = self.args.precision
+
             _show_experiments(
                 all_experiments,
                 console,
@@ -277,6 +340,7 @@ class CmdExperimentsShow(CmdBase):
                 no_timestamp=self.args.no_timestamp,
                 sort_by=self.args.sort_by,
                 sort_order=self.args.sort_order,
+                precision=precision,
             )
 
             if not self.args.no_pager:
@@ -299,19 +363,15 @@ class CmdExperimentsCheckout(CmdBase):
 
 
 def _show_diff(
-    diff, title="", markdown=False, no_path=False, old=False, precision=None
+    diff,
+    title="",
+    markdown=False,
+    no_path=False,
+    old=False,
+    precision=DEFAULT_PRECISION,
 ):
     from dvc.utils.diff import table
 
-    if precision is None:
-        precision = DEFAULT_PRECISION
-
-    def _round(val):
-        if isinstance(val, float):
-            return round(val, precision)
-
-        return val
-
     rows = []
     for fname, diff_ in diff.items():
         sorted_diff = OrderedDict(sorted(diff_.items()))
@@ -319,9 +379,13 @@ def _show_diff(
             row = [] if no_path else [fname]
             row.append(item)
             if old:
-                row.append(_round(change.get("old")))
-            row.append(_round(change["new"]))
-            row.append(_round(change.get("diff", "diff not supported")))
+                row.append(_format_field(change.get("old"), precision))
+            row.append(_format_field(change["new"], precision))
+            row.append(
+                _format_field(
+                    change.get("diff", "diff not supported"), precision
+                )
+            )
             rows.append(row)
 
     header = [] if no_path else ["Path"]
@@ -352,6 +416,11 @@ class CmdExperimentsDiff(CmdBase):
 
                 logger.info(json.dumps(diff))
             else:
+                if self.args.precision is None:
+                    precision = DEFAULT_PRECISION
+                else:
+                    precision = self.args.precision
+
                 diffs = [("metrics", "Metric"), ("params", "Param")]
                 for key, title in diffs:
                     table = _show_diff(
@@ -360,7 +429,7 @@ class CmdExperimentsDiff(CmdBase):
                         markdown=self.args.show_md,
                         no_path=self.args.no_path,
                         old=self.args.old,
-                        precision=self.args.precision,
+                        precision=precision,
                     )
                     if table:
                         logger.info(table)
@@ -373,6 +442,96 @@ class CmdExperimentsDiff(CmdBase):
         return 0
 
 
+class CmdExperimentsRun(CmdRepro):
+    def run(self):
+        if not self.repo.experiments:
+            return 0
+
+        saved_dir = os.path.realpath(os.curdir)
+        os.chdir(self.args.cwd)
+
+        # Dirty hack so the for loop below can at least enter once
+        if self.args.all_pipelines:
+            self.args.targets = [None]
+        elif not self.args.targets:
+            self.args.targets = self.default_targets
+
+        ret = 0
+        for target in self.args.targets:
+            try:
+                self.repo.experiments.run(
+                    target,
+                    queue=self.args.queue,
+                    run_all=self.args.run_all,
+                    jobs=self.args.jobs,
+                    params=self.args.params,
+                    checkpoint_resume=self.args.checkpoint_resume,
+                    **self._repro_kwargs,
+                )
+            except DvcException:
+                logger.exception("")
+                ret = 1
+                break
+
+        os.chdir(saved_dir)
+        return ret
+
+
+class CmdExperimentsGC(CmdRepro):
+    def run(self):
+        from dvc.repo.gc import _raise_error_if_all_disabled
+
+        if not self.repo.experiments:
+            return 0
+
+        _raise_error_if_all_disabled(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+        )
+
+        msg = "This will remove all experiments except those derived from "
+
+        msg += "the workspace"
+        if self.args.all_commits:
+            msg += " and all git commits"
+        elif self.args.all_branches and self.args.all_tags:
+            msg += " and all git branches and tags"
+        elif self.args.all_branches:
+            msg += " and all git branches"
+        elif self.args.all_tags:
+            msg += " and all git tags"
+        msg += " of the current repo."
+        if self.args.queued:
+            msg += " Run queued experiments will be preserved."
+        if self.args.queued:
+            msg += " Run queued experiments will be removed."
+
+        logger.warning(msg)
+
+        msg = "Are you sure you want to proceed?"
+        if not self.args.force and not prompt.confirm(msg):
+            return 1
+
+        removed = self.repo.experiments.gc(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            all_commits=self.args.all_commits,
+            workspace=self.args.workspace,
+            queued=self.args.queued,
+        )
+
+        if removed:
+            logger.info(
+                f"Removed {removed} experiments. To remove unused cache files "
+                "use 'dvc gc'."
+            )
+        else:
+            logger.info("No experiments to remove.")
+        return 0
+
+
 def add_parser(subparsers, parent_parser):
     EXPERIMENTS_HELP = "Commands to display and compare experiments."
 
@@ -477,6 +636,21 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Always show git commit SHAs instead of branch/tag names.",
     )
+    experiments_show_parser.add_argument(
+        "--show-json",
+        action="store_true",
+        default=False,
+        help="Print output in JSON format instead of a human-readable table.",
+    )
+    experiments_show_parser.add_argument(
+        "--precision",
+        type=int,
+        help=(
+            "Round metrics/params to `n` digits precision after the decimal "
+            f"point. Rounds to {DEFAULT_PRECISION} digits by default."
+        ),
+        metavar="<n>",
+    )
     experiments_show_parser.set_defaults(func=CmdExperimentsShow)
 
     EXPERIMENTS_CHECKOUT_HELP = "Checkout experiments."
@@ -552,3 +726,137 @@ def add_parser(subparsers, parent_parser):
         metavar="<n>",
     )
     experiments_diff_parser.set_defaults(func=CmdExperimentsDiff)
+
+    EXPERIMENTS_RUN_HELP = (
+        "Reproduce complete or partial experiment pipelines."
+    )
+    experiments_run_parser = experiments_subparsers.add_parser(
+        "run",
+        parents=[parent_parser],
+        description=append_doc_link(EXPERIMENTS_RUN_HELP, "experiments/run"),
+        help=EXPERIMENTS_RUN_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    _add_run_common(experiments_run_parser)
+    experiments_run_parser.add_argument(
+        "--checkpoint-resume", type=str, default=None, help=argparse.SUPPRESS,
+    )
+    experiments_run_parser.set_defaults(func=CmdExperimentsRun)
+
+    EXPERIMENTS_RESUME_HELP = "Resume checkpoint experiments."
+    experiments_resume_parser = experiments_subparsers.add_parser(
+        "resume",
+        parents=[parent_parser],
+        aliases=["res"],
+        description=append_doc_link(
+            EXPERIMENTS_RESUME_HELP, "experiments/resume"
+        ),
+        help=EXPERIMENTS_RESUME_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    _add_run_common(experiments_resume_parser)
+    experiments_resume_parser.add_argument(
+        "-r",
+        "--rev",
+        type=str,
+        default=Experiments.LAST_CHECKPOINT,
+        dest="checkpoint_resume",
+        help=(
+            "Continue the specified checkpoint experiment. "
+            "If no experiment revision is provided, "
+            "the most recently run checkpoint experiment will be used."
+        ),
+        metavar="<experiment_rev>",
+    )
+    experiments_resume_parser.set_defaults(func=CmdExperimentsRun)
+
+    EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments."
+    EXPERIMENTS_GC_DESCRIPTION = (
+        "Removes all experiments which are not derived from the specified"
+        "Git revisions."
+    )
+    experiments_gc_parser = experiments_subparsers.add_parser(
+        "gc",
+        parents=[parent_parser],
+        description=append_doc_link(
+            EXPERIMENTS_GC_DESCRIPTION, "experiments/gc"
+        ),
+        help=EXPERIMENTS_GC_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    experiments_gc_parser.add_argument(
+        "-w",
+        "--workspace",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from the current workspace.",
+    )
+    experiments_gc_parser.add_argument(
+        "-a",
+        "--all-branches",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from the tips of all Git branches.",
+    )
+    experiments_gc_parser.add_argument(
+        "-T",
+        "--all-tags",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from all Git tags.",
+    )
+    experiments_gc_parser.add_argument(
+        "--all-commits",
+        action="store_true",
+        default=False,
+        help="Keep experiments derived from all Git commits.",
+    )
+    experiments_gc_parser.add_argument(
+        "--queued",
+        action="store_true",
+        default=False,
+        help=(
+            "Keep queued experiments (experiments run queue will be cleared "
+            "by default)."
+        ),
+    )
+    experiments_gc_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force garbage collection - automatically agree to all prompts.",
+    )
+    experiments_gc_parser.set_defaults(func=CmdExperimentsGC)
+
+
+def _add_run_common(parser):
+    """Add common args for 'exp run' and 'exp resume'."""
+    # inherit arguments from `dvc repro`
+    add_repro_arguments(parser)
+    parser.add_argument(
+        "--params",
+        action="append",
+        default=[],
+        help="Use the specified param values when reproducing pipelines.",
+        metavar="[<filename>:]<params_list>",
+    )
+    parser.add_argument(
+        "--queue",
+        action="store_true",
+        default=False,
+        help="Stage this experiment in the run queue for future execution.",
+    )
+    parser.add_argument(
+        "--run-all",
+        action="store_true",
+        default=False,
+        help="Execute all experiments in the run queue.",
+    )
+    parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        help="Run the specified number of experiments at a time in parallel.",
+        metavar="<number>",
+    )

dvc/command/imp.py

@@ -15,7 +15,10 @@ class CmdImport(CmdBase):
                 self.args.url,
                 self.args.path,
                 out=self.args.out,
+                fname=self.args.file,
                 rev=self.args.rev,
+                no_exec=self.args.no_exec,
+                desc=self.args.desc,
             )
         except DvcException:
             logger.exception(
@@ -59,4 +62,24 @@ def add_parser(subparsers, parent_parser):
         help="Git revision (e.g. SHA, branch, tag)",
         metavar="<commit>",
     )
+    import_parser.add_argument(
+        "--file",
+        help="Specify name of the DVC-file this command will generate.",
+        metavar="<filename>",
+    )
+    import_parser.add_argument(
+        "--no-exec",
+        action="store_true",
+        default=False,
+        help="Only create DVC-file without actually downloading it.",
+    )
+    import_parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the data (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -16,6 +16,7 @@ class CmdImportUrl(CmdBase):
                 out=self.args.out,
                 fname=self.args.file,
                 no_exec=self.args.no_exec,
+                desc=self.args.desc,
             )
         except DvcException:
             logger.exception(
@@ -67,4 +68,13 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Only create DVC-file without actually downloading it.",
     )
+    import_parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the data (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
     import_parser.set_defaults(func=CmdImportUrl)

dvc/command/init.py

@@ -1,11 +1,38 @@
 import argparse
 import logging
 
+import colorama
+
+from dvc import analytics
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
+from dvc.utils import boxify
+from dvc.utils import format_link as fmt_link
 
 logger = logging.getLogger(__name__)
 
 
+def _welcome_message():
+    if analytics.is_enabled():
+        logger.info(
+            boxify(
+                "DVC has enabled anonymous aggregate usage analytics.\n"
+                "Read the analytics documentation (and how to opt-out) here:\n"
+                + fmt_link("https://dvc.org/doc/user-guide/analytics"),
+                border_color="red",
+            )
+        )
+
+    msg = (
+        "{yellow}What's next?{nc}\n"
+        "{yellow}------------{nc}\n"
+        f"- Check out the documentation: {fmt_link('https://dvc.org/doc')}\n"
+        f"- Get help and share ideas: {fmt_link('https://dvc.org/chat')}\n"
+        f"- Star us on GitHub: {fmt_link('https://github.com/iterative/dvc')}"
+    ).format(yellow=colorama.Fore.YELLOW, nc=colorama.Fore.RESET)
+
+    logger.info(msg)
+
+
 class CmdInit(CmdBaseNoRepo):
     def run(self):
         from dvc.exceptions import InitError
@@ -19,6 +46,7 @@ class CmdInit(CmdBaseNoRepo):
                 subdir=self.args.subdir,
             )
             self.config = self.repo.config
+            _welcome_message()
         except InitError:
             logger.exception("failed to initiate DVC")
             return 1

dvc/command/metrics.py

@@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
 DEFAULT_PRECISION = 5
 
 
-def show_metrics(
+def _show_metrics(
     metrics, all_branches=False, all_tags=False, all_commits=False
 ):
     from dvc.utils.diff import format_dict
@@ -21,24 +21,31 @@ def show_metrics(
     # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
     missing = metrics.pop(None, None)
 
+    lines = []
     for branch, val in metrics.items():
         if all_branches or all_tags or all_commits:
-            logger.info(f"{branch}:")
+            lines.append(f"{branch}:")
 
         for fname, metric in val.items():
             if not isinstance(metric, dict):
-                logger.info("\t{}: {}".format(fname, str(metric)))
+                lines.append("\t{}: {}".format(fname, str(metric)))
                 continue
 
-            logger.info(f"\t{fname}:")
+            lines.append(f"\t{fname}:")
             for key, value in flatten(format_dict(metric)).items():
-                logger.info(f"\t\t{key}: {value}")
+                lines.append(f"\t\t{key}: {value}")
 
     if missing:
         raise BadMetricError(missing)
 
+    return "\n".join(lines)
 
-class CmdMetricsShow(CmdBase):
+
+class CmdMetricsBase(CmdBase):
+    UNINITIALIZED = True
+
+
+class CmdMetricsShow(CmdMetricsBase):
     def run(self):
         try:
             metrics = self.repo.metrics.show(
@@ -54,20 +61,22 @@ class CmdMetricsShow(CmdBase):
 
                 logger.info(json.dumps(metrics))
             else:
-                show_metrics(
+                table = _show_metrics(
                     metrics,
                     self.args.all_branches,
                     self.args.all_tags,
                     self.args.all_commits,
                 )
+                if table:
+                    logger.info(table)
         except DvcException:
-            logger.exception("failed to show metrics")
+            logger.exception("")
             return 1
 
         return 0
 
 
-def _show_diff(diff, markdown=False, no_path=False, old=False, precision=None):
+def _show_diff(diff, markdown=False, no_path=False, precision=None):
     from collections import OrderedDict
 
     from dvc.utils.diff import table
@@ -87,24 +96,20 @@ def _show_diff(diff, markdown=False, no_path=False, old=False, precision=None):
         for metric, change in sorted_mdiff.items():
             row = [] if no_path else [fname]
             row.append(metric)
-            if old:
-                row.append(_round(change.get("old")))
+            row.append(_round(change.get("old")))
             row.append(_round(change["new"]))
-            row.append(_round(change.get("diff", "diff not supported")))
+            row.append(_round(change.get("diff")))
             rows.append(row)
 
     header = [] if no_path else ["Path"]
     header.append("Metric")
-    if old:
-        header.extend(["Old", "New"])
-    else:
-        header.append("Value")
+    header.extend(["Old", "New"])
     header.append("Change")
 
     return table(header, rows, markdown)
 
 
-class CmdMetricsDiff(CmdBase):
+class CmdMetricsDiff(CmdMetricsBase):
     def run(self):
         try:
             diff = self.repo.metrics.diff(
@@ -124,7 +129,6 @@ class CmdMetricsDiff(CmdBase):
                     diff,
                     self.args.show_md,
                     self.args.no_path,
-                    self.args.old,
                     precision=self.args.precision,
                 )
                 if table:
@@ -167,8 +171,8 @@ def add_parser(subparsers, parent_parser):
         "targets",
         nargs="*",
         help=(
-            "Limit command scope to these metric files. Using -R, "
-            "directories to search metric files in can also be given."
+            "Limit command scope to these metrics files. Using -R, "
+            "directories to search metrics files in can also be given."
         ),
     ).complete = completion.FILE
     metrics_show_parser.add_argument(
@@ -204,7 +208,7 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help=(
             "If any target is a directory, recursively search and process "
-            "metric files."
+            "metrics files."
         ),
     )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
@@ -232,8 +236,8 @@ def add_parser(subparsers, parent_parser):
         "--targets",
         nargs="*",
         help=(
-            "Limit command scope to these metric files. Using -R, "
-            "directories to search metric files in can also be given."
+            "Limit command scope to these metrics files. Using -R, "
+            "directories to search metrics files in can also be given."
         ),
         metavar="<paths>",
     ).complete = completion.FILE
@@ -244,7 +248,7 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help=(
             "If any target is a directory, recursively search and process "
-            "metric files."
+            "metrics files."
         ),
     )
     metrics_diff_parser.add_argument(
@@ -265,12 +269,6 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show tabulated output in the Markdown format (GFM).",
     )
-    metrics_diff_parser.add_argument(
-        "--old",
-        action="store_true",
-        default=False,
-        help="Show old metric value.",
-    )
     metrics_diff_parser.add_argument(
         "--no-path",
         action="store_true",

dvc/command/plots.py

@@ -65,12 +65,11 @@ class CmdPlots(CmdBase):
             html = PAGE_HTML.format(divs="\n".join(divs))
             path = self.args.out or "plots.html"
 
+            path = os.path.join(os.getcwd(), path)
             with open(path, "w") as fobj:
                 fobj.write(html)
 
-            logger.info(
-                "file://{}".format(os.path.join(self.repo.root_dir, path))
-            )
+            logger.info(f"file://{path}")
 
         except DvcException:
             logger.exception("")
@@ -80,11 +79,15 @@ class CmdPlots(CmdBase):
 
 
 class CmdPlotsShow(CmdPlots):
+    UNINITIALIZED = True
+
     def _func(self, *args, **kwargs):
         return self.repo.plots.show(*args, **kwargs)
 
 
 class CmdPlotsDiff(CmdPlots):
+    UNINITIALIZED = True
+
     def _func(self, *args, **kwargs):
         return self.repo.plots.diff(
             *args,
@@ -122,7 +125,7 @@ def add_parser(subparsers, parent_parser):
 
     fix_subparsers(plots_subparsers)
 
-    SHOW_HELP = "Generate plots from metric files."
+    SHOW_HELP = "Generate plots from metrics files."
     plots_show_parser = plots_subparsers.add_parser(
         "show",
         parents=[parent_parser],
@@ -133,7 +136,9 @@ def add_parser(subparsers, parent_parser):
     plots_show_parser.add_argument(
         "targets",
         nargs="*",
-        help="Plots files to visualize. Shows all plots by default.",
+        help="Files to visualize (supports any file, "
+        "even when not found as `plots` in `dvc.yaml`). "
+        "Shows all plots by default.",
     ).complete = completion.FILE
     _add_props_arguments(plots_show_parser)
     _add_output_arguments(plots_show_parser)
@@ -153,7 +158,9 @@ def add_parser(subparsers, parent_parser):
     plots_diff_parser.add_argument(
         "--targets",
         nargs="*",
-        help="Plots file to visualize. Shows all plots by default.",
+        help="Files to visualize (supports any file, "
+        "even when not found as `plots` in `dvc.yaml`). "
+        "Shows all plots by default.",
         metavar="<path>",
     ).complete = completion.FILE
     plots_diff_parser.add_argument(
@@ -170,7 +177,7 @@ def add_parser(subparsers, parent_parser):
     _add_output_arguments(plots_diff_parser)
     plots_diff_parser.set_defaults(func=CmdPlotsDiff)
 
-    PLOTS_MODIFY_HELP = "Modify display properties of plot metric files."
+    PLOTS_MODIFY_HELP = "Modify display properties of plot metrics files."
     plots_modify_parser = plots_subparsers.add_parser(
         "modify",
         parents=[parent_parser],

dvc/command/repro.py

@@ -4,7 +4,7 @@ import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.metrics import show_metrics
+from dvc.command.metrics import _show_metrics
 from dvc.command.status import CmdDataStatus
 from dvc.dvcfile import PIPELINE_FILE
 from dvc.exceptions import DvcException
@@ -15,8 +15,7 @@ logger = logging.getLogger(__name__)
 class CmdRepro(CmdBase):
     def run(self):
         saved_dir = os.path.realpath(os.curdir)
-        if self.args.cwd:
-            os.chdir(self.args.cwd)
+        os.chdir(self.args.cwd)
 
         # Dirty hack so the for loop below can at least enter once
         if self.args.all_pipelines:
@@ -27,32 +26,20 @@ class CmdRepro(CmdBase):
         ret = 0
         for target in self.args.targets:
             try:
-                stages = self.repo.reproduce(
-                    target,
-                    single_item=self.args.single_item,
-                    force=self.args.force,
-                    dry=self.args.dry,
-                    interactive=self.args.interactive,
-                    pipeline=self.args.pipeline,
-                    all_pipelines=self.args.all_pipelines,
-                    run_cache=not self.args.no_run_cache,
-                    no_commit=self.args.no_commit,
-                    downstream=self.args.downstream,
-                    recursive=self.args.recursive,
-                    force_downstream=self.args.force_downstream,
-                    experiment=self.args.experiment,
-                    queue=self.args.queue,
-                    run_all=self.args.run_all,
-                    jobs=self.args.jobs,
-                    params=self.args.params,
-                )
+                stages = self.repo.reproduce(target, **self._repro_kwargs)
 
                 if len(stages) == 0:
                     logger.info(CmdDataStatus.UP_TO_DATE_MSG)
+                else:
+                    logger.info(
+                        "Use `dvc push` to send your updates to "
+                        "remote storage."
+                    )
 
                 if self.args.metrics:
                     metrics = self.repo.metrics.show()
-                    show_metrics(metrics)
+                    logger.info(_show_metrics(metrics))
+
             except DvcException:
                 logger.exception("")
                 ret = 1
@@ -61,18 +48,25 @@ class CmdRepro(CmdBase):
         os.chdir(saved_dir)
         return ret
 
+    @property
+    def _repro_kwargs(self):
+        return {
+            "single_item": self.args.single_item,
+            "force": self.args.force,
+            "dry": self.args.dry,
+            "interactive": self.args.interactive,
+            "pipeline": self.args.pipeline,
+            "all_pipelines": self.args.all_pipelines,
+            "run_cache": not self.args.no_run_cache,
+            "no_commit": self.args.no_commit,
+            "downstream": self.args.downstream,
+            "recursive": self.args.recursive,
+            "force_downstream": self.args.force_downstream,
+            "pull": self.args.pull,
+        }
 
-def add_parser(subparsers, parent_parser):
-    REPRO_HELP = (
-        "Reproduce complete or partial pipelines by executing their stages."
-    )
-    repro_parser = subparsers.add_parser(
-        "repro",
-        parents=[parent_parser],
-        description=append_doc_link(REPRO_HELP, "repro"),
-        help=REPRO_HELP,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
+
+def add_arguments(repro_parser):
     repro_parser.add_argument(
         "targets",
         nargs="*",
@@ -97,7 +91,8 @@ def add_parser(subparsers, parent_parser):
         "-c",
         "--cwd",
         default=os.path.curdir,
-        help="Directory within your repo to reproduce from.",
+        help="Directory within your repo to reproduce from. Note: deprecated "
+        "by `dvc --cd <path>`.",
         metavar="<path>",
     )
     repro_parser.add_argument(
@@ -172,29 +167,26 @@ def add_parser(subparsers, parent_parser):
         help="Start from the specified stages when reproducing pipelines.",
     )
     repro_parser.add_argument(
-        "-e",
-        "--experiment",
+        "--pull",
         action="store_true",
         default=False,
-        help=argparse.SUPPRESS,
+        help=(
+            "Try automatically pulling missing cache for outputs restored "
+            "from the run-cache."
+        ),
     )
-    repro_parser.add_argument(
-        "--params",
-        action="append",
-        default=[],
-        help=argparse.SUPPRESS,
-        metavar="[<filename>:]<params_list>",
+
+
+def add_parser(subparsers, parent_parser):
+    REPRO_HELP = (
+        "Reproduce complete or partial pipelines by executing their stages."
     )
-    repro_parser.add_argument(
-        "--queue", action="store_true", default=False, help=argparse.SUPPRESS
-    )
-    repro_parser.add_argument(
-        "--run-all",
-        action="store_true",
-        default=False,
-        help=argparse.SUPPRESS,
-    )
-    repro_parser.add_argument(
-        "-j", "--jobs", type=int, help=argparse.SUPPRESS, metavar="<number>"
+    repro_parser = subparsers.add_parser(
+        "repro",
+        parents=[parent_parser],
+        description=append_doc_link(REPRO_HELP, "repro"),
+        help=REPRO_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    add_arguments(repro_parser)
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/run.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -20,6 +21,7 @@ class CmdRun(CmdBase):
                 self.args.plots_no_cache,
                 self.args.outs_persist,
                 self.args.outs_persist_no_cache,
+                self.args.checkpoints,
                 self.args.params,
                 self.args.command,
             ]
@@ -44,16 +46,18 @@ class CmdRun(CmdBase):
                 params=self.args.params,
                 fname=self.args.file,
                 wdir=self.args.wdir,
-                no_exec=self.args.no_exec,
+                no_exec=(self.args.no_exec or bool(self.args.checkpoints)),
                 force=self.args.force,
                 run_cache=not self.args.no_run_cache,
                 no_commit=self.args.no_commit,
                 outs_persist=self.args.outs_persist,
                 outs_persist_no_cache=self.args.outs_persist_no_cache,
+                checkpoints=self.args.checkpoints,
                 always_changed=self.args.always_changed,
                 name=self.args.name,
                 single_stage=self.args.single_stage,
                 external=self.args.external,
+                desc=self.args.desc,
             )
         except DvcException:
             logger.exception("")
@@ -99,7 +103,7 @@ def add_parser(subparsers, parent_parser):
         default=[],
         help="Declare dependencies for reproducible cmd.",
         metavar="<path>",
-    )
+    ).complete = completion.FILE
     run_parser.add_argument(
         "-n", "--name", help="Stage name.",
     )
@@ -110,7 +114,7 @@ def add_parser(subparsers, parent_parser):
         default=[],
         help="Declare output file or directory.",
         metavar="<filename>",
-    )
+    ).complete = completion.FILE
     run_parser.add_argument(
         "-O",
         "--outs-no-cache",
@@ -119,7 +123,7 @@ def add_parser(subparsers, parent_parser):
         help="Declare output file or directory "
         "(do not put into DVC cache).",
         metavar="<filename>",
-    )
+    ).complete = completion.FILE
     run_parser.add_argument(
         "-p",
         "--params",
@@ -127,13 +131,13 @@ def add_parser(subparsers, parent_parser):
         default=[],
         help="Declare parameter to use as additional dependency.",
         metavar="[<filename>:]<params_list>",
-    )
+    ).complete = completion.FILE
     run_parser.add_argument(
         "-m",
         "--metrics",
         action="append",
         default=[],
-        help="Declare output metric file.",
+        help="Declare output metrics file.",
         metavar="<path>",
     )
     run_parser.add_argument(
@@ -141,7 +145,7 @@ def add_parser(subparsers, parent_parser):
         "--metrics-no-cache",
         action="append",
         default=[],
-        help="Declare output metric file (do not put into DVC cache).",
+        help="Declare output metrics file (do not put into DVC cache).",
         metavar="<path>",
     )
     run_parser.add_argument(
@@ -211,6 +215,14 @@ def add_parser(subparsers, parent_parser):
         "removed upon repro (do not put into DVC cache).",
         metavar="<filename>",
     )
+    run_parser.add_argument(
+        "-c",
+        "--checkpoints",
+        action="append",
+        default=[],
+        help=argparse.SUPPRESS,
+        metavar="<filename>",
+    ).complete = completion.FILE
     run_parser.add_argument(
         "--always-changed",
         action="store_true",
@@ -229,6 +241,15 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Allow outputs that are outside of the DVC repository.",
     )
+    run_parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the stage (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
     run_parser.add_argument(
         "command", nargs=argparse.REMAINDER, help="Command to execute."
     )

dvc/command/status.py

@@ -2,6 +2,7 @@ import logging
 
 from dvc.command.data_sync import CmdDataBase
 from dvc.exceptions import DvcException
+from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
 
@@ -10,6 +11,11 @@ class CmdDataStatus(CmdDataBase):
     STATUS_LEN = 20
     STATUS_INDENT = "\t"
     UP_TO_DATE_MSG = "Data and pipelines are up to date."
+    IN_SYNC_MSG = "Cache and remote '{remote}' are in sync."
+    EMPTY_PROJECT_MSG = (
+        "There are no data or pipelines tracked in this project yet.\n"
+        "See {link} to get started!"
+    ).format(link=format_link("https://dvc.org/doc/start"))
 
     def _normalize(self, s):
         s += ":"
@@ -61,10 +67,17 @@ class CmdDataStatus(CmdDataBase):
                 logger.info(json.dumps(st))
             elif st:
                 self._show(st, indent)
+            elif not self.repo.stages:
+                logger.info(self.EMPTY_PROJECT_MSG)
+            elif self.args.cloud or self.args.remote:
+                remote = self.args.remote or self.repo.config["core"].get(
+                    "remote"
+                )
+                logger.info(self.IN_SYNC_MSG.format(remote=remote))
             else:
                 logger.info(self.UP_TO_DATE_MSG)
 
         except DvcException:
-            logger.exception("failed to obtain data status")
+            logger.exception("")
             return 1
         return 0

dvc/command/version.py

@@ -24,5 +24,6 @@ def add_parser(subparsers, parent_parser):
         description=append_doc_link(VERSION_HELP, "version"),
         help=VERSION_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
+        aliases=["doctor"],
     )
     version_parser.set_defaults(func=CmdVersion)

dvc/config.py

@@ -103,6 +103,7 @@ class RelPath(str):
 REMOTE_COMMON = {
     "url": str,
     "checksum_jobs": All(Coerce(int), Range(1)),
+    "jobs": All(Coerce(int), Range(1)),
     Optional("no_traverse"): Bool,  # obsoleted
     "verify": Bool,
 }
@@ -139,6 +140,7 @@ SCHEMA = {
         Optional("analytics", default=True): Bool,
         Optional("hardlink_lock", default=False): Bool,
         Optional("no_scm", default=False): Bool,
+        Optional("autostage", default=False): Bool,
         Optional("experiments", default=False): Bool,
         Optional("check_update", default=True): Bool,
     },
@@ -147,6 +149,7 @@ SCHEMA = {
         "s3": str,
         "gs": str,
         "hdfs": str,
+        "webhdfs": str,
         "ssh": str,
         "azure": str,
         # This is for default local cache
@@ -189,9 +192,17 @@ SCHEMA = {
                     "keyfile": str,
                     "timeout": Coerce(int),
                     "gss_auth": Bool,
+                    "allow_agent": Bool,
                     **REMOTE_COMMON,
                 },
                 "hdfs": {"user": str, **REMOTE_COMMON},
+                "webhdfs": {
+                    "hdfscli_config": str,
+                    "webhdfs_token": str,
+                    "user": str,
+                    "webhdfs_alias": str,
+                    **REMOTE_COMMON,
+                },
                 "azure": {"connection_string": str, **REMOTE_COMMON},
                 "oss": {
                     "oss_key_id": str,
@@ -222,6 +233,8 @@ SCHEMA = {
         "row_limit": All(Coerce(int), Range(1)),
         "row_cleanup_quota": All(Coerce(int), Range(0, 100)),
     },
+    # section for experimental features
+    "feature": {Optional("parametrization", default=False): Bool},
 }
 COMPILED_SCHEMA = Schema(SCHEMA)
 
@@ -373,6 +386,12 @@ class Config(dict):
         def resolve(path):
             if os.path.isabs(path) or re.match(r"\w+://", path):
                 return path
+
+            # on windows convert slashes to backslashes
+            # to have path compatible with abs_conf_dir
+            if os.path.sep == "\\" and "/" in path:
+                path = path.replace("/", "\\")
+
             return RelPath(os.path.join(abs_conf_dir, path))
 
         return Config._map_dirs(conf, resolve)
@@ -399,7 +418,15 @@ class Config(dict):
         dirs_schema = {
             "cache": {"dir": func},
             "remote": {
-                str: {"url": func, "gdrive_user_credentials_file": func}
+                str: {
+                    "url": func,
+                    "gdrive_user_credentials_file": func,
+                    "gdrive_service_account_p12_file_path": func,
+                    "credentialpath": func,
+                    "keyfile": func,
+                    "cert_path": func,
+                    "key_path": func,
+                }
             },
         }
         return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)

dvc/data_cloud.py

@@ -62,8 +62,11 @@ class DataCloud:
         """
         remote = self.get_remote(remote, "push")
 
-        return self.repo.cache.local.push(
-            cache, jobs=jobs, remote=remote, show_checksums=show_checksums,
+        return remote.push(
+            self.repo.cache.local,
+            cache,
+            jobs=jobs,
+            show_checksums=show_checksums,
         )
 
     def pull(
@@ -81,8 +84,11 @@ class DataCloud:
         """
         remote = self.get_remote(remote, "pull")
 
-        return self.repo.cache.local.pull(
-            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
+        return remote.pull(
+            self.repo.cache.local,
+            cache,
+            jobs=jobs,
+            show_checksums=show_checksums,
         )
 
     def status(
@@ -107,10 +113,10 @@ class DataCloud:
                 neither in cache, neither in cloud.
         """
         remote = self.get_remote(remote, "status")
-        return self.repo.cache.local.status(
+        return remote.status(
+            self.repo.cache.local,
             cache,
             jobs=jobs,
-            remote=remote,
             show_checksums=show_checksums,
             log_missing=log_missing,
         )

dvc/dependency/__init__.py

@@ -11,6 +11,9 @@ from dvc.dependency.local import LocalDependency
 from dvc.dependency.param import ParamsDependency
 from dvc.dependency.s3 import S3Dependency
 from dvc.dependency.ssh import SSHDependency
+from dvc.dependency.webdav import WebDAVDependency
+from dvc.dependency.webdavs import WebDAVSDependency
+from dvc.dependency.webhdfs import WebHDFSDependency
 from dvc.output.base import BaseOutput
 from dvc.scheme import Schemes
 
@@ -25,6 +28,9 @@ DEPS = [
     HTTPSDependency,
     S3Dependency,
     SSHDependency,
+    WebDAVDependency,
+    WebDAVSDependency,
+    WebHDFSDependency,
     # NOTE: LocalDependency is the default choice
 ]
 
@@ -37,16 +43,20 @@ DEP_MAP = {
     Schemes.HDFS: HDFSDependency,
     Schemes.HTTP: HTTPDependency,
     Schemes.HTTPS: HTTPSDependency,
+    Schemes.WEBDAV: WebDAVDependency,
+    Schemes.WEBDAVS: WebDAVSDependency,
+    Schemes.WEBHDFS: WebHDFSDependency,
 }
 
 
 # NOTE: schema for dependencies is basically the same as for outputs, but
 # without output-specific entries like 'cache' (whether or not output is
 # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
-# output is a metric file and how to parse it, see `-M` flag for `dvc run`).
+# output is a metrics file and how to parse it, see `-M` flag for `dvc run`).
 SCHEMA = output.SCHEMA.copy()
 del SCHEMA[BaseOutput.PARAM_CACHE]
 del SCHEMA[BaseOutput.PARAM_METRIC]
+del SCHEMA[BaseOutput.PARAM_DESC]
 SCHEMA.update(RepoDependency.REPO_SCHEMA)
 SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
 

dvc/dependency/base.py

@@ -1,3 +1,5 @@
+from typing import Type
+
 from dvc.exceptions import DvcException
 
 
@@ -21,9 +23,11 @@ class DependencyIsStageFileError(DvcException):
 class BaseDependency:
     IS_DEPENDENCY = True
 
-    DoesNotExistError = DependencyDoesNotExistError
-    IsNotFileOrDirError = DependencyIsNotFileOrDirError
-    IsStageFileError = DependencyIsStageFileError
+    DoesNotExistError = DependencyDoesNotExistError  # type: Type[DvcException]
+    IsNotFileOrDirError = (
+        DependencyIsNotFileOrDirError
+    )  # type: Type[DvcException]
+    IsStageFileError = DependencyIsStageFileError  # type: Type[DvcException]
 
     def update(self, rev=None):
         pass

dvc/dependency/webdav.py

@@ -0,0 +1,8 @@
+from dvc.dependency.base import BaseDependency
+from dvc.output.base import BaseOutput
+
+from ..tree.webdav import WebDAVTree
+
+
+class WebDAVDependency(BaseDependency, BaseOutput):
+    TREE_CLS = WebDAVTree

dvc/dependency/webdavs.py

@@ -0,0 +1,6 @@
+from ..tree.webdavs import WebDAVSTree
+from .webdav import WebDAVDependency
+
+
+class WebDAVSDependency(WebDAVDependency):
+    TREE_CLS = WebDAVSTree

dvc/dependency/webhdfs.py

@@ -0,0 +1,6 @@
+from dvc.dependency.base import BaseDependency
+from dvc.output.webhdfs import WebHDFSOutput
+
+
+class WebHDFSDependency(BaseDependency, WebHDFSOutput):
+    pass

dvc/dir_info.py

@@ -0,0 +1,102 @@
+import posixpath
+from operator import itemgetter
+
+from pygtrie import Trie
+
+from .hash_info import HashInfo
+
+
+def _diff(ancestor, other, allow_removed=False):
+    from dictdiffer import diff
+
+    from dvc.exceptions import MergeError
+
+    allowed = ["add"]
+    if allow_removed:
+        allowed.append("remove")
+
+    result = list(diff(ancestor, other))
+    for typ, _, _ in result:
+        if typ not in allowed:
+            raise MergeError(
+                "unable to auto-merge directories with diff that contains "
+                f"'{typ}'ed files"
+            )
+    return result
+
+
+def _merge(ancestor, our, their):
+    import copy
+
+    from dictdiffer import patch
+
+    our_diff = _diff(ancestor, our)
+    if not our_diff:
+        return copy.deepcopy(their)
+
+    their_diff = _diff(ancestor, their)
+    if not their_diff:
+        return copy.deepcopy(our)
+
+    # make sure there are no conflicting files
+    _diff(our, their, allow_removed=True)
+
+    return patch(our_diff + their_diff, ancestor)
+
+
+class DirInfo:
+    PARAM_RELPATH = "relpath"
+
+    def __init__(self):
+        self.trie = Trie()
+
+    @property
+    def size(self):
+        try:
+            return sum(
+                hash_info.size
+                for _, hash_info in self.trie.iteritems()  # noqa: B301
+            )
+        except TypeError:
+            return None
+
+    @property
+    def nfiles(self):
+        return len(self.trie)
+
+    def items(self, path_info=None):
+        for key, hash_info in self.trie.iteritems():  # noqa: B301
+            path = posixpath.sep.join(key)
+            if path_info is not None:
+                path = path_info / path
+            yield path, hash_info
+
+    @classmethod
+    def from_list(cls, lst):
+        ret = DirInfo()
+        for _entry in lst:
+            entry = _entry.copy()
+            relpath = entry.pop(cls.PARAM_RELPATH)
+            parts = tuple(relpath.split(posixpath.sep))
+            ret.trie[parts] = HashInfo.from_dict(entry)
+        return ret
+
+    def to_list(self):
+        # Sorting the list by path to ensure reproducibility
+        return sorted(
+            (
+                {
+                    # NOTE: not using hash_info.to_dict() because we don't want
+                    # size/nfiles fields at this point.
+                    hash_info.name: hash_info.value,
+                    self.PARAM_RELPATH: posixpath.sep.join(parts),
+                }
+                for parts, hash_info in self.trie.iteritems()  # noqa: B301
+            ),
+            key=itemgetter(self.PARAM_RELPATH),
+        )
+
+    def merge(self, ancestor, their):
+        merged = DirInfo()
+        merged.trie = _merge(ancestor.trie, self.trie, their.trie)
+        return merged

dvc/dvcfile.py

@@ -36,6 +36,10 @@ class LockfileCorruptedError(DvcException):
     pass
 
 
+class ParametrizedDumpError(DvcException):
+    pass
+
+
 def is_valid_filename(path):
     return path.endswith(DVC_FILE_SUFFIX) or os.path.basename(path) in [
         DVC_FILE,
@@ -45,10 +49,14 @@ def is_valid_filename(path):
 
 def is_dvc_file(path):
     return os.path.isfile(path) and (
-        is_valid_filename(path) or os.path.basename(path) == PIPELINE_LOCK
+        is_valid_filename(path) or is_lock_file(path)
     )
 
 
+def is_lock_file(path):
+    return os.path.basename(path) == PIPELINE_LOCK
+
+
 def check_dvc_filename(path):
     if not is_valid_filename(path):
         raise StageFileBadNameError(
@@ -193,7 +201,13 @@ class PipelineFile(FileMixin):
     def _dump_lockfile(self, stage):
         self._lockfile.dump(stage)
 
+    @staticmethod
+    def _check_if_parametrized(stage):
+        if stage.raw_data.parametrized:
+            raise ParametrizedDumpError(f"cannot dump a parametrized {stage}")
+
     def _dump_pipeline_file(self, stage):
+        self._check_if_parametrized(stage)
         stage_data = serialize.to_pipeline_file(stage)
 
         with modify_yaml(self.path, tree=self.repo.tree) as data:
@@ -227,7 +241,7 @@ class PipelineFile(FileMixin):
     def stages(self):
         data, _ = self._load()
         lockfile_data = self._lockfile.load()
-        return StageLoader(self, data.get("stages", {}), lockfile_data)
+        return StageLoader(self, data, lockfile_data)
 
     def remove(self, force=False):
         if not force:

dvc/env.py

@@ -1,2 +1,4 @@
+DVC_CHECKPOINT = "DVC_CHECKPOINT"
 DVC_DAEMON = "DVC_DAEMON"
 DVC_PAGER = "DVC_PAGER"
+DVC_ROOT = "DVC_ROOT"

dvc/exceptions.py

@@ -42,7 +42,7 @@ class OutputDuplicationError(DvcException):
 
 
 class OutputNotFoundError(DvcException):
-    """Thrown if a file/directory not found in repository pipelines.
+    """Thrown if a file/directory is not found as an output in any pipeline.
 
     Args:
         output (unicode): path to the file/directory.
@@ -105,7 +105,7 @@ class ArgumentDuplicationError(DvcException):
 
 
 class MoveNotDataSourceError(DvcException):
-    """Thrown if attempted to move a file/directory that is not an output
+    """Thrown when trying to move a file/directory that is not an output
     in a data source stage.
 
     Args:
@@ -164,7 +164,7 @@ class BadMetricError(DvcException):
     def __init__(self, paths):
         super().__init__(
             "the following metrics do not exist, "
-            "are not metric files or are malformed: {paths}".format(
+            "are not metrics files or are malformed: {paths}".format(
                 paths=", ".join(f"'{path}'" for path in paths)
             )
         )
@@ -174,11 +174,20 @@ class NoMetricsError(DvcException):
     pass
 
 
-class NoPlotsError(DvcException):
-    def __init__(self):
+class NoMetricsParsedError(NoMetricsError):
+    def __init__(self, command):
         super().__init__(
-            "no plots in this repository. Use `--plots/--plots-no-cache` "
-            "options for `dvc run` to mark stage outputs as plots."
+            f"Could not parse {command} files. Use `-v` option to see more "
+            "details."
+        )
+
+
+class NoMetricsFoundError(NoMetricsError):
+    def __init__(self, command, run_options):
+        super().__init__(
+            f"No {command} files in this repository. "
+            f"Use `{run_options}` options for "
+            f"`dvc run` to mark stage outputs as {command}."
         )
 
 
@@ -218,6 +227,11 @@ class FileMissingError(DvcException):
         )
 
 
+class FileOwnershipError(DvcException):
+    def __init__(self, path):
+        super().__init__(f"file '{path}' not owned by user! ")
+
+
 class DvcIgnoreInCollectedDirError(DvcException):
     def __init__(self, ignore_dirname):
         super().__init__(
@@ -338,3 +352,18 @@ class NoOutputOrStageError(DvcException):
 
 class MergeError(DvcException):
     pass
+
+
+class CacheLinkError(DvcException):
+    SUPPORT_LINK = "See {} for more information.".format(
+        format_link(
+            "https://dvc.org/doc/user-guide/troubleshooting#cache-types"
+        )
+    )
+
+    def __init__(self, path_infos):
+        msg = "No possible cache link types for '{}'. {}".format(
+            ", ".join([str(path) for path in path_infos]), self.SUPPORT_LINK,
+        )
+        super().__init__(msg)
+        self.path_infos = path_infos

dvc/external_repo.py

@@ -4,7 +4,7 @@ import tempfile
 import threading
 from contextlib import contextmanager
 from distutils.dir_util import copy_tree
-from typing import Iterable
+from typing import Dict, Iterable
 
 from funcy import cached_property, reraise, retry, wrap_with
 
@@ -22,7 +22,6 @@ from dvc.path_info import PathInfo
 from dvc.repo import Repo
 from dvc.scm.base import CloneError
 from dvc.scm.git import Git
-from dvc.state import StateNoop
 from dvc.tree.local import LocalTree
 from dvc.tree.repo import RepoTree
 from dvc.utils import relpath
@@ -51,12 +50,10 @@ def external_repo(url, rev=None, for_write=False, **kwargs):
         scm=None if for_write else Git(root_dir),
         rev=None if for_write else rev,
         for_write=for_write,
+        uninitialized=True,
         **kwargs,
     )
-    try:
-        repo = ExternalRepo(**conf)
-    except NotDvcRepoError:
-        repo = ExternalGitRepo(**conf)
+    repo = ExternalRepo(**conf)
 
     try:
         yield repo
@@ -76,8 +73,8 @@ def external_repo(url, rev=None, for_write=False, **kwargs):
             _remove(path)
 
 
-CLONES = {}
-CACHE_DIRS = {}
+CLONES: Dict[str, str] = {}
+CACHE_DIRS: Dict[str, str] = {}
 
 
 def clean_repos():
@@ -90,9 +87,34 @@ def clean_repos():
         _remove(path)
 
 
-class BaseExternalRepo:
+class ExternalRepo(Repo):
     # pylint: disable=no-member
 
+    def __init__(
+        self,
+        root_dir,
+        url,
+        scm=None,
+        rev=None,
+        for_write=False,
+        cache_dir=None,
+        cache_types=None,
+        uninitialized=False,
+        **kwargs,
+    ):
+        super().__init__(
+            root_dir, scm=scm, rev=rev, uninitialized=uninitialized
+        )
+
+        self.url = url
+        self.for_write = for_write
+        self.cache_dir = cache_dir or self._get_cache_dir()
+        self.cache_types = cache_types
+
+        self._setup_cache(self)
+        self._fix_upstream(self)
+        self.tree_confs = kwargs
+
     def __str__(self):
         return self.url
 
@@ -108,13 +130,15 @@ class BaseExternalRepo:
             return self.scm.get_rev()
         return self.tree.rev
 
-    def _fetch_to_cache(self, path_info, repo, callback):
+    def _fetch_to_cache(self, path_info, repo, callback, **kwargs):
         # don't support subrepo traversal as it might fail due to difference
         # in remotes
         tree = self._get_tree_for(repo)
         cache = repo.cache.local
 
-        hash_info = tree.get_hash(path_info, download_callback=callback)
+        hash_info = tree.get_hash(
+            path_info, download_callback=callback, **kwargs
+        )
         cache.save(
             path_info,
             tree,
@@ -148,7 +172,9 @@ class BaseExternalRepo:
 
             self._check_repo(path, metadata.repo)
             repo = metadata.repo
-            hash_info = self._fetch_to_cache(path, repo, download_update)
+            hash_info = self._fetch_to_cache(
+                path, repo, download_update, **kwargs
+            )
             hash_infos.append(hash_info)
 
         return sum(download_results), failed, hash_infos
@@ -230,7 +256,8 @@ class BaseExternalRepo:
         return repo
 
     def _setup_cache(self, repo):
-        repo.cache.local.cache_dir = self.cache_dir
+        repo.config["cache"]["dir"] = self.cache_dir
+        repo.cache = Cache(repo)
         if self.cache_types:
             repo.cache.local.cache_types = self.cache_types
 
@@ -262,86 +289,6 @@ class BaseExternalRepo:
         return cache_dir
 
 
-class ExternalRepo(BaseExternalRepo, Repo):
-    def __init__(
-        self,
-        root_dir,
-        url,
-        scm=None,
-        rev=None,
-        for_write=False,
-        cache_dir=None,
-        cache_types=None,
-        **kwargs,
-    ):
-        super().__init__(root_dir, scm=scm, rev=rev)
-
-        self.url = url
-        self.for_write = for_write
-        self.cache_dir = cache_dir or self._get_cache_dir()
-        self.cache_types = cache_types
-
-        self._setup_cache(self)
-        self._fix_upstream(self)
-        self.tree_confs = kwargs
-
-
-class ExternalGitRepo(BaseExternalRepo):
-    def __init__(
-        self,
-        root_dir,
-        url,
-        scm=None,
-        rev=None,
-        for_write=False,
-        cache_dir=None,
-        cache_types=None,
-        **kwargs,
-    ):
-        self.root_dir = os.path.realpath(root_dir)
-        self.scm = scm
-
-        self.url = url
-        self.for_write = for_write
-        self.cache_dir = cache_dir or self._get_cache_dir()
-        self.cache_types = cache_types
-
-        self.rev = rev
-        self.tree_confs = kwargs
-
-        self.config = {"cache": {"dir": self.cache_dir}}
-        self.cache = Cache(self)
-        if cache_types:
-            self.cache.local.cache_types = cache_types
-
-        self.state = StateNoop()
-
-    @cached_property
-    def tree(self):
-        if self.scm:
-            return self.scm.get_tree(self.rev)
-        return LocalTree(self, {"url": self.root_dir})
-
-    def close(self):
-        if self.scm:
-            self.scm.close()
-
-    def find_out_by_relpath(self, path):
-        raise OutputNotFoundError(path, self)
-
-    @contextmanager
-    def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
-        """Opens a specified resource as a file object."""
-        path_info = PathInfo(self.root_dir) / path
-        try:
-            with self.repo_tree.open(
-                path_info, mode=mode, encoding=encoding, **kwargs
-            ) as fobj:
-                yield fobj
-        except FileNotFoundError as exc:
-            raise PathMissingError(path, self.url) from exc
-
-
 def _cached_clone(url, rev, for_write=False):
     """Clone an external git repo to a temporary directory.
 

dvc/hash_info.py

@@ -1,26 +1,49 @@
+from collections import OrderedDict
 from dataclasses import dataclass, field
+from typing import Dict, Optional
 
 HASH_DIR_SUFFIX = ".dir"
 
+DirInfo = Dict[str, str]
+
 
 @dataclass
 class HashInfo:
-    name: str
-    value: str
-    dir_info: list = field(default=None, compare=False)
+    PARAM_SIZE = "size"
+    PARAM_NFILES = "nfiles"
+
+    name: Optional[str]
+    value: Optional[str]
+    dir_info: Optional[DirInfo] = field(default=None, compare=False)
+    size: Optional[int] = field(default=None, compare=False)
+    nfiles: Optional[int] = field(default=None, compare=False)
 
     def __bool__(self):
         return bool(self.value)
 
     @classmethod
     def from_dict(cls, d):
-        if not d:
+        _d = d.copy() if d else {}
+        size = _d.pop(cls.PARAM_SIZE, None)
+        nfiles = _d.pop(cls.PARAM_NFILES, None)
+
+        if not _d:
             return cls(None, None)
-        ((name, value),) = d.items()
-        return cls(name, value)
+
+        ((name, value),) = _d.items()
+        return cls(name, value, size=size, nfiles=nfiles)
 
     def to_dict(self):
-        return {self.name: self.value} if self else {}
+        ret = OrderedDict()
+        if not self:
+            return ret
+
+        ret[self.name] = self.value
+        if self.size is not None:
+            ret[self.PARAM_SIZE] = self.size
+        if self.nfiles is not None:
+            ret[self.PARAM_NFILES] = self.nfiles
+        return ret
 
     @property
     def isdir(self):

dvc/info.py

@@ -8,7 +8,7 @@ from dvc.exceptions import DvcException, NotDvcRepoError
 from dvc.repo import Repo
 from dvc.scm.base import SCMError
 from dvc.system import System
-from dvc.tree import TREES
+from dvc.tree import TREES, get_tree_cls, get_tree_config
 from dvc.utils import error_link
 from dvc.utils.pkg import PKG
 from dvc.version import __version__
@@ -50,6 +50,10 @@ def get_dvc_info():
         else:
             info.append("Cache types: " + error_link("no-dvc-cache"))
 
+        info.append(f"Caches: {_get_caches(repo.cache)}")
+
+        info.append(f"Remotes: {_get_remotes(repo.config)}")
+
     except NotDvcRepoError:
         pass
     except SCMError:
@@ -63,6 +67,26 @@ def get_dvc_info():
     return "\n".join(info)
 
 
+def _get_caches(cache):
+    caches = (
+        cache_type
+        for cache_type, cache_instance in cache.by_scheme()
+        if cache_instance
+    )
+
+    # Caches will be always non-empty including the local cache
+    return ", ".join(caches)
+
+
+def _get_remotes(config):
+    schemes = (
+        get_tree_cls(get_tree_config(config, name=remote)).scheme
+        for remote in config["remote"]
+    )
+
+    return ", ".join(schemes) or "None"
+
+
 def _get_linktype_support_info(repo):
 
     links = {

dvc/lock.py

@@ -2,6 +2,7 @@
 
 import hashlib
 import os
+from abc import ABC, abstractmethod
 from datetime import timedelta
 
 import flufl.lock
@@ -14,10 +15,13 @@ from dvc.utils import format_link
 
 DEFAULT_TIMEOUT = 3
 
+
 FAILED_TO_LOCK_MESSAGE = (
-    "cannot perform the command because another DVC process seems to be "
-    "running on this project. If that is not the case, manually remove "
-    "`.dvc/tmp/lock` and try again."
+    "Unable to acquire lock. Most likely another DVC process is running or "
+    "was terminated abruptly. Check the page {} for other possible reasons "
+    "and to learn how to resolve this."
+).format(
+    format_link("https://dvc.org/doc/user-guide/troubleshooting#lock-issue")
 )
 
 
@@ -25,20 +29,74 @@ class LockError(DvcException):
     """Thrown when unable to acquire the lock for DVC repo."""
 
 
-class Lock:
+class LockBase(ABC):
+    @abstractmethod
+    def __init__(self, lockfile):
+        self._lockfile = lockfile
+
+    @property
+    def lockfile(self):
+        return self._lockfile
+
+    @abstractmethod
+    def lock(self):
+        pass
+
+    @abstractmethod
+    def unlock(self):
+        pass
+
+    @property
+    @abstractmethod
+    def is_locked(self):
+        pass
+
+    @abstractmethod
+    def __enter__(self):
+        pass
+
+    @abstractmethod
+    def __exit__(self, typ, value, tbck):
+        pass
+
+
+class LockNoop(LockBase):
+    def __init__(
+        self, *args, **kwargs
+    ):  # pylint: disable=super-init-not-called
+        self._lock = False
+
+    def lock(self):
+        self._lock = True
+
+    def unlock(self):
+        self._lock = False
+
+    @property
+    def is_locked(self):
+        return self._lock
+
+    def __enter__(self):
+        self.lock()
+
+    def __exit__(self, typ, value, tbck):
+        self.unlock()
+
+
+class Lock(LockBase):
     """Class for DVC repo lock.
 
     Uses zc.lockfile as backend.
     """
 
     def __init__(self, lockfile, friendly=False, **kwargs):
+        super().__init__(lockfile)
         self._friendly = friendly
-        self.lockfile = lockfile
         self._lock = None
 
     @property
     def files(self):
-        return [self.lockfile]
+        return [self._lockfile]
 
     def _do_lock(self):
         try:
@@ -51,7 +109,7 @@ class Lock:
                     )
                 ),
             ):
-                self._lock = zc.lockfile.LockFile(self.lockfile)
+                self._lock = zc.lockfile.LockFile(self._lockfile)
         except zc.lockfile.LockError:
             raise LockError(FAILED_TO_LOCK_MESSAGE)
 
@@ -76,7 +134,7 @@ class Lock:
         self.unlock()
 
 
-class HardlinkLock(flufl.lock.Lock):
+class HardlinkLock(flufl.lock.Lock, LockBase):
     """Class for DVC repo lock.
 
     Args:
@@ -90,6 +148,8 @@ class HardlinkLock(flufl.lock.Lock):
     ):  # pylint: disable=super-init-not-called
         import socket
 
+        super().__init__(lockfile)
+
         self._tmp_dir = tmp_dir
 
         # NOTE: this is basically Lock.__init__ copy-paste, except that
@@ -105,17 +165,12 @@ class HardlinkLock(flufl.lock.Lock):
         # [1] https://github.com/iterative/dvc/issues/2582
         self._hostname = socket.gethostname()
 
-        self._lockfile = lockfile
         self._lifetime = timedelta(days=365)  # Lock for good by default
         self._separator = flufl.lock.SEP
         self._set_claimfile()
         self._owned = True
         self._retry_errnos = []
 
-    @property
-    def lockfile(self):
-        return self._lockfile
-
     def lock(self):  # pylint: disable=arguments-differ
         try:
             super().lock(timedelta(seconds=DEFAULT_TIMEOUT))

dvc/main.py

@@ -42,6 +42,19 @@ def profile(enable, dump):
     prof.dump_stats(dump)
 
 
+@contextmanager
+def debug(enable):
+    try:
+        yield
+        return
+    except Exception:
+        if enable:
+            import pdb  # noqa: T100
+
+            pdb.post_mortem()
+        raise
+
+
 def main(argv=None):  # noqa: C901
     """Run dvc CLI command.
 
@@ -72,8 +85,9 @@ def main(argv=None):  # noqa: C901
         logger.trace(args)
 
         with profile(enable=args.cprofile, dump=args.cprofile_dump):
-            cmd = args.func(args)
-            ret = cmd.run()
+            with debug(args.pdb):
+                cmd = args.func(args)
+                ret = cmd.run()
     except ConfigError:
         logger.exception("configuration error")
         ret = 251
@@ -101,9 +115,11 @@ def main(argv=None):  # noqa: C901
             from dvc.info import get_dvc_info
 
             logger.exception("unexpected error")
-            logger.info(FOOTER)
+
             dvc_info = get_dvc_info()
-            logger.info(dvc_info)
+            logger.debug("Version info for developers:\n%s", dvc_info)
+
+            logger.info(FOOTER)
         ret = 255
 
     try:

dvc/output/__init__.py

@@ -4,24 +4,28 @@ from urllib.parse import urlparse
 from funcy import collecting, project
 from voluptuous import And, Any, Coerce, Length, Lower, Required, SetTo
 
+from dvc.hash_info import HashInfo
 from dvc.output.base import BaseOutput
 from dvc.output.gs import GSOutput
 from dvc.output.hdfs import HDFSOutput
 from dvc.output.local import LocalOutput
 from dvc.output.s3 import S3Output
 from dvc.output.ssh import SSHOutput
+from dvc.output.webhdfs import WebHDFSOutput
 from dvc.scheme import Schemes
 
 from ..tree import get_cloud_tree
 from ..tree.hdfs import HDFSTree
 from ..tree.local import LocalTree
 from ..tree.s3 import S3Tree
+from ..tree.webhdfs import WebHDFSTree
 
 OUTS = [
     HDFSOutput,
     S3Output,
     GSOutput,
     SSHOutput,
+    WebHDFSOutput,
     # NOTE: LocalOutput is the default choice
 ]
 
@@ -31,6 +35,7 @@ OUTS_MAP = {
     Schemes.GS: GSOutput,
     Schemes.SSH: SSHOutput,
     Schemes.LOCAL: LocalOutput,
+    Schemes.WEBHDFS: WebHDFSOutput,
 }
 
 CHECKSUM_SCHEMA = Any(
@@ -51,6 +56,7 @@ CHECKSUMS_SCHEMA = {
     LocalTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
     S3Tree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
     HDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
+    WebHDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
 }
 
 SCHEMA = CHECKSUMS_SCHEMA.copy()
@@ -59,10 +65,22 @@ SCHEMA[BaseOutput.PARAM_CACHE] = bool
 SCHEMA[BaseOutput.PARAM_METRIC] = BaseOutput.METRIC_SCHEMA
 SCHEMA[BaseOutput.PARAM_PLOT] = bool
 SCHEMA[BaseOutput.PARAM_PERSIST] = bool
+SCHEMA[BaseOutput.PARAM_CHECKPOINT] = bool
+SCHEMA[HashInfo.PARAM_SIZE] = int
+SCHEMA[HashInfo.PARAM_NFILES] = int
+SCHEMA[BaseOutput.PARAM_DESC] = str
 
 
 def _get(
-    stage, p, info=None, cache=True, metric=False, plot=False, persist=False
+    stage,
+    p,
+    info=None,
+    cache=True,
+    metric=False,
+    plot=False,
+    persist=False,
+    checkpoint=False,
+    desc=None,
 ):
     parsed = urlparse(p)
 
@@ -77,6 +95,8 @@ def _get(
             metric=metric,
             plot=plot,
             persist=persist,
+            checkpoint=checkpoint,
+            desc=desc,
         )
 
     for o in OUTS:
@@ -90,6 +110,8 @@ def _get(
                 metric=metric,
                 plot=plot,
                 persist=persist,
+                checkpoint=checkpoint,
+                desc=desc,
             )
     return LocalOutput(
         stage,
@@ -100,6 +122,8 @@ def _get(
         metric=metric,
         plot=plot,
         persist=persist,
+        checkpoint=checkpoint,
+        desc=desc,
     )
 
 
@@ -111,6 +135,8 @@ def loadd_from(stage, d_list):
         metric = d.pop(BaseOutput.PARAM_METRIC, False)
         plot = d.pop(BaseOutput.PARAM_PLOT, False)
         persist = d.pop(BaseOutput.PARAM_PERSIST, False)
+        checkpoint = d.pop(BaseOutput.PARAM_CHECKPOINT, False)
+        desc = d.pop(BaseOutput.PARAM_DESC, False)
         ret.append(
             _get(
                 stage,
@@ -120,13 +146,21 @@ def loadd_from(stage, d_list):
                 metric=metric,
                 plot=plot,
                 persist=persist,
+                checkpoint=checkpoint,
+                desc=desc,
             )
         )
     return ret
 
 
 def loads_from(
-    stage, s_list, use_cache=True, metric=False, plot=False, persist=False
+    stage,
+    s_list,
+    use_cache=True,
+    metric=False,
+    plot=False,
+    persist=False,
+    checkpoint=False,
 ):
     return [
         _get(
@@ -137,6 +171,7 @@ def loads_from(
             metric=metric,
             plot=plot,
             persist=persist,
+            checkpoint=checkpoint,
         )
         for s in s_list
     ]
@@ -180,5 +215,12 @@ def load_from_pipeline(stage, s_list, typ="outs"):
             from dvc.schema import PLOT_PROPS
 
             plt_d, flags = _split_dict(flags, keys=PLOT_PROPS.keys())
-        extra = project(flags, ["cache", "persist"])
+        extra = project(
+            flags,
+            [
+                BaseOutput.PARAM_CACHE,
+                BaseOutput.PARAM_PERSIST,
+                BaseOutput.PARAM_CHECKPOINT,
+            ],
+        )
         yield _get(stage, path, {}, plot=plt_d or plot, metric=metric, **extra)

dvc/output/base.py

@@ -1,6 +1,7 @@
 import logging
 import os
 from copy import copy
+from typing import Type
 from urllib.parse import urlparse
 
 from voluptuous import Any
@@ -8,6 +9,7 @@ from voluptuous import Any
 import dvc.prompt as prompt
 from dvc.cache import NamedCache
 from dvc.exceptions import (
+    CheckoutError,
     CollectCacheError,
     DvcException,
     MergeError,
@@ -60,6 +62,7 @@ class BaseOutput:
 
     PARAM_PATH = "path"
     PARAM_CACHE = "cache"
+    PARAM_CHECKPOINT = "checkpoint"
     PARAM_METRIC = "metric"
     PARAM_METRIC_TYPE = "type"
     PARAM_METRIC_XPATH = "xpath"
@@ -72,6 +75,7 @@ class BaseOutput:
     PARAM_PLOT_TITLE = "title"
     PARAM_PLOT_HEADER = "header"
     PARAM_PERSIST = "persist"
+    PARAM_DESC = "desc"
 
     METRIC_SCHEMA = Any(
         None,
@@ -82,10 +86,10 @@ class BaseOutput:
         },
     )
 
-    DoesNotExistError = OutputDoesNotExistError
-    IsNotFileOrDirError = OutputIsNotFileOrDirError
-    IsStageFileError = OutputIsStageFileError
-    IsIgnoredError = OutputIsIgnoredError
+    DoesNotExistError = OutputDoesNotExistError  # type: Type[DvcException]
+    IsNotFileOrDirError = OutputIsNotFileOrDirError  # type: Type[DvcException]
+    IsStageFileError = OutputIsStageFileError  # type: Type[DvcException]
+    IsIgnoredError = OutputIsIgnoredError  # type: Type[DvcException]
 
     sep = "/"
 
@@ -99,6 +103,8 @@ class BaseOutput:
         metric=False,
         plot=False,
         persist=False,
+        checkpoint=False,
+        desc=None,
     ):
         self._validate_output_path(path, stage)
         # This output (and dependency) objects have too many paths/urls
@@ -123,6 +129,8 @@ class BaseOutput:
         self.metric = False if self.IS_DEPENDENCY else metric
         self.plot = False if self.IS_DEPENDENCY else plot
         self.persist = persist
+        self.checkpoint = checkpoint
+        self.desc = desc
 
         self.path_info = self._parse_path(tree, path)
         if self.use_cache and self.cache is None:
@@ -281,6 +289,9 @@ class BaseOutput:
         self.hash_info = self.get_hash()
 
     def commit(self):
+        if not self.exists:
+            raise self.DoesNotExistError(self)
+
         assert self.hash_info
         if self.use_cache:
             self.cache.save(self.path_info, self.cache.tree, self.hash_info)
@@ -292,6 +303,9 @@ class BaseOutput:
         if self.IS_DEPENDENCY:
             return ret
 
+        if self.desc:
+            ret[self.PARAM_DESC] = self.desc
+
         if not self.use_cache:
             ret[self.PARAM_CACHE] = self.use_cache
 
@@ -311,6 +325,9 @@ class BaseOutput:
         if self.persist:
             ret[self.PARAM_PERSIST] = self.persist
 
+        if self.checkpoint:
+            ret[self.PARAM_CHECKPOINT] = self.checkpoint
+
         return ret
 
     def verify_metric(self):
@@ -325,6 +342,8 @@ class BaseOutput:
         progress_callback=None,
         relink=False,
         filter_info=None,
+        allow_missing=False,
+        **kwargs,
     ):
         if not self.use_cache:
             if progress_callback:
@@ -333,14 +352,20 @@ class BaseOutput:
                 )
             return None
 
-        return self.cache.checkout(
-            self.path_info,
-            self.hash_info,
-            force=force,
-            progress_callback=progress_callback,
-            relink=relink,
-            filter_info=filter_info,
-        )
+        try:
+            return self.cache.checkout(
+                self.path_info,
+                self.hash_info,
+                force=force,
+                progress_callback=progress_callback,
+                relink=relink,
+                filter_info=filter_info,
+                **kwargs,
+            )
+        except CheckoutError:
+            if allow_missing or self.checkpoint:
+                return None
+            raise
 
     def remove(self, ignore_remove=False):
         self.tree.remove(self.path_info)
@@ -385,6 +410,9 @@ class BaseOutput:
                 show_checksums=False,
                 **kwargs,
             )
+            self.hash_info.dir_info = None
+        if not self.hash_info.dir_info:
+            self.cache.set_dir_info(self.hash_info)
         return self.dir_cache
 
     def collect_used_dir_cache(
@@ -433,9 +461,7 @@ class BaseOutput:
         path = str(self.path_info)
         filter_path = str(filter_info) if filter_info else None
         is_win = os.name == "nt"
-        for entry in self.dir_cache:
-            checksum = entry[self.tree.PARAM_CHECKSUM]
-            entry_relpath = entry[self.tree.PARAM_RELPATH]
+        for entry_relpath, entry_hash_info in self.dir_cache.items():
             if is_win:
                 entry_relpath = entry_relpath.replace("/", os.sep)
             entry_path = os.path.join(path, entry_relpath)
@@ -444,7 +470,7 @@ class BaseOutput:
                 or entry_path == filter_path
                 or entry_path.startswith(filter_path + os.sep)
             ):
-                cache.add(self.scheme, checksum, entry_path)
+                cache.add(self.scheme, entry_hash_info.value, entry_path)
 
         return cache
 
@@ -515,8 +541,15 @@ class BaseOutput:
         my = self.dumpd()
         other = out.dumpd()
 
-        my.pop(self.tree.PARAM_CHECKSUM)
-        other.pop(self.tree.PARAM_CHECKSUM)
+        ignored = [
+            self.tree.PARAM_CHECKSUM,
+            HashInfo.PARAM_SIZE,
+            HashInfo.PARAM_NFILES,
+        ]
+
+        for opt in ignored:
+            my.pop(opt, None)
+            other.pop(opt, None)
 
         if my != other:
             raise MergeError(

dvc/output/gs.py

@@ -1,7 +1,7 @@
-from dvc.output.s3 import S3Output
+from dvc.output.base import BaseOutput
 
 from ..tree.gs import GSTree
 
 
-class GSOutput(S3Output):
+class GSOutput(BaseOutput):
     TREE_CLS = GSTree

dvc/output/webhdfs.py

@@ -0,0 +1,7 @@
+from dvc.output.base import BaseOutput
+
+from ..tree.webhdfs import WebHDFSTree
+
+
+class WebHDFSOutput(BaseOutput):
+    TREE_CLS = WebHDFSTree

dvc/parsing/__init__.py

@@ -0,0 +1,287 @@
+import logging
+import os
+from collections import defaultdict
+from collections.abc import Mapping, Sequence
+from copy import deepcopy
+from itertools import starmap
+from typing import TYPE_CHECKING, List, Set
+
+from funcy import join
+
+from dvc.dependency.param import ParamsDependency
+from dvc.exceptions import DvcException
+from dvc.parsing.interpolate import ParseError
+from dvc.path_info import PathInfo
+from dvc.utils import relpath
+
+from .context import (
+    Context,
+    ContextError,
+    KeyNotInContext,
+    MergeError,
+    Meta,
+    Node,
+    ParamsFileNotFound,
+    SetError,
+)
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+logger = logging.getLogger(__name__)
+
+STAGES_KWD = "stages"
+VARS_KWD = "vars"
+WDIR_KWD = "wdir"
+DEFAULT_PARAMS_FILE = ParamsDependency.DEFAULT_PARAMS_FILE
+PARAMS_KWD = "params"
+FOREACH_KWD = "foreach"
+DO_KWD = "do"
+SET_KWD = "set"
+
+DEFAULT_SENTINEL = object()
+
+JOIN = "@"
+
+
+class ResolveError(DvcException):
+    pass
+
+
+def format_and_raise(exc, msg, path):
+    spacing = "\n" if isinstance(exc, (ParseError, MergeError)) else " "
+    message = f"failed to parse {msg} in '{path}':{spacing}{str(exc)}"
+
+    # FIXME: cannot reraise because of how we log "cause" of the exception
+    # the error message is verbose, hence need control over the spacing
+    _reraise_err(ResolveError, message, from_exc=exc)
+
+
+def _reraise_err(exc_cls, *args, from_exc=None):
+    err = exc_cls(*args)
+    if from_exc and logger.isEnabledFor(logging.DEBUG):
+        raise err from from_exc
+    raise err
+
+
+class DataResolver:
+    def __init__(self, repo: "Repo", wdir: PathInfo, d: dict):
+        self.data: dict = d
+        self.wdir = wdir
+        self.repo = repo
+        self.tree = self.repo.tree
+        self.imported_files: Set[str] = set()
+        self.relpath = relpath(self.wdir / "dvc.yaml")
+
+        to_import: PathInfo = wdir / DEFAULT_PARAMS_FILE
+        if self.tree.exists(to_import):
+            self.imported_files = {os.path.abspath(to_import)}
+            self.global_ctx = Context.load_from(self.tree, to_import)
+        else:
+            self.global_ctx = Context()
+            logger.debug(
+                "%s does not exist, it won't be used in parametrization",
+                to_import,
+            )
+
+        vars_ = d.get(VARS_KWD, [])
+        try:
+            self.load_from_vars(
+                self.global_ctx, vars_, wdir, skip_imports=self.imported_files
+            )
+        except (ParamsFileNotFound, MergeError) as exc:
+            format_and_raise(exc, "'vars'", self.relpath)
+
+    def load_from_vars(
+        self,
+        context: "Context",
+        vars_: List,
+        wdir: PathInfo,
+        skip_imports: Set[str],
+        stage_name: str = None,
+    ):
+        stage_name = stage_name or ""
+        for index, item in enumerate(vars_):
+            assert isinstance(item, (str, dict))
+            if isinstance(item, str):
+                path_info = wdir / item
+                path = os.path.abspath(path_info)
+                if path in skip_imports:
+                    continue
+
+                context.merge_from(self.tree, path_info)
+                skip_imports.add(path)
+            else:
+                joiner = "." if stage_name else ""
+                meta = Meta(source=f"{stage_name}{joiner}vars[{index}]")
+                context.merge_update(Context(item, meta=meta))
+
+    def _resolve_entry(self, name: str, definition):
+        context = Context.clone(self.global_ctx)
+        if FOREACH_KWD in definition:
+            assert DO_KWD in definition
+            self.set_context_from(
+                context, definition.get(SET_KWD, {}), source=[name, "set"]
+            )
+            return self._foreach(
+                context, name, definition[FOREACH_KWD], definition[DO_KWD]
+            )
+
+        try:
+            return self._resolve_stage(context, name, definition)
+        except ContextError as exc:
+            format_and_raise(exc, f"stage '{name}'", self.relpath)
+
+    def resolve(self):
+        stages = self.data.get(STAGES_KWD, {})
+        data = join(starmap(self._resolve_entry, stages.items())) or {}
+        logger.trace("Resolved dvc.yaml:\n%s", data)
+        return {STAGES_KWD: data}
+
+    def _resolve_stage(self, context: Context, name: str, definition) -> dict:
+        definition = deepcopy(definition)
+        self.set_context_from(
+            context, definition.pop(SET_KWD, {}), source=[name, "set"]
+        )
+        wdir = self._resolve_wdir(context, name, definition.get(WDIR_KWD))
+        if self.wdir != wdir:
+            logger.debug(
+                "Stage %s has different wdir than dvc.yaml file", name
+            )
+
+        vars_ = definition.pop(VARS_KWD, [])
+        # FIXME: Should `vars` be templatized?
+        self.load_from_vars(
+            context,
+            vars_,
+            wdir,
+            skip_imports=deepcopy(self.imported_files),
+            stage_name=name,
+        )
+
+        logger.trace(  # type: ignore[attr-defined]
+            "Context during resolution of stage %s:\n%s", name, context
+        )
+
+        with context.track():
+            resolved = {}
+            for key, value in definition.items():
+                # NOTE: we do not pop "wdir", and resolve it again
+                # this does not affect anything and is done to try to
+                # track the source of `wdir` interpolation.
+                # This works because of the side-effect that we do not
+                # allow overwriting and/or str interpolating complex objects.
+                # Fix if/when those assumptions are no longer valid.
+                try:
+                    resolved[key] = context.resolve(value)
+                except (ParseError, KeyNotInContext) as exc:
+                    format_and_raise(
+                        exc, f"'stages.{name}.{key}'", self.relpath
+                    )
+
+        # FIXME: Decide if we should track them or not (it does right now)
+        params = resolved.get(PARAMS_KWD, []) + self._resolve_params(
+            context, wdir
+        )
+        if params:
+            resolved[PARAMS_KWD] = params
+        return {name: resolved}
+
+    def _resolve_params(self, context: Context, wdir):
+        tracked = defaultdict(set)
+        for src, keys in context.tracked.items():
+            tracked[str(PathInfo(src).relative_to(wdir))].update(keys)
+
+        return [{file: list(keys)} for file, keys in tracked.items()]
+
+    def _resolve_wdir(
+        self, context: Context, name: str, wdir: str = None
+    ) -> PathInfo:
+        if not wdir:
+            return self.wdir
+
+        try:
+            wdir = str(context.resolve_str(wdir, unwrap=True))
+        except (ContextError, ParseError) as exc:
+            format_and_raise(exc, f"'stages.{name}.wdir'", self.relpath)
+        return self.wdir / str(wdir)
+
+    def _foreach(self, context: Context, name: str, foreach_data, do_data):
+        iterable = self._resolve_foreach_data(context, name, foreach_data)
+        args = (context, name, do_data, iterable)
+        it = (
+            range(len(iterable))
+            if not isinstance(iterable, Mapping)
+            else iterable
+        )
+        gen = (self._each_iter(*args, i) for i in it)
+        return join(gen)
+
+    def _each_iter(self, context: Context, name: str, do_data, iterable, key):
+        value = iterable[key]
+        c = Context.clone(context)
+        suffix = c["item"] = value
+        if isinstance(iterable, Mapping):
+            suffix = c["key"] = key
+
+        generated = f"{name}{JOIN}{suffix}"
+        try:
+            return self._resolve_stage(c, generated, do_data)
+        except ContextError as exc:
+            # pylint: disable=no-member
+            if isinstance(exc, MergeError) and exc.key in self._inserted_keys(
+                iterable
+            ):
+                raise ResolveError(
+                    f"attempted to redefine '{exc.key}' in stage '{generated}'"
+                    " generated through 'foreach'"
+                )
+            format_and_raise(
+                exc, f"stage '{generated}' (gen. from '{name}')", self.relpath
+            )
+
+    def _resolve_foreach_data(
+        self, context: "Context", name: str, foreach_data
+    ):
+        try:
+            iterable = context.resolve(foreach_data, unwrap=False)
+        except (ContextError, ParseError) as exc:
+            format_and_raise(exc, f"'stages.{name}.foreach'", self.relpath)
+        if isinstance(iterable, str) or not isinstance(
+            iterable, (Sequence, Mapping)
+        ):
+            raise ResolveError(
+                f"failed to resolve 'stages.{name}.foreach'"
+                f" in '{self.relpath}': expected list/dictionary, got "
+                + type(
+                    iterable.value if isinstance(iterable, Node) else iterable
+                ).__name__
+            )
+
+        warn_for = [k for k in self._inserted_keys(iterable) if k in context]
+        if warn_for:
+            logger.warning(
+                "%s %s already specified, "
+                "will be overwritten for stages generated from '%s'",
+                " and ".join(warn_for),
+                "is" if len(warn_for) == 1 else "are",
+                name,
+            )
+
+        return iterable
+
+    @staticmethod
+    def _inserted_keys(iterable):
+        keys = ["item"]
+        if isinstance(iterable, Mapping):
+            keys.append("key")
+        return keys
+
+    @classmethod
+    def set_context_from(cls, context: Context, to_set, source=None):
+        try:
+            for key, value in to_set.items():
+                src_set = [*(source or []), key]
+                context.set(key, value, source=".".join(src_set))
+        except SetError as exc:
+            _reraise_err(ResolveError, str(exc), from_exc=exc)

dvc/parsing/context.py

@@ -0,0 +1,436 @@
+import os
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from collections.abc import Mapping, MutableMapping, MutableSequence, Sequence
+from contextlib import contextmanager
+from copy import deepcopy
+from dataclasses import dataclass, field, replace
+from typing import Any, List, Optional, Union
+
+from funcy import identity, rpartial
+
+from dvc.exceptions import DvcException
+from dvc.parsing.interpolate import (
+    ParseError,
+    get_expression,
+    get_matches,
+    is_exact_string,
+    is_interpolated_string,
+    str_interpolate,
+)
+from dvc.path_info import PathInfo
+from dvc.utils import relpath
+from dvc.utils.serialize import LOADERS
+
+SeqOrMap = Union[Sequence, Mapping]
+
+
+class ContextError(DvcException):
+    pass
+
+
+class SetError(ContextError):
+    pass
+
+
+class MergeError(ContextError):
+    def __init__(self, key, new, into):
+        self.key = key
+        if not isinstance(into[key], Node) or not isinstance(new, Node):
+            super().__init__(
+                f"cannot merge '{key}' as it already exists in {into}"
+            )
+            return
+
+        preexisting = into[key].meta.source
+        new_src = new.meta.source
+        path = new.meta.path()
+        super().__init__(
+            f"cannot redefine '{path}' from '{new_src}'"
+            f" as it already exists in '{preexisting}'"
+        )
+
+
+class ParamsFileNotFound(ContextError):
+    pass
+
+
+class KeyNotInContext(ContextError):
+    def __init__(self, key) -> None:
+        self.key = key
+        super().__init__(f"Could not find '{key}'")
+
+
+def _merge(into, update, overwrite):
+    for key, val in update.items():
+        if isinstance(into.get(key), Mapping) and isinstance(val, Mapping):
+            _merge(into[key], val, overwrite)
+        else:
+            if key in into and not overwrite:
+                raise MergeError(key, val, into)
+            into[key] = val
+            assert isinstance(into[key], Node)
+
+
+@dataclass
+class Meta:
+    source: Optional[str] = None
+    dpaths: List[str] = field(default_factory=list)
+    local: bool = True
+
+    @staticmethod
+    def update_path(meta: "Meta", path: Union[str, int]):
+        dpaths = meta.dpaths[:] + [str(path)]
+        return replace(meta, dpaths=dpaths)
+
+    def __str__(self):
+        string = self.source or "<local>"
+        string += ":" + self.path()
+        return string
+
+    def path(self):
+        return ".".join(self.dpaths)
+
+
+def _default_meta():
+    return Meta(source=None)
+
+
+class Node:
+    def get_sources(self):
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def value(self):
+        pass
+
+
+@dataclass
+class Value(Node):
+    _value: Any
+    meta: Meta = field(
+        compare=False, default_factory=_default_meta, repr=False
+    )
+
+    def __repr__(self):
+        return repr(self._value)
+
+    def __str__(self) -> str:
+        return str(self._value)
+
+    def get_sources(self):
+        return {self.meta.source: self.meta.path()}
+
+    @property
+    def value(self):
+        return self._value
+
+
+PRIMITIVES = (int, float, str, bytes, bool)
+
+
+class Container(Node, ABC):
+    meta: Meta
+    data: Union[list, dict]
+    _key_transform = staticmethod(identity)
+
+    def __init__(self, meta=None) -> None:
+        self.meta = meta or _default_meta()
+
+    def _convert(self, key, value):
+        meta = Meta.update_path(self.meta, key)
+        return self._convert_with_meta(value, meta)
+
+    @staticmethod
+    def _convert_with_meta(value, meta: Meta = None):
+        if value is None or isinstance(value, PRIMITIVES):
+            assert meta
+            return Value(value, meta=meta)
+        elif isinstance(value, Node):
+            return value
+        elif isinstance(value, (list, dict)):
+            assert meta
+            container = CtxDict if isinstance(value, dict) else CtxList
+            return container(value, meta=meta)
+        else:
+            msg = (
+                "Unsupported value of type "
+                f"'{type(value).__name__}' in '{meta}'"
+            )
+            raise TypeError(msg)
+
+    def __repr__(self):
+        return repr(self.data)
+
+    def __getitem__(self, key):
+        return self.data[key]
+
+    def __setitem__(self, key, value):
+        self.data[key] = self._convert(key, value)
+
+    def __delitem__(self, key):
+        del self.data[key]
+
+    def __len__(self):
+        return len(self.data)
+
+    def __iter__(self):
+        return iter(self.data)
+
+    def __eq__(self, o):
+        container = type(self)
+        if isinstance(o, container):
+            return o.data == self.data
+        return container(o) == self
+
+    def select(self, key: str):
+        index, *rems = key.split(sep=".", maxsplit=1)
+        index = index.strip()
+        index = self._key_transform(index)
+        try:
+            d = self[index]
+        except LookupError as exc:
+            raise ValueError(
+                f"Could not find '{index}' in {self.data}"
+            ) from exc
+
+        if not rems:
+            return d
+
+        rem = rems[0]
+        if not isinstance(d, Container):
+            raise ValueError(
+                f"{index} is a primitive value, cannot get '{rem}'"
+            )
+        return d.select(rem)
+
+    def get_sources(self):
+        return {}
+
+
+class CtxList(Container, MutableSequence):
+    _key_transform = staticmethod(int)
+
+    def __init__(self, values: Sequence, meta: Meta = None):
+        super().__init__(meta=meta)
+        self.data: list = []
+        self.extend(values)
+
+    def insert(self, index: int, value):
+        self.data.insert(index, self._convert(index, value))
+
+    def get_sources(self):
+        return {self.meta.source: self.meta.path()}
+
+    @property
+    def value(self):
+        return [node.value for node in self]
+
+
+class CtxDict(Container, MutableMapping):
+    def __init__(self, mapping: Mapping = None, meta: Meta = None, **kwargs):
+        super().__init__(meta=meta)
+
+        self.data: dict = {}
+        if mapping:
+            self.update(mapping)
+        self.update(kwargs)
+
+    def __setitem__(self, key, value):
+        if not isinstance(key, str):
+            # limitation for the interpolation
+            # ignore other kinds of keys
+            return
+        return super().__setitem__(key, value)
+
+    def merge_update(self, *args, overwrite=False):
+        for d in args:
+            _merge(self, d, overwrite=overwrite)
+
+    @property
+    def value(self):
+        return {key: node.value for key, node in self.items()}
+
+
+class Context(CtxDict):
+    def __init__(self, *args, **kwargs):
+        """
+        Top level mutable dict, with some helpers to create context and track
+        """
+        super().__init__(*args, **kwargs)
+        self._track = False
+        self._tracked_data = defaultdict(set)
+
+    @contextmanager
+    def track(self):
+        self._track = True
+        yield
+        self._track = False
+
+    def _track_data(self, node):
+        if not self._track:
+            return
+
+        if node.meta and node.meta.local:
+            return
+
+        for source, keys in node.get_sources().items():
+            if not source:
+                continue
+            params_file = self._tracked_data[source]
+            keys = [keys] if isinstance(keys, str) else keys
+            params_file.update(keys)
+
+    @property
+    def tracked(self):
+        return self._tracked_data
+
+    def select(
+        self, key: str, unwrap=False
+    ):  # pylint: disable=arguments-differ
+        """Select the item using key, similar to `__getitem__`
+           but can track the usage of the data on interpolation
+           as well and can get from nested data structure by using
+           "." separated key (eg: "key1.key2.key3")
+
+        Args:
+            key: key to select value from
+            unwrap: Convert CtxList/CtxDict/Value items to it's original data
+                    Defaults to False. Note that the default is different from
+                    `resolve`.
+        """
+        try:
+            node = super().select(key)
+        except ValueError as exc:
+            raise KeyNotInContext(key) from exc
+
+        self._track_data(node)
+        assert isinstance(node, Node)
+        return node.value if unwrap else node
+
+    @classmethod
+    def load_from(cls, tree, path: PathInfo) -> "Context":
+        file = relpath(path)
+        if not tree.exists(path):
+            raise ParamsFileNotFound(f"'{file}' does not exist")
+
+        _, ext = os.path.splitext(file)
+        loader = LOADERS[ext]
+
+        meta = Meta(source=file, local=False)
+        return cls(loader(path, tree=tree), meta=meta)
+
+    def merge_from(self, tree, path: PathInfo, overwrite=False):
+        self.merge_update(Context.load_from(tree, path), overwrite=overwrite)
+
+    @classmethod
+    def clone(cls, ctx: "Context") -> "Context":
+        """Clones given context."""
+        return cls(deepcopy(ctx.data))
+
+    def set(self, key, value, source=None):
+        """
+        Sets a value, either non-interpolated values to a key,
+        or an interpolated string after resolving it.
+
+        >>> c = Context({"foo": "foo", "bar": [1, 2], "lorem": {"a": "z"}})
+        >>> c.set("foobar", "${bar}")
+        >>> c
+        {'foo': 'foo', 'bar': [1, 2], 'lorem': {'a': 'z'}, 'foobar': [1, 2]}
+        """
+        try:
+            self._set(key, value, source)
+        except (ParseError, ValueError, ContextError) as exc:
+            sp = "\n" if isinstance(exc, ParseError) else " "
+            raise SetError(f"Failed to set '{source}':{sp}{str(exc)}") from exc
+
+    def _set(self, key, value, source):
+        if key in self:
+            raise ValueError(f"Cannot set '{key}', key already exists")
+        if isinstance(value, str):
+            self._check_joined_with_interpolation(key, value)
+            value = self.resolve_str(value, unwrap=False)
+        elif isinstance(value, (Sequence, Mapping)):
+            self._check_not_nested_collection(key, value)
+            self._check_interpolation_collection(key, value)
+        self[key] = self._convert_with_meta(value, Meta(source=source))
+
+    def resolve(self, src, unwrap=True):
+        """Recursively resolves interpolation and returns resolved data.
+
+        Args:
+            src: Data (str/list/dict etc.) to resolve
+            unwrap: Unwrap CtxDict/CtxList/Value to it's original data if
+                    inside `src`. Defaults to True.
+
+        >>> c = Context({"three": 3})
+        >>> c.resolve({"lst": [1, 2, "${three}"]})
+        {'lst': [1, 2, 3]}
+        """
+        Seq = (list, tuple, set)
+
+        resolve = rpartial(self.resolve, unwrap)
+        if isinstance(src, Mapping):
+            return dict(map(resolve, kv) for kv in src.items())
+        elif isinstance(src, Seq):
+            return type(src)(map(resolve, src))
+        elif isinstance(src, str):
+            return self.resolve_str(src, unwrap=unwrap)
+        return src
+
+    def resolve_str(self, src: str, unwrap=True):
+        """Resolves interpolated string to it's original value,
+        or in case of multiple interpolations, a combined string.
+
+        >>> c = Context({"enabled": True})
+        >>> c.resolve_str("${enabled}")
+        True
+        >>> c.resolve_str("enabled? ${enabled}")
+        'enabled? True'
+        """
+        matches = get_matches(src)
+        if is_exact_string(src, matches):
+            # replace "${enabled}", if `enabled` is a boolean, with it's actual
+            # value rather than it's string counterparts.
+            expr = get_expression(matches[0])
+            return self.select(expr, unwrap=unwrap)
+        # but not "${num} days"
+        return str_interpolate(src, matches, self)
+
+    @staticmethod
+    def _check_not_nested_collection(key: str, value: SeqOrMap):
+        values = value.values() if isinstance(value, Mapping) else value
+        has_nested = any(
+            not isinstance(item, str) and isinstance(item, (Mapping, Sequence))
+            for item in values
+        )
+        if has_nested:
+            raise ValueError(f"Cannot set '{key}', has nested dict/list")
+
+    @staticmethod
+    def _check_interpolation_collection(key: str, value: SeqOrMap):
+        values = value.values() if isinstance(value, Mapping) else value
+        interpolated = any(is_interpolated_string(item) for item in values)
+        if interpolated:
+            raise ValueError(
+                f"Cannot set '{key}', "
+                "having interpolation inside "
+                f"'{type(value).__name__}' is not supported."
+            )
+
+    @staticmethod
+    def _check_joined_with_interpolation(key: str, value: str):
+        matches = get_matches(value)
+        if matches and not is_exact_string(value, matches):
+            raise ValueError(
+                f"Cannot set '{key}', "
+                "joining string with interpolated string "
+                "is not supported"
+            )
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

dvc/parsing/interpolate.py

@@ -0,0 +1,115 @@
+import re
+import typing
+
+from pyparsing import (
+    CharsNotIn,
+    ParseException,
+    ParserElement,
+    Suppress,
+    ZeroOrMore,
+)
+
+from dvc.exceptions import DvcException
+from dvc.utils import colorize
+
+if typing.TYPE_CHECKING:
+    from typing import List, Match
+
+    from .context import Context
+
+LBRACK = "["
+RBRACK = "]"
+PERIOD = "."
+KEYCRE = re.compile(
+    r"""
+    (?<!\\)                   # escape \${} or ${{}}
+    \$                        # starts with $
+    (?:({{)|({))              # either starts with double braces or single
+    (.*?)                     # match every char inside
+    (?(1)}})(?(2)})           # end with same kinds of braces it opened with
+""",
+    re.VERBOSE,
+)
+
+ParserElement.enablePackrat()
+
+
+word = CharsNotIn(f"{PERIOD}{LBRACK}{RBRACK}")
+idx = Suppress(LBRACK) + word + Suppress(RBRACK)
+attr = Suppress(PERIOD) + word
+parser = word + ZeroOrMore(attr ^ idx)
+parser.setParseAction(PERIOD.join)
+
+
+class ParseError(DvcException):
+    pass
+
+
+def get_matches(template: str):
+    return list(KEYCRE.finditer(template))
+
+
+def is_interpolated_string(val):
+    return bool(get_matches(val)) if isinstance(val, str) else False
+
+
+def format_and_raise_parse_error(exc):
+    raise ParseError(_format_exc_msg(exc))
+
+
+def _format_exc_msg(exc: ParseException):
+    exc.loc += 2  # 2 because we append `${` at the start of expr below
+
+    expr = exc.pstr
+    exc.pstr = "${" + exc.pstr + "}"
+    error = ParseException.explain(exc, depth=0)
+
+    _, pointer, *explains = error.splitlines()
+    pstr = "{brace_open}{expr}{brace_close}".format(
+        brace_open=colorize("${", color="blue"),
+        expr=colorize(expr, color="magenta"),
+        brace_close=colorize("}", color="blue"),
+    )
+    msg = "\n".join(explains)
+    pointer = colorize(pointer, color="red")
+    return "\n".join([pstr, pointer, colorize(msg, color="red", style="bold")])
+
+
+def parse_expr(s: str):
+    try:
+        result = parser.parseString(s, parseAll=True)
+    except ParseException as exc:
+        format_and_raise_parse_error(exc)
+
+    joined = result.asList()
+    assert len(joined) == 1
+    return joined[0]
+
+
+def get_expression(match: "Match"):
+    _, _, inner = match.groups()
+    return parse_expr(inner)
+
+
+def str_interpolate(template: str, matches: "List[Match]", context: "Context"):
+    from .context import PRIMITIVES
+
+    index, buf = 0, ""
+    for match in matches:
+        start, end = match.span(0)
+        expr = get_expression(match)
+        value = context.select(expr, unwrap=True)
+        if value is not None and not isinstance(value, PRIMITIVES):
+            raise ParseError(
+                f"Cannot interpolate data of type '{type(value).__name__}'"
+            )
+        buf += template[index:start] + str(value)
+        index = end
+    buf += template[index:]
+    # regex already backtracks and avoids any `${` starting with
+    # backslashes(`\`). We just need to replace those by `${`.
+    return buf.replace(r"\${", "${")
+
+
+def is_exact_string(src: str, matches: "List[Match]"):
+    return len(matches) == 1 and src == matches[0].group(0)

dvc/path_info.py

@@ -2,6 +2,7 @@
 import os
 import pathlib
 import posixpath
+from typing import Callable
 from urllib.parse import urlparse
 
 from funcy import cached_property
@@ -75,6 +76,17 @@ class PathInfo(pathlib.PurePath, _BasePath):
         n = len(other._cparts)
         return len(self._cparts) > n and self._cparts[:n] == other._cparts
 
+    def relative_to(self, other):  # pylint: disable=arguments-differ
+        # pathlib relative_to raises exception when one path is not a direct
+        # descendant of the other when os.path.relpath would return abspath.
+        # For DVC PathInfo we only need the relpath behavior.
+        # See: https://bugs.python.org/issue40358
+        try:
+            path = super().relative_to(other)
+        except ValueError:
+            path = relpath(self, other)
+        return self.__class__(path)
+
 
 class WindowsPathInfo(PathInfo, pathlib.PureWindowsPath):
     pass
@@ -113,7 +125,6 @@ class URLInfo(_BasePath):
         p = urlparse(url)
         assert not p.query and not p.params and not p.fragment
         assert p.password is None
-
         self._fill_parts(p.scheme, p.hostname, p.username, p.port, p.path)
 
     @classmethod
@@ -243,7 +254,7 @@ class CloudURLInfo(URLInfo):
 
 
 class HTTPURLInfo(URLInfo):
-    __hash__ = URLInfo.__hash__
+    __hash__: Callable[["HTTPURLInfo"], int] = URLInfo.__hash__
 
     def __init__(self, url):
         p = urlparse(url)

dvc/progress.py

@@ -31,9 +31,12 @@ class Tqdm(tqdm):
     BAR_FMT_NOTOTAL = (
         "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
     )
-    BYTES_DEFAULTS = dict(
-        unit="B", unit_scale=True, unit_divisor=1024, miniters=1
-    )
+    BYTES_DEFAULTS = {
+        "unit": "B",
+        "unit_scale": True,
+        "unit_divisor": 1024,
+        "miniters": 1,
+    }
 
     def __init__(
         self,

dvc/remote/__init__.py

@@ -1,13 +1,10 @@
 from ..tree import get_cloud_tree
 from .base import Remote
 from .local import LocalRemote
-from .ssh import SSHRemote
 
 
 def get_remote(repo, **kwargs):
     tree = get_cloud_tree(repo, **kwargs)
     if tree.scheme == "local":
         return LocalRemote(tree)
-    if tree.scheme == "ssh":
-        return SSHRemote(tree)
     return Remote(tree)

dvc/remote/base.py

@@ -1,19 +1,59 @@
+import errno
 import hashlib
 import logging
-from functools import wraps
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from functools import partial, wraps
 
+from funcy import concat
+
+from dvc.exceptions import DownloadError, UploadError
+from dvc.hash_info import HashInfo
+
+from ..progress import Tqdm
 from .index import RemoteIndex, RemoteIndexNoop
 
 logger = logging.getLogger(__name__)
 
+STATUS_OK = 1
+STATUS_MISSING = 2
+STATUS_NEW = 3
+STATUS_DELETED = 4
+
+STATUS_MAP = {
+    # (local_exists, remote_exists)
+    (True, True): STATUS_OK,
+    (False, False): STATUS_MISSING,
+    (True, False): STATUS_NEW,
+    (False, True): STATUS_DELETED,
+}
+
+
+def _log_exceptions(func, operation):
+    @wraps(func)
+    def wrapper(from_info, to_info, *args, **kwargs):
+        try:
+            func(from_info, to_info, *args, **kwargs)
+            return 0
+        except Exception as exc:  # pylint: disable=broad-except
+            # NOTE: this means we ran out of file descriptors and there is no
+            # reason to try to proceed, as we will hit this error anyways.
+            # pylint: disable=no-member
+            if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
+                raise
+
+            logger.exception(
+                "failed to %s '%s' to '%s'", operation, from_info, to_info
+            )
+            return 1
+
+    return wrapper
+
 
 def index_locked(f):
     @wraps(f)
-    def wrapper(obj, named_cache, remote, *args, **kwargs):
-        if hasattr(remote, "index"):
-            with remote.index:
-                return f(obj, named_cache, remote, *args, **kwargs)
-        return f(obj, named_cache, remote, *args, **kwargs)
+    def wrapper(obj, *args, **kwargs):
+        with obj.index:
+            return f(obj, *args, **kwargs)
 
     return wrapper
 
@@ -28,8 +68,11 @@ class Remote:
     INDEX_CLS = RemoteIndex
 
     def __init__(self, tree):
+        from dvc.cache import get_cloud_cache
+
         self.tree = tree
         self.repo = tree.repo
+        self.cache = get_cloud_cache(self.tree)
 
         config = tree.config
         url = config.get("url")
@@ -47,123 +90,393 @@ class Remote:
             path_info=self.tree.path_info or "No path",
         )
 
-    @property
-    def cache(self):
-        return getattr(self.repo.cache, self.tree.scheme)
+    @index_locked
+    def gc(self, *args, **kwargs):
+        removed = self.cache.gc(*args, **kwargs)
 
-    def hashes_exist(self, hashes, jobs=None, name=None):
-        """Check if the given hashes are stored in the remote.
+        if removed:
+            self.index.clear()
 
-        There are two ways of performing this check:
+        return removed
 
-        - Traverse method: Get a list of all the files in the remote
-            (traversing the cache directory) and compare it with
-            the given hashes. Cache entries will be retrieved in parallel
-            threads according to prefix (i.e. entries starting with, "00...",
-            "01...", and so on) and a progress bar will be displayed.
-
-        - Exists method: For each given hash, run the `exists`
-            method and filter the hashes that aren't on the remote.
-            This is done in parallel threads.
-            It also shows a progress bar when performing the check.
-
-        The reason for such an odd logic is that most of the remotes
-        take much shorter time to just retrieve everything they have under
-        a certain prefix (e.g. s3, gs, ssh, hdfs). Other remotes that can
-        check if particular file exists much quicker, use their own
-        implementation of hashes_exist (see ssh, local).
-
-        Which method to use will be automatically determined after estimating
-        the size of the remote cache, and comparing the estimated size with
-        len(hashes). To estimate the size of the remote cache, we fetch
-        a small subset of cache entries (i.e. entries starting with "00...").
-        Based on the number of entries in that subset, the size of the full
-        cache can be estimated, since the cache is evenly distributed according
-        to hash.
-
-        Returns:
-            A list with hashes that were found in the remote
-        """
-        # Remotes which do not use traverse prefix should override
-        # hashes_exist() (see ssh, local)
-        assert self.tree.TRAVERSE_PREFIX_LEN >= 2
+    @index_locked
+    def status(
+        self,
+        cache,
+        named_cache,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+        log_missing=True,
+    ):
+        # Return flattened dict containing all status info
+        dir_status, file_status, _ = self._status(
+            cache,
+            named_cache,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
+            log_missing=log_missing,
+        )
+        return dict(dir_status, **file_status)
 
+    def hashes_exist(self, hashes, **kwargs):
         hashes = set(hashes)
         indexed_hashes = set(self.index.intersection(hashes))
         hashes -= indexed_hashes
+        indexed_hashes = list(indexed_hashes)
         logger.debug("Matched '{}' indexed hashes".format(len(indexed_hashes)))
         if not hashes:
             return indexed_hashes
 
-        if len(hashes) == 1 or not self.tree.CAN_TRAVERSE:
-            remote_hashes = self.tree.list_hashes_exists(hashes, jobs, name)
-            return list(indexed_hashes) + remote_hashes
+        return indexed_hashes + self.cache.hashes_exist(list(hashes), **kwargs)
 
-        # Max remote size allowed for us to use traverse method
-        remote_size, remote_hashes = self.tree.estimate_remote_size(
-            hashes, name
+    def _status(
+        self,
+        cache,
+        named_cache,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+        log_missing=True,
+    ):
+        """Return a tuple of (dir_status_info, file_status_info, dir_contents).
+
+        dir_status_info contains status for .dir files, file_status_info
+        contains status for all other files, and dir_contents is a dict of
+        {dir_hash: set(file_hash, ...)} which can be used to map
+        a .dir file to its file contents.
+        """
+        logger.debug(f"Preparing to collect status from {self.tree.path_info}")
+        md5s = set(named_cache.scheme_keys(cache.tree.scheme))
+
+        logger.debug("Collecting information from local cache...")
+        local_exists = frozenset(
+            cache.hashes_exist(md5s, jobs=jobs, name=cache.cache_dir)
         )
 
-        traverse_pages = remote_size / self.tree.LIST_OBJECT_PAGE_SIZE
-        # For sufficiently large remotes, traverse must be weighted to account
-        # for performance overhead from large lists/sets.
-        # From testing with S3, for remotes with 1M+ files, object_exists is
-        # faster until len(hashes) is at least 10k~100k
-        if remote_size > self.tree.TRAVERSE_THRESHOLD_SIZE:
-            traverse_weight = (
-                traverse_pages * self.tree.TRAVERSE_WEIGHT_MULTIPLIER
-            )
+        # This is a performance optimization. We can safely assume that,
+        # if the resources that we want to fetch are already cached,
+        # there's no need to check the remote storage for the existence of
+        # those files.
+        if download and local_exists == md5s:
+            remote_exists = local_exists
         else:
-            traverse_weight = traverse_pages
-        if len(hashes) < traverse_weight:
-            logger.debug(
-                "Large remote ('{}' hashes < '{}' traverse weight), "
-                "using object_exists for remaining hashes".format(
-                    len(hashes), traverse_weight
+            logger.debug("Collecting information from remote cache...")
+            remote_exists = set()
+            dir_md5s = set(named_cache.dir_keys(cache.tree.scheme))
+            if dir_md5s:
+                remote_exists.update(
+                    self._indexed_dir_hashes(cache, named_cache, dir_md5s)
                 )
-            )
-            return (
-                list(indexed_hashes)
-                + list(hashes & remote_hashes)
-                + self.tree.list_hashes_exists(
-                    hashes - remote_hashes, jobs, name
+                md5s.difference_update(remote_exists)
+            if md5s:
+                remote_exists.update(
+                    self.hashes_exist(
+                        md5s, jobs=jobs, name=str(self.tree.path_info)
+                    )
                 )
-            )
+        return self._make_status(
+            cache,
+            named_cache,
+            show_checksums,
+            local_exists,
+            remote_exists,
+            log_missing,
+        )
 
-        logger.debug("Querying '{}' hashes via traverse".format(len(hashes)))
-        remote_hashes = set(
-            self.tree.list_hashes_traverse(
-                remote_size, remote_hashes, jobs, name
+    def _make_status(
+        self,
+        cache,
+        named_cache,
+        show_checksums,
+        local_exists,
+        remote_exists,
+        log_missing,
+    ):
+        def make_names(hash_, names):
+            return {"name": hash_ if show_checksums else " ".join(names)}
+
+        dir_status = {}
+        file_status = {}
+        dir_contents = {}
+        for hash_, item in named_cache[cache.tree.scheme].items():
+            if item.children:
+                dir_status[hash_] = make_names(hash_, item.names)
+                dir_contents[hash_] = set()
+                for child_hash, child in item.children.items():
+                    file_status[child_hash] = make_names(
+                        child_hash, child.names
+                    )
+                    dir_contents[hash_].add(child_hash)
+            else:
+                file_status[hash_] = make_names(hash_, item.names)
+
+        self._fill_statuses(dir_status, local_exists, remote_exists)
+        self._fill_statuses(file_status, local_exists, remote_exists)
+
+        if log_missing:
+            self._log_missing_caches(dict(dir_status, **file_status))
+
+        return dir_status, file_status, dir_contents
+
+    def _indexed_dir_hashes(self, cache, named_cache, dir_md5s):
+        # Validate our index by verifying all indexed .dir hashes
+        # still exist on the remote
+        indexed_dirs = set(self.index.dir_hashes())
+        indexed_dir_exists = set()
+        if indexed_dirs:
+            indexed_dir_exists.update(
+                self.cache.list_hashes_exists(indexed_dirs)
+            )
+            missing_dirs = indexed_dirs.difference(indexed_dir_exists)
+            if missing_dirs:
+                logger.debug(
+                    "Remote cache missing indexed .dir hashes '{}', "
+                    "clearing remote index".format(", ".join(missing_dirs))
+                )
+                self.index.clear()
+
+        # Check if non-indexed (new) dir hashes exist on remote
+        dir_exists = dir_md5s.intersection(indexed_dir_exists)
+        dir_exists.update(self.cache.list_hashes_exists(dir_md5s - dir_exists))
+
+        # If .dir hash exists on the remote, assume directory contents
+        # still exists on the remote
+        for dir_hash in dir_exists:
+            file_hashes = list(
+                named_cache.child_keys(cache.tree.scheme, dir_hash)
+            )
+            if dir_hash not in self.index:
+                logger.debug(
+                    "Indexing new .dir '{}' with '{}' nested files".format(
+                        dir_hash, len(file_hashes)
+                    )
+                )
+                self.index.update([dir_hash], file_hashes)
+            yield dir_hash
+            yield from file_hashes
+
+    @staticmethod
+    def _fill_statuses(hash_info_dir, local_exists, remote_exists):
+        # Using sets because they are way faster for lookups
+        local = set(local_exists)
+        remote = set(remote_exists)
+
+        for md5, info in hash_info_dir.items():
+            status = STATUS_MAP[(md5 in local, md5 in remote)]
+            info["status"] = status
+
+    def _get_plans(self, cache_obj, download, status_info, status):
+        cache = []
+        path_infos = []
+        names = []
+        hashes = []
+        missing = []
+        for md5, info in Tqdm(
+            status_info.items(), desc="Analysing status", unit="file"
+        ):
+            if info["status"] == status:
+                cache.append(cache_obj.tree.hash_to_path_info(md5))
+                path_infos.append(self.tree.hash_to_path_info(md5))
+                names.append(info["name"])
+                hashes.append(md5)
+            elif info["status"] == STATUS_MISSING:
+                missing.append(md5)
+
+        if download:
+            to_infos = cache
+            from_infos = path_infos
+        else:
+            to_infos = path_infos
+            from_infos = cache
+
+        return (from_infos, to_infos, names, hashes), missing
+
+    def _process(
+        self,
+        cache,
+        named_cache,
+        jobs=None,
+        show_checksums=False,
+        download=False,
+    ):
+        logger.debug(
+            "Preparing to {} '{}'".format(
+                "download data from" if download else "upload data to",
+                self.tree.path_info,
             )
         )
-        return list(indexed_hashes) + list(hashes & set(remote_hashes))
 
-    @classmethod
+        if download:
+            func = partial(
+                _log_exceptions(self.tree.download, "download"),
+                dir_mode=cache.tree.dir_mode,
+                file_mode=cache.tree.file_mode,
+            )
+            status = STATUS_DELETED
+            desc = "Downloading"
+        else:
+            func = _log_exceptions(self.tree.upload, "upload")
+            status = STATUS_NEW
+            desc = "Uploading"
+
+        if jobs is None:
+            jobs = self.tree.JOBS
+
+        dir_status, file_status, dir_contents = self._status(
+            cache,
+            named_cache,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=download,
+        )
+
+        dir_plans, _ = self._get_plans(cache, download, dir_status, status)
+        file_plans, missing_files = self._get_plans(
+            cache, download, file_status, status
+        )
+
+        total = len(dir_plans[0]) + len(file_plans[0])
+        if total == 0:
+            return 0
+
+        with Tqdm(total=total, unit="file", desc=desc) as pbar:
+            func = pbar.wrap_fn(func)
+            with ThreadPoolExecutor(max_workers=jobs) as executor:
+                if download:
+                    from_infos, to_infos, names, _ = (
+                        d + f for d, f in zip(dir_plans, file_plans)
+                    )
+                    fails = sum(
+                        executor.map(func, from_infos, to_infos, names)
+                    )
+                else:
+                    # for uploads, push files first, and any .dir files last
+
+                    file_futures = {}
+                    for from_info, to_info, name, hash_ in zip(*file_plans):
+                        file_futures[hash_] = executor.submit(
+                            func, from_info, to_info, name
+                        )
+                    dir_futures = {}
+                    for from_info, to_info, name, dir_hash in zip(*dir_plans):
+                        # if for some reason a file contained in this dir is
+                        # missing both locally and in the remote, we want to
+                        # push whatever file content we have, but should not
+                        # push .dir file
+                        for file_hash in missing_files:
+                            if file_hash in dir_contents[dir_hash]:
+                                logger.debug(
+                                    "directory '%s' contains missing files,"
+                                    "skipping .dir file upload",
+                                    name,
+                                )
+                                break
+                        else:
+                            wait_futures = {
+                                future
+                                for file_hash, future in file_futures.items()
+                                if file_hash in dir_contents[dir_hash]
+                            }
+                            dir_futures[dir_hash] = executor.submit(
+                                self._dir_upload,
+                                func,
+                                wait_futures,
+                                from_info,
+                                to_info,
+                                name,
+                            )
+                    fails = sum(
+                        future.result()
+                        for future in concat(
+                            file_futures.values(), dir_futures.values()
+                        )
+                    )
+
+        if fails:
+            if download:
+                self.index.clear()
+                raise DownloadError(fails)
+            raise UploadError(fails)
+
+        if not download:
+            # index successfully pushed dirs
+            for dir_hash, future in dir_futures.items():
+                if future.result() == 0:
+                    file_hashes = dir_contents[dir_hash]
+                    logger.debug(
+                        "Indexing pushed dir '{}' with "
+                        "'{}' nested files".format(dir_hash, len(file_hashes))
+                    )
+                    self.index.update([dir_hash], file_hashes)
+
+        return len(dir_plans[0]) + len(file_plans[0])
+
+    @staticmethod
+    def _dir_upload(func, futures, from_info, to_info, name):
+        for future in as_completed(futures):
+            if future.result():
+                # do not upload this .dir file if any file in this
+                # directory failed to upload
+                logger.debug(
+                    "failed to upload full contents of '{}', "
+                    "aborting .dir file upload".format(name)
+                )
+                logger.error(f"failed to upload '{from_info}' to '{to_info}'")
+                return 1
+        return func(from_info, to_info, name)
+
     @index_locked
-    def gc(cls, named_cache, remote, jobs=None):
-        tree = remote.tree
-        used = set(named_cache.scheme_keys("local"))
+    def push(self, cache, named_cache, jobs=None, show_checksums=False):
+        return self._process(
+            cache,
+            named_cache,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=False,
+        )
 
-        if tree.scheme != "":
-            used.update(named_cache.scheme_keys(tree.scheme))
+    @index_locked
+    def pull(self, cache, named_cache, jobs=None, show_checksums=False):
+        ret = self._process(
+            cache,
+            named_cache,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=True,
+        )
 
-        removed = False
-        # hashes must be sorted to ensure we always remove .dir files first
-        for hash_ in sorted(
-            tree.all(jobs, str(tree.path_info)),
-            key=tree.is_dir_hash,
-            reverse=True,
-        ):
-            if hash_ in used:
-                continue
-            path_info = tree.hash_to_path_info(hash_)
-            if tree.is_dir_hash(hash_):
-                # backward compatibility
-                # pylint: disable=protected-access
-                tree._remove_unpacked_dir(hash_)
-            tree.remove(path_info)
-            removed = True
+        if not self.tree.verify:
+            with cache.tree.state:
+                for checksum in named_cache.scheme_keys("local"):
+                    cache_file = cache.tree.hash_to_path_info(checksum)
+                    if cache.tree.exists(cache_file):
+                        # We can safely save here, as existing corrupted files
+                        # will be removed upon status, while files corrupted
+                        # during download will not be moved from tmp_file
+                        # (see `BaseTree.download()`)
+                        hash_info = HashInfo(
+                            cache.tree.PARAM_CHECKSUM, checksum
+                        )
+                        cache.tree.state.save(cache_file, hash_info)
 
-        if removed and hasattr(remote, "index"):
-            remote.index.clear()
-        return removed
+        return ret
+
+    @staticmethod
+    def _log_missing_caches(hash_info_dict):
+        missing_caches = [
+            (md5, info)
+            for md5, info in hash_info_dict.items()
+            if info["status"] == STATUS_MISSING
+        ]
+        if missing_caches:
+            missing_desc = "\n".join(
+                "name: {}, md5: {}".format(info["name"], md5)
+                for md5, info in missing_caches
+            )
+            msg = (
+                "Some of the cache files do not exist neither locally "
+                "nor on remote. Missing cache files:\n{}".format(missing_desc)
+            )
+            logger.warning(msg)

dvc/remote/index.py

@@ -6,6 +6,7 @@ import threading
 from funcy import lchunks
 
 from dvc.state import _connect_sqlite
+from dvc.utils.fs import makedirs
 
 logger = logging.getLogger(__name__)
 
@@ -67,9 +68,12 @@ class RemoteIndex:
     VERSION = 1
     INDEX_TABLE = "remote_index"
     INDEX_TABLE_LAYOUT = "checksum TEXT PRIMARY KEY, dir INTEGER NOT NULL"
+    INDEX_DIR = "index"
 
     def __init__(self, repo, name, dir_suffix=".dir"):
-        self.path = os.path.join(repo.index_dir, f"{name}{self.INDEX_SUFFIX}")
+        self.path = os.path.join(
+            repo.tmp_dir, self.INDEX_DIR, f"{name}{self.INDEX_SUFFIX}"
+        )
 
         self.dir_suffix = dir_suffix
         self.database = None
@@ -150,6 +154,7 @@ class RemoteIndex:
             assert self.cursor is None
 
             empty = not os.path.isfile(self.path)
+            makedirs(os.path.dirname(self.path), exist_ok=True)
             self.database = _connect_sqlite(self.path, {"nolock": 1})
             self.cursor = self.database.cursor()
 

dvc/remote/local.py

@@ -3,4 +3,4 @@ from .index import RemoteIndexNoop
 
 
 class LocalRemote(Remote):
-    INDEX_CLS = RemoteIndexNoop
+    INDEX_CLS = RemoteIndexNoop  # type: ignore[assignment]

dvc/remote/slow_link_detection.py

@@ -1,3 +1,4 @@
+# type: ignore
 import logging
 import sys
 import time

dvc/repo/__init__.py

@@ -2,8 +2,10 @@ import logging
 import os
 from contextlib import contextmanager
 from functools import wraps
+from typing import TYPE_CHECKING
 
-from funcy import cached_property, cat, first
+from funcy import cached_property, cat
+from git import InvalidGitRepositoryError
 
 from dvc.config import Config
 from dvc.dvcfile import PIPELINE_FILE, Dvcfile, is_valid_filename
@@ -15,21 +17,28 @@ from dvc.exceptions import (
     OutputNotFoundError,
 )
 from dvc.path_info import PathInfo
+from dvc.scm import Base
+from dvc.scm.base import SCMError
 from dvc.tree.repo import RepoTree
 from dvc.utils.fs import path_isin
 
 from ..stage.exceptions import StageFileDoesNotExistError, StageNotFound
 from ..utils import parse_target
-from .graph import check_acyclic, get_pipeline, get_pipelines
+from .graph import build_graph, build_outs_graph, get_pipeline, get_pipelines
+from .trie import build_outs_trie
+
+if TYPE_CHECKING:
+    from dvc.tree.base import BaseTree
+
 
 logger = logging.getLogger(__name__)
 
 
 @contextmanager
-def lock_repo(repo):
+def lock_repo(repo: "Repo"):
     # pylint: disable=protected-access
-    depth = getattr(repo, "_lock_depth", 0)
-    repo._lock_depth = depth + 1
+    depth = repo._lock_depth
+    repo._lock_depth += 1
 
     try:
         if depth > 0:
@@ -65,12 +74,12 @@ class Repo:
     from dvc.repo.fetch import fetch
     from dvc.repo.freeze import freeze, unfreeze
     from dvc.repo.gc import gc
-    from dvc.repo.get import get
-    from dvc.repo.get_url import get_url
+    from dvc.repo.get import get as _get
+    from dvc.repo.get_url import get_url as _get_url
     from dvc.repo.imp import imp
     from dvc.repo.imp_url import imp_url
     from dvc.repo.install import install
-    from dvc.repo.ls import ls
+    from dvc.repo.ls import ls as _ls
     from dvc.repo.move import move
     from dvc.repo.pull import pull
     from dvc.repo.push import push
@@ -80,87 +89,125 @@ class Repo:
     from dvc.repo.status import status
     from dvc.repo.update import update
 
-    def __init__(self, root_dir=None, scm=None, rev=None, subrepos=False):
+    ls = staticmethod(_ls)
+    get = staticmethod(_get)
+    get_url = staticmethod(_get_url)
+
+    def _get_repo_dirs(
+        self,
+        root_dir: str = None,
+        scm: Base = None,
+        rev: str = None,
+        uninitialized: bool = False,
+    ):
+        assert bool(scm) == bool(rev)
+
+        from dvc.scm import SCM
+        from dvc.scm.git import Git
+        from dvc.utils.fs import makedirs
+
+        dvc_dir = None
+        tmp_dir = None
+        try:
+            tree = scm.get_tree(rev) if isinstance(scm, Git) and rev else None
+            root_dir = self.find_root(root_dir, tree)
+            dvc_dir = os.path.join(root_dir, self.DVC_DIR)
+            tmp_dir = os.path.join(dvc_dir, "tmp")
+            makedirs(tmp_dir, exist_ok=True)
+        except NotDvcRepoError:
+            if not uninitialized:
+                raise
+
+            try:
+                scm = SCM(root_dir or os.curdir)
+            except (SCMError, InvalidGitRepositoryError):
+                scm = SCM(os.curdir, no_scm=True)
+
+            assert isinstance(scm, Base)
+            root_dir = scm.root_dir
+
+        return root_dir, dvc_dir, tmp_dir
+
+    def __init__(
+        self,
+        root_dir=None,
+        scm=None,
+        rev=None,
+        subrepos=False,
+        uninitialized=False,
+    ):
         from dvc.cache import Cache
         from dvc.data_cloud import DataCloud
-        from dvc.lock import make_lock
+        from dvc.lock import LockNoop, make_lock
         from dvc.repo.experiments import Experiments
         from dvc.repo.metrics import Metrics
         from dvc.repo.params import Params
         from dvc.repo.plots import Plots
-        from dvc.scm import SCM
         from dvc.stage.cache import StageCache
         from dvc.state import State, StateNoop
         from dvc.tree.local import LocalTree
-        from dvc.utils.fs import makedirs
 
-        if scm:
-            tree = scm.get_tree(rev)
-            self.root_dir = self.find_root(root_dir, tree)
-            self.scm = scm
-            self.tree = scm.get_tree(
-                rev, use_dvcignore=True, dvcignore_root=self.root_dir
-            )
-            self.state = StateNoop()
-        else:
-            root_dir = self.find_root(root_dir)
-            self.root_dir = os.path.abspath(os.path.realpath(root_dir))
-            self.tree = LocalTree(
-                self,
-                {"url": self.root_dir},
-                use_dvcignore=True,
-                dvcignore_root=self.root_dir,
-            )
-
-        self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
-        self.config = Config(self.dvc_dir, tree=self.tree)
-
-        if not scm:
-            no_scm = self.config["core"].get("no_scm", False)
-            self.scm = SCM(self.root_dir, no_scm=no_scm)
-
-        self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
-        self.index_dir = os.path.join(self.tmp_dir, "index")
-        makedirs(self.index_dir, exist_ok=True)
-
-        hardlink_lock = self.config["core"].get("hardlink_lock", False)
-        self.lock = make_lock(
-            os.path.join(self.tmp_dir, "lock"),
-            tmp_dir=self.tmp_dir,
-            hardlink_lock=hardlink_lock,
-            friendly=True,
+        self.root_dir, self.dvc_dir, self.tmp_dir = self._get_repo_dirs(
+            root_dir=root_dir, scm=scm, rev=rev, uninitialized=uninitialized
         )
+
+        tree_kwargs = {"use_dvcignore": True, "dvcignore_root": self.root_dir}
+        if scm:
+            self.tree = scm.get_tree(rev, **tree_kwargs)
+        else:
+            self.tree = LocalTree(self, {"url": self.root_dir}, **tree_kwargs)
+
+        self.config = Config(self.dvc_dir, tree=self.tree)
+        self._scm = scm
+
         # used by RepoTree to determine if it should traverse subrepos
         self.subrepos = subrepos
 
         self.cache = Cache(self)
         self.cloud = DataCloud(self)
 
-        if not scm:
+        if scm or not self.dvc_dir:
+            self.lock = LockNoop()
+            self.state = StateNoop()
+        else:
+            self.lock = make_lock(
+                os.path.join(self.tmp_dir, "lock"),
+                tmp_dir=self.tmp_dir,
+                hardlink_lock=self.config["core"].get("hardlink_lock", False),
+                friendly=True,
+            )
+
             # NOTE: storing state and link_state in the repository itself to
             # avoid any possible state corruption in 'shared cache dir'
             # scenario.
-            self.state = State(self.cache.local)
+            self.state = State(self)
+            self.stage_cache = StageCache(self)
 
-        self.stage_cache = StageCache(self)
+            try:
+                self.experiments = Experiments(self)
+            except NotImplementedError:
+                self.experiments = None
+
+            self._ignore()
 
         self.metrics = Metrics(self)
         self.plots = Plots(self)
         self.params = Params(self)
+        self._lock_depth = 0
 
-        try:
-            self.experiments = Experiments(self)
-        except NotImplementedError:
-            self.experiments = None
+    @cached_property
+    def scm(self):
+        from dvc.scm import SCM
 
-        self._ignore()
+        no_scm = self.config["core"].get("no_scm", False)
+        return self._scm if self._scm else SCM(self.root_dir, no_scm=no_scm)
 
     @property
-    def tree(self):
+    def tree(self) -> "BaseTree":
         return self._tree
 
     @tree.setter
-    def tree(self, tree):
+    def tree(self, tree: "BaseTree"):
         self._tree = tree
         # Our graph cache is no longer valid, as it was based on the previous
         # tree.
@@ -170,7 +217,7 @@ class Repo:
         return f"{self.__class__.__name__}: '{self.root_dir}'"
 
     @classmethod
-    def find_root(cls, root=None, tree=None):
+    def find_root(cls, root=None, tree=None) -> str:
         root_dir = os.path.realpath(root or os.curdir)
 
         if tree:
@@ -257,7 +304,7 @@ class Repo:
         #
         # [1] https://github.com/iterative/dvc/issues/2671
         if not getattr(self, "_skip_graph_checks", False):
-            self._collect_graph(self.stages + new_stages)
+            build_graph(self.stages + new_stages)
 
     def _collect_inside(self, path, graph):
         import networkx as nx
@@ -416,114 +463,17 @@ class Repo:
 
         return cache
 
-    def _collect_graph(self, stages):
-        """Generate a graph by using the given stages on the given directory
-
-        The nodes of the graph are the stage's path relative to the root.
-
-        Edges are created when the output of one stage is used as a
-        dependency in other stage.
-
-        The direction of the edges goes from the stage to its dependency:
-
-        For example, running the following:
-
-            $ dvc run -o A "echo A > A"
-            $ dvc run -d A -o B "echo B > B"
-            $ dvc run -d B -o C "echo C > C"
-
-        Will create the following graph:
-
-               ancestors <--
-                           |
-                C.dvc -> B.dvc -> A.dvc
-                |          |
-                |          --> descendants
-                |
-                ------- pipeline ------>
-                           |
-                           v
-              (weakly connected components)
-
-        Args:
-            stages (list): used to build a graph, if None given, collect stages
-                in the repository.
-
-        Raises:
-            OutputDuplicationError: two outputs with the same path
-            StagePathAsOutputError: stage inside an output directory
-            OverlappingOutputPathsError: output inside output directory
-            CyclicGraphError: resulting graph has cycles
-        """
-        import networkx as nx
-        from pygtrie import Trie
-
-        from dvc.exceptions import (
-            OutputDuplicationError,
-            OverlappingOutputPathsError,
-            StagePathAsOutputError,
-        )
-
-        G = nx.DiGraph()
-        stages = stages or self.stages
-        outs = Trie()  # Use trie to efficiently find overlapping outs and deps
-
-        for stage in filter(bool, stages):  # bug? not using it later
-            for out in stage.outs:
-                out_key = out.path_info.parts
-
-                # Check for dup outs
-                if out_key in outs:
-                    dup_stages = [stage, outs[out_key].stage]
-                    raise OutputDuplicationError(str(out), dup_stages)
-
-                # Check for overlapping outs
-                if outs.has_subtrie(out_key):
-                    parent = out
-                    overlapping = first(outs.values(prefix=out_key))
-                else:
-                    parent = outs.shortest_prefix(out_key).value
-                    overlapping = out
-                if parent and overlapping:
-                    msg = (
-                        "Paths for outs:\n'{}'('{}')\n'{}'('{}')\n"
-                        "overlap. To avoid unpredictable behaviour, "
-                        "rerun command with non overlapping outs paths."
-                    ).format(
-                        str(parent),
-                        parent.stage.addressing,
-                        str(overlapping),
-                        overlapping.stage.addressing,
-                    )
-                    raise OverlappingOutputPathsError(parent, overlapping, msg)
-
-                outs[out_key] = out
-
-        for stage in stages:
-            out = outs.shortest_prefix(PathInfo(stage.path).parts).value
-            if out:
-                raise StagePathAsOutputError(stage, str(out))
-
-        # Building graph
-        G.add_nodes_from(stages)
-        for stage in stages:
-            for dep in stage.deps:
-                if dep.path_info is None:
-                    continue
-
-                dep_key = dep.path_info.parts
-                overlapping = [n.value for n in outs.prefixes(dep_key)]
-                if outs.has_subtrie(dep_key):
-                    overlapping.extend(outs.values(prefix=dep_key))
-
-                G.add_edges_from((stage, out.stage) for out in overlapping)
-        check_acyclic(G)
-
-        return G
+    @cached_property
+    def outs_trie(self):
+        return build_outs_trie(self.stages)
 
     @cached_property
     def graph(self):
-        return self._collect_graph(self.stages)
+        return build_graph(self.stages, self.outs_trie)
+
+    @cached_property
+    def outs_graph(self):
+        return build_outs_graph(self.graph, self.outs_trie)
 
     @cached_property
     def pipelines(self):
@@ -542,12 +492,6 @@ class Repo:
         """
         return self._collect_stages()
 
-    @cached_property
-    def plot_templates(self):
-        from .plots.template import PlotTemplates
-
-        return PlotTemplates(self.dvc_dir)
-
     def _collect_stages(self):
         stages = []
         outs = set()
@@ -606,7 +550,7 @@ class Repo:
         """Opens a specified resource as a file descriptor"""
 
         tree = RepoTree(self, stream=True, subrepos=True)
-        path = os.path.join(self.root_dir, path)
+        path = PathInfo(self.root_dir) / path
         try:
             with self.state:
                 with tree.open(
@@ -622,6 +566,8 @@ class Repo:
         self.scm.close()
 
     def _reset(self):
+        self.__dict__.pop("outs_trie", None)
+        self.__dict__.pop("outs_graph", None)
         self.__dict__.pop("graph", None)
         self.__dict__.pop("stages", None)
         self.__dict__.pop("pipelines", None)

dvc/repo/add.py

@@ -6,6 +6,7 @@ import colorama
 from dvc.dvcfile import Dvcfile, is_dvc_file
 
 from ..exceptions import (
+    CacheLinkError,
     OutputDuplicationError,
     OverlappingOutputPathsError,
     RecursiveAddingWhileUsingFilename,
@@ -22,7 +23,7 @@ logger = logging.getLogger(__name__)
 @locked
 @scm_context
 def add(
-    repo, targets, recursive=False, no_commit=False, fname=None, external=False
+    repo, targets, recursive=False, no_commit=False, fname=None, **kwargs,
 ):
     if recursive and fname:
         raise RecursiveAddingWhileUsingFilename()
@@ -30,6 +31,7 @@ def add(
     if isinstance(targets, str):
         targets = [targets]
 
+    link_failures = []
     stages_list = []
     num_targets = len(targets)
     with Tqdm(total=num_targets, desc="Add", unit="file", leave=True) as pbar:
@@ -55,7 +57,7 @@ def add(
                 )
 
             stages = _create_stages(
-                repo, sub_targets, fname, pbar=pbar, external=external
+                repo, sub_targets, fname, pbar=pbar, **kwargs,
             )
 
             try:
@@ -79,33 +81,56 @@ def add(
                     exc.output, set(exc.stages) - set(stages)
                 )
 
-            with Tqdm(
-                total=len(stages),
-                desc="Processing",
-                unit="file",
-                disable=len(stages) == 1,
-            ) as pbar_stages:
-                for stage in stages:
-                    try:
-                        stage.save()
-                    except OutputDoesNotExistError:
-                        pbar.n -= 1
-                        raise
-
-                    if not no_commit:
-                        stage.commit()
-
-                    Dvcfile(repo, stage.path).dump(stage)
-                    pbar_stages.update()
-
+            link_failures.extend(
+                _process_stages(repo, stages, no_commit, pbar)
+            )
             stages_list += stages
 
         if num_targets == 1:  # restore bar format for stats
             pbar.bar_format = pbar.BAR_FMT_DEFAULT
 
+    if link_failures:
+        msg = (
+            "Some targets could not be linked from cache to workspace.\n{}\n"
+            "To re-link these targets, reconfigure cache types and then run:\n"
+            "\n\tdvc checkout {}"
+        ).format(
+            CacheLinkError.SUPPORT_LINK,
+            " ".join([str(stage.relpath) for stage in link_failures]),
+        )
+        logger.warning(msg)
+
     return stages_list
 
 
+def _process_stages(repo, stages, no_commit, pbar):
+    link_failures = []
+
+    with Tqdm(
+        total=len(stages),
+        desc="Processing",
+        unit="file",
+        disable=len(stages) == 1,
+    ) as pbar_stages:
+        for stage in stages:
+            try:
+                stage.save()
+            except OutputDoesNotExistError:
+                pbar.n -= 1
+                raise
+
+            try:
+                if not no_commit:
+                    stage.commit()
+            except CacheLinkError:
+                link_failures.append(stage)
+
+            Dvcfile(repo, stage.path).dump(stage)
+            pbar_stages.update()
+
+    return link_failures
+
+
 def _find_all_targets(repo, target, recursive):
     if os.path.isdir(target) and recursive:
         return [
@@ -124,15 +149,27 @@ def _find_all_targets(repo, target, recursive):
     return [target]
 
 
-def _create_stages(repo, targets, fname, pbar=None, external=False):
+def _create_stages(
+    repo, targets, fname, pbar=None, external=False, glob=False, desc=None,
+):
+    from glob import iglob
+
     from dvc.stage import Stage, create_stage
 
-    stages = []
+    if glob:
+        expanded_targets = [
+            exp_target
+            for target in targets
+            for exp_target in iglob(target, recursive=True)
+        ]
+    else:
+        expanded_targets = targets
 
+    stages = []
     for out in Tqdm(
-        targets,
+        expanded_targets,
         desc="Creating DVC-files",
-        disable=len(targets) < LARGE_DIR_SIZE,
+        disable=len(expanded_targets) < LARGE_DIR_SIZE,
         unit="file",
     ):
         path, wdir, out = resolve_paths(repo, out)
@@ -146,6 +183,8 @@ def _create_stages(repo, targets, fname, pbar=None, external=False):
         )
         if stage:
             Dvcfile(repo, stage.path).remove()
+            if desc:
+                stage.outs[0].desc = desc
 
         repo._reset()  # pylint: disable=protected-access
 

dvc/repo/brancher.py

@@ -4,7 +4,12 @@ from dvc.tree.local import LocalTree
 
 
 def brancher(  # noqa: E302
-    self, revs=None, all_branches=False, all_tags=False, all_commits=False
+    self,
+    revs=None,
+    all_branches=False,
+    all_tags=False,
+    all_commits=False,
+    sha_only=False,
 ):
     """Generator that iterates over specified revisions.
 
@@ -13,6 +18,7 @@ def brancher(  # noqa: E302
         all_branches (bool): iterate over all available branches.
         all_commits (bool): iterate over all commits.
         all_tags (bool): iterate over all available tags.
+        sha_only (bool): only return git SHA for a revision.
 
     Yields:
         str: the display name for the currently selected tree, it could be:
@@ -53,6 +59,9 @@ def brancher(  # noqa: E302
                 # ignore revs that don't contain repo root
                 # (i.e. revs from before a subdir=True repo was init'ed)
                 if self.tree.exists(self.root_dir):
-                    yield ", ".join(names)
+                    if sha_only:
+                        yield sha
+                    else:
+                        yield ", ".join(names)
     finally:
         self.tree = saved_tree

dvc/repo/checkout.py

@@ -46,6 +46,8 @@ def checkout(
     force=False,
     relink=False,
     recursive=False,
+    allow_missing=False,
+    **kwargs,
 ):
     from dvc.stage.exceptions import (
         StageFileBadNameError,
@@ -96,6 +98,8 @@ def checkout(
                 progress_callback=pbar.update_msg,
                 relink=relink,
                 filter_info=filter_info,
+                allow_missing=allow_missing,
+                **kwargs,
             )
             for key, items in result.items():
                 stats[key].extend(_fspath_dir(path) for path in items)

dvc/repo/collect.py

@@ -0,0 +1,87 @@
+import logging
+import os
+from typing import Callable, Iterable, Set, Tuple
+
+from dvc.output.base import BaseOutput
+from dvc.path_info import PathInfo
+from dvc.repo import Repo
+from dvc.tree.repo import RepoTree
+from dvc.types import DvcPath
+
+logger = logging.getLogger(__name__)
+
+
+FilterFn = Callable[[BaseOutput], bool]
+Outputs = Set[BaseOutput]
+DvcPaths = Set[DvcPath]
+
+
+def _collect_outs(
+    repo: Repo, output_filter: FilterFn = None, deps: bool = False,
+) -> Outputs:
+    outs = {
+        out
+        for stage in repo.stages
+        for out in (stage.deps if deps else stage.outs)
+    }
+    return set(filter(output_filter, outs)) if output_filter else outs
+
+
+def _collect_paths(
+    repo: Repo,
+    targets: Iterable[str],
+    recursive: bool = False,
+    rev: str = None,
+):
+    path_infos = {PathInfo(os.path.abspath(target)) for target in targets}
+    tree = RepoTree(repo)
+
+    target_infos = set()
+    for path_info in path_infos:
+
+        if recursive and tree.isdir(path_info):
+            target_infos.update(set(tree.walk_files(path_info)))
+
+        if not tree.isfile(path_info):
+            if not recursive:
+                logger.warning(
+                    "'%s' was not found at: '%s'.", path_info, rev,
+                )
+            continue
+        target_infos.add(path_info)
+    return target_infos
+
+
+def _filter_duplicates(
+    outs: Outputs, path_infos: DvcPaths
+) -> Tuple[Outputs, DvcPaths]:
+    res_outs: Outputs = set()
+    res_infos = set(path_infos)
+
+    for out in outs:
+        if out.path_info in path_infos:
+            res_outs.add(out)
+            res_infos.remove(out.path_info)
+
+    return res_outs, res_infos
+
+
+def collect(
+    repo: Repo,
+    deps: bool = False,
+    targets: Iterable[str] = None,
+    output_filter: FilterFn = None,
+    rev: str = None,
+    recursive: bool = False,
+) -> Tuple[Outputs, DvcPaths]:
+    assert targets or output_filter
+
+    outs: Outputs = _collect_outs(repo, output_filter=output_filter, deps=deps)
+
+    if not targets:
+        path_infos: DvcPaths = set()
+        return outs, path_infos
+
+    target_infos = _collect_paths(repo, targets, recursive=recursive, rev=rev)
+
+    return _filter_duplicates(outs, target_infos)

dvc/repo/diff.py

@@ -111,7 +111,7 @@ def _dir_output_paths(repo_tree, output):
 
     try:
         for fname in repo_tree.walk_files(output.path_info):
-            yield str(fname), repo_tree.get_hash(fname).value
+            yield str(fname), repo_tree.get_file_hash(fname).value
     except NoRemoteError:
         logger.warning("dir cache entry for '%s' is missing", output)
 
@@ -122,5 +122,5 @@ def _filter_missing(repo, paths):
         metadata = repo_tree.metadata(path)
         if metadata.is_dvc:
             out = metadata.outs[0]
-            if out.status()[str(out)] == "not in cache":
+            if out.status().get(str(out)) == "not in cache":
                 yield path

dvc/repo/experiments/__init__.py

@@ -1,24 +1,26 @@
 import logging
 import os
 import re
+import stat
 import tempfile
+import threading
+from collections import namedtuple
 from collections.abc import Mapping
-from concurrent.futures import (
-    ProcessPoolExecutor,
-    ThreadPoolExecutor,
-    as_completed,
-)
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from contextlib import contextmanager
 from functools import partial, wraps
-from typing import Iterable, Optional
+from typing import Iterable, Optional, no_type_check
 
 from funcy import cached_property, first
 
+from dvc.dvcfile import is_lock_file
 from dvc.exceptions import DownloadError, DvcException, UploadError
 from dvc.path_info import PathInfo
 from dvc.progress import Tqdm
 from dvc.repo.experiments.executor import ExperimentExecutor, LocalExecutor
 from dvc.scm.git import Git
+from dvc.stage import PipelineStage
+from dvc.stage.run import CheckpointKilledError
 from dvc.stage.serialize import to_lockfile
 from dvc.tree.repo import RepoTree
 from dvc.utils import dict_sha256, env2bool, relpath
@@ -41,7 +43,8 @@ def scm_locked(f):
 def hash_exp(stages):
     exp_data = {}
     for stage in stages:
-        exp_data.update(to_lockfile(stage))
+        if isinstance(stage, PipelineStage):
+            exp_data.update(to_lockfile(stage))
     return dict_sha256(exp_data)
 
 
@@ -63,6 +66,28 @@ class BaselineMismatchError(DvcException):
         self.expected_rev = expected
 
 
+class CheckpointExistsError(DvcException):
+    def __init__(self, rev, continue_rev):
+        msg = (
+            f"Checkpoint experiment containing '{rev[:7]}' already exists."
+            " To restart the experiment run:\n\n"
+            "\tdvc exp run -f ...\n\n"
+            "To resume the experiment, run:\n\n"
+            f"\tdvc exp resume {continue_rev[:7]}\n"
+        )
+        super().__init__(msg)
+        self.rev = rev
+
+
+class MultipleBranchError(DvcException):
+    def __init__(self, rev):
+        super().__init__(
+            f"Ambiguous commit '{rev[:7]}' belongs to multiple experiment "
+            "branches."
+        )
+        self.rev = rev
+
+
 class Experiments:
     """Class that manages experiments in a DVC repo.
 
@@ -72,12 +97,20 @@ class Experiments:
 
     EXPERIMENTS_DIR = "experiments"
     PACKED_ARGS_FILE = "repro.dat"
-    STASH_MSG_PREFIX = "dvc-exp-"
+    STASH_EXPERIMENT_FORMAT = "dvc-exp:{rev}:{baseline_rev}"
     STASH_EXPERIMENT_RE = re.compile(
-        r"(?:On \(.*\): )dvc-exp-(?P<baseline_rev>[0-9a-f]+)$"
+        r"(?:On \(.*\): )"
+        r"dvc-exp:(?P<rev>[0-9a-f]+):(?P<baseline_rev>[0-9a-f]+)"
+        r"(:(?P<branch>.+))?$"
     )
     BRANCH_RE = re.compile(
-        r"^(?P<baseline_rev>[a-f0-9]{7})-(?P<exp_sha>[a-f0-9]+)$"
+        r"^(?P<baseline_rev>[a-f0-9]{7})-(?P<exp_sha>[a-f0-9]+)"
+        r"(?P<checkpoint>-checkpoint)?$"
+    )
+    LAST_CHECKPOINT = ":last"
+
+    StashEntry = namedtuple(
+        "StashEntry", ["index", "rev", "baseline_rev", "branch"]
     )
 
     def __init__(self, repo):
@@ -134,8 +167,10 @@ class Experiments:
 
     @property
     def stash_reflog(self):
+        """Return stash reflog in Git 'stash@{...}' index order."""
         if "refs/stash" in self.scm.repo.refs:
-            return self.scm.repo.refs["refs/stash"].log()
+            # gitpython reflog log() returns commits oldest to newest
+            return reversed(self.scm.repo.refs["refs/stash"].log())
         return []
 
     @property
@@ -144,7 +179,12 @@ class Experiments:
         for i, entry in enumerate(self.stash_reflog):
             m = self.STASH_EXPERIMENT_RE.match(entry.message)
             if m:
-                revs[entry.newhexsha] = (i, m.group("baseline_rev"))
+                revs[entry.newhexsha] = self.StashEntry(
+                    i,
+                    m.group("rev"),
+                    m.group("baseline_rev"),
+                    m.group("branch"),
+                )
         return revs
 
     def _init_clone(self):
@@ -164,19 +204,37 @@ class Experiments:
 
     def _scm_checkout(self, rev):
         self.scm.repo.git.reset(hard=True)
+        self.scm.repo.git.clean(force=True)
         if self.scm.repo.head.is_detached:
             self._checkout_default_branch()
-        if not Git.is_sha(rev) or not self.scm.has_rev(rev):
+        if not self.scm.has_rev(rev):
             self.scm.pull()
         logger.debug("Checking out experiment commit '%s'", rev)
         self.scm.checkout(rev)
 
     def _checkout_default_branch(self):
+        from git.refs.symbolic import SymbolicReference
+
         # switch to default branch
         git_repo = self.scm.repo
+        git_repo.git.reset(hard=True)
+        git_repo.git.clean(force=True)
         origin_refs = git_repo.remotes["origin"].refs
-        ref = origin_refs["HEAD"].reference
+
+        # origin/HEAD will point to tip of the default branch unless we
+        # initially cloned a repo that was in a detached-HEAD state.
+        #
+        # If we are currently detached because we cloned a detached
+        # repo, we can't actually tell what branch should be considered
+        # default, so we just fall back to the first available reference.
+        if "HEAD" in origin_refs:
+            ref = origin_refs["HEAD"].reference
+        else:
+            ref = origin_refs[0]
+            if not isinstance(ref, SymbolicReference):
+                ref = ref.reference
         branch_name = ref.name.split("/")[-1]
+
         if branch_name in git_repo.heads:
             branch = git_repo.heads[branch_name]
         else:
@@ -184,45 +242,98 @@ class Experiments:
             branch.set_tracking_branch(ref)
         branch.checkout()
 
-    def _stash_exp(self, *args, params: Optional[dict] = None, **kwargs):
+    def _stash_exp(
+        self,
+        *args,
+        params: Optional[dict] = None,
+        baseline_rev: Optional[str] = None,
+        branch: Optional[str] = None,
+        allow_unchanged: Optional[bool] = True,
+        apply_workspace: Optional[bool] = True,
+        **kwargs,
+    ):
         """Stash changes from the current (parent) workspace as an experiment.
 
         Args:
             params: Optional dictionary of parameter values to be used.
                 Values take priority over any parameters specified in the
                 user's workspace.
+            baseline_rev: Optional baseline rev for this experiment, defaults
+                to the current SCM rev.
+            branch: Optional experiment branch name. If specified, the
+                experiment will be added to `branch` instead of creating
+                a new branch.
+            allow_unchanged: Force experiment reproduction even if params are
+                unchanged from the baseline.
+            apply_workspace: Apply changes from the user workspace to the
+                experiment workspace.
         """
-        rev = self.scm.get_rev()
+        if branch:
+            rev = self.scm.resolve_rev(branch)
+        else:
+            rev = self.scm.get_rev()
 
-        # patch user's workspace into experiments clone
-        tmp = tempfile.NamedTemporaryFile(delete=False).name
-        try:
-            self.repo.scm.repo.git.diff(
-                patch=True, full_index=True, binary=True, output=tmp
-            )
-            if os.path.getsize(tmp):
-                logger.debug("Patching experiment workspace")
-                self.scm.repo.git.apply(tmp)
-            elif not params:
-                # experiment matches original baseline
-                raise UnchangedExperimentError(rev)
-        finally:
-            remove(tmp)
+        if apply_workspace:
+            if branch:
+                raise DvcException(
+                    "Cannot apply workspace changes into existing experiment"
+                    "branch."
+                )
+            # patch user's workspace into experiments clone
+            tmp = tempfile.NamedTemporaryFile(delete=False).name
+            try:
+                self.repo.scm.repo.git.diff(
+                    patch=True, full_index=True, binary=True, output=tmp
+                )
+                if os.path.getsize(tmp):
+                    logger.debug("Patching experiment workspace")
+                    self.scm.repo.git.apply(tmp)
+            finally:
+                remove(tmp)
 
         # update experiment params from command line
         if params:
             self._update_params(params)
 
+        if not self._check_dirty() and not allow_unchanged:
+            # experiment matches original baseline
+            raise UnchangedExperimentError(rev)
+
         # save additional repro command line arguments
         self._pack_args(*args, **kwargs)
 
         # save experiment as a stash commit w/message containing baseline rev
         # (stash commits are merge commits and do not contain a parent commit
         # SHA)
-        msg = f"{self.STASH_MSG_PREFIX}{rev}"
+        msg = self._stash_msg(rev, baseline_rev=baseline_rev, branch=branch)
         self.scm.repo.git.stash("push", "-m", msg)
         return self.scm.resolve_rev("stash@{0}")
 
+    def _check_dirty(self) -> bool:
+        # NOTE: dirty DVC lock files must be restored to index state to
+        # avoid checking out incorrect persist or checkpoint outs
+        dirty = [diff.a_path for diff in self.scm.repo.index.diff(None)]
+        to_checkout = [fname for fname in dirty if is_lock_file(fname)]
+        self.scm.repo.index.checkout(paths=to_checkout, force=True)
+
+        untracked = self.scm.repo.untracked_files
+        to_remove = [fname for fname in untracked if is_lock_file(fname)]
+        for fname in to_remove:
+            remove(fname)
+        return (
+            len(dirty) - len(to_checkout) + len(untracked) - len(to_remove)
+        ) != 0
+
+    def _stash_msg(self, rev, baseline_rev=None, branch=None):
+        if not baseline_rev:
+            baseline_rev = rev
+        msg = self.STASH_EXPERIMENT_FORMAT.format(
+            rev=rev, baseline_rev=baseline_rev
+        )
+        if branch:
+            return f"{msg}:{branch}"
+        return msg
+
     def _pack_args(self, *args, **kwargs):
         ExperimentExecutor.pack_repro_args(self.args_file, *args, **kwargs)
         self.scm.add(self.args_file)
@@ -232,43 +343,70 @@ class Experiments:
 
     def _update_params(self, params: dict):
         """Update experiment params files with the specified values."""
+        from benedict import benedict
+
         from dvc.utils.serialize import MODIFIERS
 
         logger.debug("Using experiment params '%s'", params)
 
-        # recursive dict update
-        def _update(dict_, other):
-            for key, value in other.items():
-                if isinstance(value, Mapping):
-                    dict_[key] = _update(dict_.get(key, {}), value)
-                else:
-                    dict_[key] = value
-            return dict_
-
         for params_fname in params:
             path = PathInfo(self.exp_dvc.root_dir) / params_fname
             suffix = path.suffix.lower()
             modify_data = MODIFIERS[suffix]
             with modify_data(path, tree=self.exp_dvc.tree) as data:
-                _update(data, params[params_fname])
+                benedict(data).merge(params[params_fname], overwrite=True)
 
-    def _commit(self, exp_hash, check_exists=True, branch=True):
+        # Force params file changes to be staged in git
+        # Otherwise in certain situations the changes to params file may be
+        # ignored when we `git stash` them since mtime is used to determine
+        # whether the file is dirty
+        self.scm.add(list(params.keys()))
+
+    def _commit(
+        self,
+        exp_hash: str,
+        baseline_rev: Optional[str] = None,
+        check_exists: Optional[bool] = True,
+        create_branch: Optional[bool] = True,
+        checkpoint: Optional[bool] = False,
+        checkpoint_reset: Optional[bool] = False,
+    ):
         """Commit stages as an experiment and return the commit SHA."""
-        if not self.scm.is_dirty():
+        if not self.scm.is_dirty(untracked_files=True):
             raise UnchangedExperimentError(self.scm.get_rev())
 
-        rev = self.scm.get_rev()
-        exp_name = f"{rev[:7]}-{exp_hash}"
-        if branch:
-            if check_exists and exp_name in self.scm.list_branches():
-                logger.debug("Using existing experiment branch '%s'", exp_name)
-                return self.scm.resolve_rev(exp_name)
+        rev = baseline_rev or self.scm.get_rev()
+        suffix = "-checkpoint" if checkpoint else ""
+        exp_name = f"{rev[:7]}-{exp_hash}{suffix}"
+        if create_branch:
+            if (
+                check_exists or checkpoint
+            ) and exp_name in self.scm.list_branches():
+                branch_tip = self.scm.resolve_rev(exp_name)
+                if checkpoint:
+                    self._reset_checkpoint_branch(
+                        exp_name, rev, branch_tip, checkpoint_reset
+                    )
+                else:
+                    logger.debug(
+                        "Using existing experiment branch '%s'", exp_name
+                    )
+                    return branch_tip
             self.scm.checkout(exp_name, create_new=True)
-        logger.debug("Commit new experiment branch '%s'", exp_name)
+            logger.debug("Commit new experiment branch '%s'", exp_name)
+        else:
+            logger.debug("Commit to current experiment branch")
         self.scm.repo.git.add(A=True)
         self.scm.commit(f"Add experiment {exp_name}")
         return self.scm.get_rev()
 
+    def _reset_checkpoint_branch(self, branch, rev, branch_tip, reset):
+        if not reset:
+            raise CheckpointExistsError(rev, branch_tip)
+        self._checkout_default_branch()
+        logger.debug("Removing existing checkpoint branch '%s'", branch)
+        self.scm.repo.git.branch(branch, D=True)
+
     def reproduce_one(self, queue=False, **kwargs):
         """Reproduce and checkout a single experiment."""
         stash_rev = self.new(**kwargs)
@@ -296,16 +434,37 @@ class Experiments:
         return results
 
     @scm_locked
-    def new(self, *args, **kwargs):
+    def new(
+        self,
+        *args,
+        branch: Optional[str] = None,
+        checkpoint_resume: Optional[str] = None,
+        **kwargs,
+    ):
         """Create a new experiment.
 
         Experiment will be reproduced and checked out into the user's
         workspace.
         """
-        rev = self.repo.scm.get_rev()
+        if checkpoint_resume is not None:
+            return self._resume_checkpoint(
+                *args, checkpoint_resume=checkpoint_resume, **kwargs
+            )
+
+        if branch:
+            rev = self.scm.resolve_rev(branch)
+            logger.debug(
+                "Using '%s' (tip of branch '%s') as baseline", rev, branch
+            )
+        else:
+            rev = self.repo.scm.get_rev()
         self._scm_checkout(rev)
+
+        force = kwargs.get("force", False)
         try:
-            stash_rev = self._stash_exp(*args, **kwargs)
+            stash_rev = self._stash_exp(
+                *args, branch=branch, checkpoint_reset=force, **kwargs,
+            )
         except UnchangedExperimentError as exc:
             logger.info("Reproducing existing experiment '%s'.", rev[:7])
             raise exc
@@ -314,6 +473,70 @@ class Experiments:
         )
         return stash_rev
 
+    def _resume_checkpoint(
+        self, *args, checkpoint_resume: Optional[str] = None, **kwargs,
+    ):
+        """Resume an existing (checkpoint) experiment.
+
+        Experiment will be reproduced and checked out into the user's
+        workspace.
+        """
+        assert checkpoint_resume
+
+        branch = None
+        if checkpoint_resume == self.LAST_CHECKPOINT:
+            # Continue from most recently committed checkpoint
+            branch = self._get_last_checkpoint()
+            resume_rev = self.scm.resolve_rev(branch)
+        else:
+            rev = self.scm.resolve_rev(checkpoint_resume)
+            resume_rev = rev
+            branch = self._get_branch_containing(rev)
+            if not branch:
+                raise DvcException(
+                    "Could not find checkpoint experiment "
+                    f"'{checkpoint_resume}'"
+                )
+
+        baseline_rev = self._get_baseline(branch)
+        if kwargs.get("params", None):
+            logger.debug(
+                "Branching from checkpoint '%s' with modified params",
+                checkpoint_resume,
+            )
+            rev = resume_rev
+            branch = None
+        else:
+            logger.debug(
+                "Continuing checkpoint experiment '%s'", checkpoint_resume
+            )
+            rev = self.scm.resolve_rev(branch)
+            logger.debug(
+                "Using '%s' (tip of branch '%s') as baseline", rev, branch
+            )
+        self._scm_checkout(rev)
+
+        kwargs["apply_workspace"] = False
+        stash_rev = self._stash_exp(
+            *args, baseline_rev=baseline_rev, branch=branch, **kwargs
+        )
+        logger.debug(
+            "Stashed experiment '%s' for future execution.", stash_rev[:7]
+        )
+        return stash_rev
+
+    def _get_last_checkpoint(self):
+        for head in sorted(
+            self.scm.repo.heads,
+            key=lambda h: h.commit.committed_date,
+            reverse=True,
+        ):
+            exp_branch = head.name
+            m = self.BRANCH_RE.match(exp_branch)
+            if m and m.group("checkpoint"):
+                return exp_branch
+        raise DvcException("No existing checkpoint experiment to continue")
+
     @scm_locked
     def reproduce(
         self,
@@ -332,18 +555,17 @@ class Experiments:
         stash_revs = self.stash_revs
 
         # to_run contains mapping of:
-        #   input_rev: (stash_index, baseline_rev)
+        #   input_rev: (stash_index, rev, baseline_rev)
         # where input_rev contains the changes to execute (usually a stash
-        # commit) and baseline_rev is the baseline to compare output against.
-        # The final experiment commit will be branched from baseline_rev.
+        # commit), rev is the original SCM commit to be checked out, and
+        # baseline_rev is the experiment baseline.
         if revs is None:
-            to_run = {
-                rev: baseline_rev
-                for rev, (_, baseline_rev) in stash_revs.items()
-            }
+            to_run = dict(stash_revs)
         else:
             to_run = {
-                rev: stash_revs[rev][1] if rev in stash_revs else rev
+                rev: stash_revs[rev]
+                if rev in stash_revs
+                else self.StashEntry(None, rev, rev, None)
                 for rev in revs
             }
 
@@ -355,16 +577,20 @@ class Experiments:
         # setup executors - unstash experiment, generate executor, upload
         # contents of (unstashed) exp workspace to the executor tree
         executors = {}
-        for rev, baseline_rev in to_run.items():
-            self._scm_checkout(baseline_rev)
+        for rev, item in to_run.items():
+            self._scm_checkout(item.rev)
             self.scm.repo.git.stash("apply", rev)
-            repro_args, repro_kwargs = self._unpack_args()
+            packed_args, packed_kwargs = self._unpack_args()
+            checkpoint_reset = packed_kwargs.pop("checkpoint_reset", False)
             executor = LocalExecutor(
-                baseline_rev,
-                repro_args=repro_args,
-                repro_kwargs=repro_kwargs,
+                rev=item.rev,
+                baseline_rev=item.baseline_rev,
+                branch=item.branch,
+                repro_args=packed_args,
+                repro_kwargs=packed_kwargs,
                 dvc_dir=self.dvc_dir,
                 cache_dir=self.repo.cache.local.cache_dir,
+                checkpoint_reset=checkpoint_reset,
             )
             self._collect_input(executor)
             executors[rev] = executor
@@ -400,55 +626,139 @@ class Experiments:
 
         Returns dict containing successfully executed experiments.
         """
-        result = {}
+        result: dict = {}
 
-        with ProcessPoolExecutor(max_workers=jobs) as workers:
+        collect_lock = threading.Lock()
+
+        with ThreadPoolExecutor(max_workers=jobs) as workers:
             futures = {}
             for rev, executor in executors.items():
+                checkpoint_func = partial(
+                    self._checkpoint_callback,
+                    result,
+                    collect_lock,
+                    rev,
+                    executor,
+                )
                 future = workers.submit(
                     executor.reproduce,
                     executor.dvc_dir,
                     cwd=executor.dvc.root_dir,
+                    checkpoint_func=checkpoint_func,
                     **executor.repro_kwargs,
                 )
                 futures[future] = (rev, executor)
+
             for future in as_completed(futures):
                 rev, executor = futures[future]
                 exc = future.exception()
-                if exc is None:
-                    exp_hash = future.result()
-                    self._scm_checkout(executor.baseline_rev)
-                    try:
-                        self._collect_output(executor)
-                    except DownloadError:
-                        logger.error(
-                            "Failed to collect output for experiment '%s'",
-                            rev,
-                        )
-                        continue
-                    finally:
-                        if os.path.exists(self.args_file):
-                            remove(self.args_file)
 
-                    try:
-                        exp_rev = self._commit(exp_hash)
-                    except UnchangedExperimentError:
-                        logger.debug(
-                            "Experiment '%s' identical to baseline '%s'",
-                            rev,
-                            executor.baseline_rev,
+                try:
+                    if exc is None:
+                        stages = future.result()
+                        self._collect_executor(
+                            rev, executor, stages, result, collect_lock
                         )
-                        exp_rev = executor.baseline_rev
-                    logger.info("Reproduced experiment '%s'.", exp_rev[:7])
-                    result[rev] = {exp_rev: exp_hash}
-                else:
-                    logger.exception(
-                        "Failed to reproduce experiment '%s'", rev
-                    )
-                executor.cleanup()
+                    else:
+                        # Checkpoint errors have already been logged
+                        if not isinstance(exc, CheckpointKilledError):
+                            logger.exception(
+                                "Failed to reproduce experiment '%s'",
+                                rev[:7],
+                                exc_info=exc,
+                            )
+                finally:
+                    executor.cleanup()
 
         return result
 
+    def _collect_executor(self, rev, executor, stages, result, lock):
+        exp_hash = hash_exp(stages)
+        checkpoint = any(stage.is_checkpoint for stage in stages)
+
+        lock.acquire()
+        try:
+            # NOTE: GitPython Repo instances cannot be re-used
+            # after process has received SIGINT or SIGTERM, so we
+            # need this hack to re-instantiate git instances after
+            # checkpoint runs. See:
+            # https://github.com/gitpython-developers/GitPython/issues/427
+            del self.repo.scm
+            del self.scm
+
+            if executor.branch:
+                self._scm_checkout(executor.branch)
+            else:
+                self._scm_checkout(executor.rev)
+            exp_rev = self._collect_and_commit(
+                rev, executor, exp_hash, checkpoint=checkpoint
+            )
+            if exp_rev:
+                logger.info("Reproduced experiment '%s'.", exp_rev[:7])
+                result[rev] = {exp_rev: exp_hash}
+        finally:
+            lock.release()
+
+    @no_type_check
+    def _checkpoint_callback(
+        self,
+        result: Mapping,
+        lock: threading.Lock,
+        rev: str,
+        executor: LocalExecutor,
+        unchanged: Iterable,
+        stages: Iterable,
+    ):
+        lock.acquire()
+        try:
+            if executor.branch:
+                self._scm_checkout(executor.branch)
+            else:
+                self._scm_checkout(executor.rev)
+
+            exp_hash = hash_exp(stages + unchanged)
+            exp_rev = self._collect_and_commit(
+                rev, executor, exp_hash, checkpoint=True
+            )
+            if exp_rev:
+                if not executor.branch:
+                    branch = self._get_branch_containing(exp_rev)
+                    executor.branch = branch
+                logger.info(
+                    "Checkpoint experiment iteration '%s'.", exp_rev[:7]
+                )
+                result[rev] = {exp_rev: exp_hash}
+        finally:
+            lock.release()
+
+    def _collect_and_commit(self, rev, executor, exp_hash, **kwargs):
+        try:
+            self._collect_output(executor)
+        except DownloadError:
+            logger.error(
+                "Failed to collect output for experiment '%s'", rev,
+            )
+            return None
+        finally:
+            if os.path.exists(self.args_file):
+                remove(self.args_file)
+
+        try:
+            create_branch = not executor.branch
+            exp_rev = self._commit(
+                exp_hash,
+                baseline_rev=executor.baseline_rev,
+                create_branch=create_branch,
+                checkpoint_reset=executor.checkpoint_reset,
+                **kwargs,
+            )
+        except UnchangedExperimentError as exc:
+            logger.debug(
+                "Experiment '%s' identical to '%s'", rev, exc.rev,
+            )
+            exp_rev = exc.rev
+        return exp_rev
+
     def _collect_input(self, executor: ExperimentExecutor):
         """Copy (upload) input from the experiments workspace to the executor
         tree.
@@ -475,7 +785,7 @@ class Experiments:
 
     @staticmethod
     def _process(dest_tree, src_tree, collected_files, download=False):
-        from dvc.cache.local import _log_exceptions
+        from dvc.remote.base import _log_exceptions
 
         from_infos = []
         to_infos = []
@@ -491,7 +801,6 @@ class Experiments:
             func = partial(
                 _log_exceptions(src_tree.download, "download"),
                 dir_mode=dest_tree.dir_mode,
-                file_mode=dest_tree.file_mode,
             )
             desc = "Downloading"
         else:
@@ -503,7 +812,23 @@ class Experiments:
             # TODO: parallelize this, currently --jobs for repro applies to
             # number of repro executors not download threads
             with ThreadPoolExecutor(max_workers=1) as dl_executor:
-                fails = sum(dl_executor.map(func, from_infos, to_infos, names))
+                mode = None
+                stat_func = getattr(src_tree, "stat", None)
+                futures = []
+                for from_info, to_info, name in zip(
+                    from_infos, to_infos, names
+                ):
+                    if stat_func:
+                        mode = stat.S_IMODE(stat_func(from_info).st_mode)
+                    futures.append(
+                        dl_executor.submit(
+                            func, from_info, to_info, name, file_mode=mode
+                        )
+                    )
+
+                fails = sum(
+                    future.result() for future in as_completed(futures)
+                )
 
         if fails:
             if download:
@@ -511,21 +836,26 @@ class Experiments:
             raise UploadError(fails)
 
     @scm_locked
-    def checkout_exp(self, rev):
+    def checkout_exp(self, rev, **kwargs):
         """Checkout an experiment to the user's workspace."""
         from git.exc import GitCommandError
 
         from dvc.repo.checkout import checkout as dvc_checkout
 
-        self._check_baseline(rev)
+        baseline_rev = self._check_baseline(rev)
         self._scm_checkout(rev)
 
+        branch = self._get_branch_containing(rev)
+        m = self.BRANCH_RE.match(branch) if branch else None
+        if m and m.group("checkpoint"):
+            kwargs.update({"allow_missing": True, "quiet": True})
+
         tmp = tempfile.NamedTemporaryFile(delete=False).name
         self.scm.repo.head.commit.diff(
-            "HEAD~1", patch=True, full_index=True, binary=True, output=tmp
+            baseline_rev, patch=True, full_index=True, binary=True, output=tmp
         )
 
-        dirty = self.repo.scm.is_dirty()
+        dirty = self.repo.scm.is_dirty(untracked_files=True)
         if dirty:
             logger.debug("Stashing workspace changes.")
             self.repo.scm.repo.git.stash("push", "--include-untracked")
@@ -543,17 +873,26 @@ class Experiments:
             remove(tmp)
             if dirty:
                 self._unstash_workspace()
+            args_file = os.path.join(self.repo.tmp_dir, self.PACKED_ARGS_FILE)
+            if os.path.exists(args_file):
+                remove(args_file)
 
         if need_checkout:
-            dvc_checkout(self.repo)
+            dvc_checkout(self.repo, **kwargs)
 
     def _check_baseline(self, exp_rev):
         baseline_sha = self.repo.scm.get_rev()
-        exp_commit = self.scm.repo.rev_parse(exp_rev)
-        parent = first(exp_commit.parents)
-        if parent is not None and parent.hexsha == baseline_sha:
-            return
-        raise BaselineMismatchError(parent, baseline_sha)
+        if exp_rev == baseline_sha:
+            return exp_rev
+
+        exp_baseline = self._get_baseline(exp_rev)
+        if exp_baseline is None:
+            # if we can't tell from branch name, fall back to parent commit
+            exp_commit = self.scm.repo.rev_parse(exp_rev)
+            exp_baseline = first(exp_commit.parents).hexsha
+        if exp_baseline == baseline_sha:
+            return exp_baseline
+        raise BaselineMismatchError(exp_baseline, baseline_sha)
 
     def _unstash_workspace(self):
         # Essentially we want `git stash pop` with `-X ours` merge strategy
@@ -581,6 +920,9 @@ class Experiments:
     @scm_locked
     def get_baseline(self, rev):
         """Return the baseline rev for an experiment rev."""
+        return self._get_baseline(rev)
+
+    def _get_baseline(self, rev):
         from git.exc import GitCommandError
 
         rev = self.scm.resolve_rev(rev)
@@ -588,16 +930,43 @@ class Experiments:
             name = self.scm.repo.git.name_rev(rev, name_only=True)
         except GitCommandError:
             return None
-        if not name:
-            return None
         if name in ("undefined", "stash"):
-            _, baseline = self.stash_revs.get(rev, (None, None))
-            return baseline
+            entry = self.stash_revs.get(rev)
+            if entry:
+                return entry.baseline_rev
+            return None
         m = self.BRANCH_RE.match(name)
         if m:
             return self.scm.resolve_rev(m.group("baseline_rev"))
         return None
 
+    def _get_branch_containing(self, rev):
+        from git.exc import GitCommandError
+
+        try:
+            names = self.scm.repo.git.branch(contains=rev).strip().splitlines()
+
+            if (
+                names
+                and self.scm.repo.head.is_detached
+                and names[0].startswith("* (HEAD detached")
+            ):
+                # Ignore detached head entry if it exists
+                del names[0]
+
+            if not names:
+                return None
+
+            if len(names) > 1:
+                raise MultipleBranchError(rev)
+            name = names[0]
+            if name.startswith("*"):
+                name = name[1:]
+            return name.rsplit("/")[-1].strip()
+        except GitCommandError:
+            pass
+        return None
+
     def checkout(self, *args, **kwargs):
         from dvc.repo.experiments.checkout import checkout
 
@@ -612,3 +981,13 @@ class Experiments:
         from dvc.repo.experiments.show import show
 
         return show(self.repo, *args, **kwargs)
+
+    def run(self, *args, **kwargs):
+        from dvc.repo.experiments.run import run
+
+        return run(self.repo, *args, **kwargs)
+
+    def gc(self, *args, **kwargs):
+        from dvc.repo.experiments.gc import gc
+
+        return gc(self.repo, *args, **kwargs)

dvc/repo/experiments/executor.py

@@ -2,7 +2,7 @@ import logging
 import os
 import pickle
 from tempfile import TemporaryDirectory
-from typing import Iterable
+from typing import Iterable, Optional
 
 from funcy import cached_property
 
@@ -19,17 +19,29 @@ class ExperimentExecutor:
     """Base class for executing experiments in parallel.
 
     Args:
-        baseline_rev: baseline revision that this experiment is derived from.
+        baseline_rev: Baseline revision that this experiment is derived from.
 
     Optional keyword args:
+        branch: Existing git branch for this experiment.
+        rev: Git revision to be checked out for this experiment, defaults to
+            branch, baseline_rev in that order.
         repro_args: Args to be passed into reproduce.
         repro_kwargs: Keyword args to be passed into reproduce.
     """
 
-    def __init__(self, baseline_rev: str, **kwargs):
+    def __init__(
+        self,
+        baseline_rev: str,
+        branch: Optional[str] = None,
+        rev: Optional[str] = None,
+        **kwargs,
+    ):
         self.baseline_rev = baseline_rev
+        self._rev = rev
+        self.branch = branch
         self.repro_args = kwargs.pop("repro_args", [])
         self.repro_kwargs = kwargs.pop("repro_kwargs", {})
+        self.tmp_dir: Optional[TemporaryDirectory] = None
 
     @property
     def tree(self) -> BaseTree:
@@ -55,7 +67,15 @@ class ExperimentExecutor:
     # TODO: come up with better way to stash repro arguments
     @staticmethod
     def pack_repro_args(path, *args, tree=None, **kwargs):
-        open_func = tree.open if tree else open
+        dpath = os.path.dirname(path)
+        if tree:
+            open_func = tree.open
+            tree.makedirs(dpath)
+        else:
+            from dvc.utils.fs import makedirs
+
+            open_func = open
+            makedirs(dpath, exist_ok=True)
         data = {"args": args, "kwargs": kwargs}
         with open_func(path, "wb") as fobj:
             pickle.dump(data, fobj)
@@ -67,16 +87,22 @@ class ExperimentExecutor:
             data = pickle.load(fobj)
         return data["args"], data["kwargs"]
 
+    @staticmethod
+    def collect_files(tree: BaseTree, repo_tree: RepoTree):
+        raise NotImplementedError
+
 
 class LocalExecutor(ExperimentExecutor):
     """Local machine experiment executor."""
 
-    def __init__(self, baseline_rev: str, **kwargs):
+    def __init__(
+        self, checkpoint_reset: Optional[bool] = False, **kwargs,
+    ):
         from dvc.repo import Repo
 
         dvc_dir = kwargs.pop("dvc_dir")
         cache_dir = kwargs.pop("cache_dir")
-        super().__init__(baseline_rev, **kwargs)
+        super().__init__(**kwargs)
         self.tmp_dir = TemporaryDirectory()
 
         # init empty DVC repo (will be overwritten when input is uploaded)
@@ -84,7 +110,7 @@ class LocalExecutor(ExperimentExecutor):
         logger.debug(
             "Init local executor in dir '%s' with baseline '%s'.",
             self.tmp_dir,
-            baseline_rev[:7],
+            self.baseline_rev[:7],
         )
         self.dvc_dir = os.path.join(self.tmp_dir.name, dvc_dir)
         self._config(cache_dir)
@@ -92,6 +118,8 @@ class LocalExecutor(ExperimentExecutor):
         # override default CACHE_MODE since files must be writable in order
         # to run repro
         self._tree.CACHE_MODE = 0o644
+        self.checkpoint_reset = checkpoint_reset
+        self.checkpoint = False
 
     def _config(self, cache_dir):
         local_config = os.path.join(self.dvc_dir, "config.local")
@@ -114,17 +142,21 @@ class LocalExecutor(ExperimentExecutor):
     def tree(self):
         return self._tree
 
+    @property
+    def rev(self):
+        return self._rev if self._rev else self.baseline_rev
+
     @staticmethod
     def reproduce(dvc_dir, cwd=None, **kwargs):
         """Run dvc repro and return the result."""
         from dvc.repo import Repo
-        from dvc.repo.experiments import hash_exp
 
         unchanged = []
 
-        def filter_pipeline(stage):
-            if isinstance(stage, PipelineStage):
-                unchanged.append(stage)
+        def filter_pipeline(stages):
+            unchanged.extend(
+                [stage for stage in stages if isinstance(stage, PipelineStage)]
+            )
 
         if cwd:
             old_cwd = os.getcwd()
@@ -136,7 +168,19 @@ class LocalExecutor(ExperimentExecutor):
         try:
             logger.debug("Running repro in '%s'", cwd)
             dvc = Repo(dvc_dir)
-            dvc.checkout()
+
+            # NOTE: for checkpoint experiments we handle persist outs slightly
+            # differently than normal:
+            #
+            # - checkpoint out may not yet exist if this is the first time this
+            #   experiment has been run, this is not an error condition for
+            #   experiments
+            # - at the start of a repro run, we need to remove the persist out
+            #   and restore it to its last known (committed) state (which may
+            #   be removed/does not yet exist) so that our executor workspace
+            #   is not polluted with the (persistent) out from an unrelated
+            #   experiment run
+            dvc.checkout(force=True, quiet=True)
             stages = dvc.reproduce(on_unchanged=filter_pipeline, **kwargs)
         finally:
             if old_cwd is not None:
@@ -145,7 +189,7 @@ class LocalExecutor(ExperimentExecutor):
         # ideally we would return stages here like a normal repro() call, but
         # stages is not currently picklable and cannot be returned across
         # multiprocessing calls
-        return hash_exp(stages + unchanged)
+        return stages + unchanged
 
     def collect_output(self) -> Iterable["PathInfo"]:
         repo_tree = RepoTree(self.dvc)

dvc/repo/experiments/gc.py

@@ -0,0 +1,51 @@
+import logging
+from typing import Optional
+
+from dvc.repo import locked
+
+logger = logging.getLogger(__name__)
+
+
+@locked
+def gc(
+    repo,
+    all_branches: Optional[bool] = False,
+    all_tags: Optional[bool] = False,
+    all_commits: Optional[bool] = False,
+    workspace: Optional[bool] = False,
+    queued: Optional[bool] = False,
+):
+    keep_revs = set(
+        repo.brancher(
+            all_branches=all_branches,
+            all_tags=all_tags,
+            all_commits=all_commits,
+            sha_only=True,
+        )
+    )
+    if workspace:
+        keep_revs.add(repo.scm.get_rev())
+
+    if not keep_revs:
+        return 0
+
+    delete_branches = []
+    for exp_branch in repo.experiments.scm.list_branches():
+        m = repo.experiments.BRANCH_RE.match(exp_branch)
+        if m:
+            rev = repo.scm.resolve_rev(m.group("baseline_rev"))
+            if rev not in keep_revs:
+                delete_branches.append(exp_branch)
+    if delete_branches:
+        repo.experiments.scm.repo.delete_head(*delete_branches, force=True)
+    removed = len(delete_branches)
+
+    delete_stashes = []
+    for _, entry in repo.experiments.stash_revs.items():
+        if not queued or entry.baseline_rev not in keep_revs:
+            delete_stashes.append(entry.index)
+    for index in sorted(delete_stashes, reverse=True):
+        repo.experiments.scm.repo.git.stash("drop", index)
+    removed += len(delete_stashes)
+
+    return removed

dvc/repo/experiments/run.py

@@ -0,0 +1,68 @@
+import logging
+from typing import Iterable, Optional
+
+from dvc.exceptions import InvalidArgumentError
+from dvc.repo import locked
+from dvc.repo.experiments import UnchangedExperimentError
+
+logger = logging.getLogger(__name__)
+
+
+def _parse_params(path_params: Iterable):
+    from ruamel.yaml import YAMLError
+
+    from dvc.dependency.param import ParamsDependency
+    from dvc.utils.serialize import loads_yaml
+
+    ret = {}
+    for path_param in path_params:
+        path, _, params_str = path_param.rpartition(":")
+        # remove empty strings from params, on condition such as `-p "file1:"`
+        params = {}
+        for param_str in filter(bool, params_str.split(",")):
+            try:
+                # interpret value strings using YAML rules
+                key, value = param_str.split("=")
+                params[key] = loads_yaml(value)
+            except (ValueError, YAMLError):
+                raise InvalidArgumentError(
+                    f"Invalid param/value pair '{param_str}'"
+                )
+        if not path:
+            path = ParamsDependency.DEFAULT_PARAMS_FILE
+        ret[path] = params
+    return ret
+
+
+@locked
+def run(
+    repo,
+    target: Optional[str] = None,
+    params: Optional[Iterable] = None,
+    run_all: Optional[bool] = False,
+    jobs: Optional[int] = 1,
+    **kwargs,
+) -> dict:
+    """Reproduce the specified target as an experiment.
+
+    Accepts the same additional kwargs as Repo.reproduce.
+
+    Returns a dict mapping new experiment SHAs to the results
+    of `repro` for that experiment.
+    """
+    if run_all:
+        return repo.experiments.reproduce_queued(jobs=jobs)
+
+    if params:
+        params = _parse_params(params)
+    else:
+        params = []
+    try:
+        return repo.experiments.reproduce_one(
+            target=target, params=params, **kwargs
+        )
+    except UnchangedExperimentError:
+        # If experiment contains no changes, just run regular repro
+        kwargs.pop("queue", None)
+        kwargs.pop("checkpoint_resume", None)
+        return {None: repo.reproduce(target=target, **kwargs)}

dvc/repo/experiments/show.py

@@ -9,25 +9,25 @@ from dvc.repo.params.show import _collect_configs, _read_params
 logger = logging.getLogger(__name__)
 
 
-def _collect_experiment(repo, branch, stash=False, sha_only=True):
+def _collect_experiment(repo, rev, stash=False, sha_only=True):
     from git.exc import GitCommandError
 
     res = defaultdict(dict)
-    for rev in repo.brancher(revs=[branch]):
+    for rev in repo.brancher(revs=[rev]):
         if rev == "workspace":
             res["timestamp"] = None
         else:
-            commit = repo.scm.repo.rev_parse(rev)
+            commit = repo.scm.resolve_commit(rev)
             res["timestamp"] = datetime.fromtimestamp(commit.committed_date)
 
-        configs = _collect_configs(repo)
+        configs = _collect_configs(repo, rev=rev)
         params = _read_params(repo, configs, rev)
         if params:
             res["params"] = params
 
         res["queued"] = stash
         if not stash:
-            metrics = _collect_metrics(repo, None, False)
+            metrics = _collect_metrics(repo, None, rev, False)
             vals = _read_metrics(repo, metrics, rev)
             res["metrics"] = vals
 
@@ -44,6 +44,24 @@ def _collect_experiment(repo, branch, stash=False, sha_only=True):
     return res
 
 
+def _collect_checkpoint_experiment(res, repo, branch, baseline, **kwargs):
+    exp_rev = repo.scm.resolve_rev(branch)
+    prev = None
+    for rev in repo.scm.branch_revs(exp_rev, baseline):
+        exp = {"checkpoint_tip": exp_rev}
+        if prev:
+            res[prev]["checkpoint_parent"] = rev
+        if rev in res:
+            res[rev].update(exp)
+            res.move_to_end(rev)
+        else:
+            exp.update(_collect_experiment(repo, rev, **kwargs))
+            res[rev] = exp
+        prev = rev
+    res[prev]["checkpoint_parent"] = baseline
+    return res
+
+
 @locked
 def show(
     repo,
@@ -65,6 +83,7 @@ def show(
             all_branches=all_branches,
             all_tags=all_tags,
             all_commits=all_commits,
+            sha_only=True,
         )
     )
 
@@ -74,25 +93,35 @@ def show(
         )
 
     # collect reproduced experiments
-    for exp_branch in repo.experiments.scm.list_branches():
+    for head in sorted(
+        repo.experiments.scm.repo.heads,
+        key=lambda h: h.commit.committed_date,
+        reverse=True,
+    ):
+        exp_branch = head.name
         m = repo.experiments.BRANCH_RE.match(exp_branch)
         if m:
             rev = repo.scm.resolve_rev(m.group("baseline_rev"))
             if rev in revs:
-                exp_rev = repo.experiments.scm.resolve_rev(exp_branch)
                 with repo.experiments.chdir():
-                    experiment = _collect_experiment(
-                        repo.experiments.exp_dvc, exp_branch
-                    )
-                res[rev][exp_rev] = experiment
+                    if m.group("checkpoint"):
+                        _collect_checkpoint_experiment(
+                            res[rev], repo.experiments.exp_dvc, exp_branch, rev
+                        )
+                    else:
+                        exp_rev = repo.experiments.scm.resolve_rev(exp_branch)
+                        experiment = _collect_experiment(
+                            repo.experiments.exp_dvc, exp_branch
+                        )
+                        res[rev][exp_rev] = experiment
 
     # collect queued (not yet reproduced) experiments
-    for stash_rev, (_, baseline_rev) in repo.experiments.stash_revs.items():
-        if baseline_rev in revs:
+    for stash_rev, entry in repo.experiments.stash_revs.items():
+        if entry.baseline_rev in revs:
             with repo.experiments.chdir():
                 experiment = _collect_experiment(
                     repo.experiments.exp_dvc, stash_rev, stash=True
                 )
-            res[baseline_rev][stash_rev] = experiment
+            res[entry.baseline_rev][stash_rev] = experiment
 
     return res

dvc/repo/fetch.py

@@ -36,8 +36,6 @@ def fetch(
             remote is configured
     """
 
-    used_run_cache = self.stage_cache.pull(remote) if run_cache else []
-
     if isinstance(targets, str):
         targets = [targets]
 
@@ -51,13 +49,14 @@ def fetch(
         remote=remote,
         jobs=jobs,
         recursive=recursive,
-        used_run_cache=used_run_cache,
     )
 
     downloaded = 0
     failed = 0
 
     try:
+        if run_cache:
+            self.stage_cache.pull(remote)
         downloaded += self.cloud.pull(
             used, jobs, remote=remote, show_checksums=show_checksums,
         )
@@ -75,7 +74,7 @@ def fetch(
     if failed:
         raise DownloadError(failed)
 
-    return downloaded + len(used_run_cache)
+    return downloaded
 
 
 def _fetch_external(self, repo_url, repo_rev, files, jobs):

dvc/repo/gc.py

@@ -3,19 +3,12 @@ import logging
 from dvc.cache import NamedCache
 from dvc.exceptions import InvalidArgumentError
 
+from ..scheme import Schemes
 from . import locked
 
 logger = logging.getLogger(__name__)
 
 
-def _do_gc(typ, remote, clist, jobs=None):
-    from dvc.remote.base import Remote
-
-    removed = Remote.gc(clist, remote, jobs=jobs)
-    if not removed:
-        logger.info(f"No unused '{typ}' cache to remove.")
-
-
 def _raise_error_if_all_disabled(**kwargs):
     if not any(kwargs.values()):
         raise InvalidArgumentError(
@@ -76,23 +69,18 @@ def gc(
                 )
             )
 
-    # treat caches as remotes for garbage collection
-    _do_gc("local", self.cache.local, used, jobs)
+    for scheme, cache in self.cache.by_scheme():
+        if not cache:
+            continue
 
-    if self.cache.s3:
-        _do_gc("s3", self.cache.s3, used, jobs)
+        removed = cache.gc(set(used.scheme_keys(scheme)), jobs=jobs)
+        if not removed:
+            logger.info(f"No unused '{scheme}' cache to remove.")
 
-    if self.cache.gs:
-        _do_gc("gs", self.cache.gs, used, jobs)
+    if not cloud:
+        return
 
-    if self.cache.ssh:
-        _do_gc("ssh", self.cache.ssh, used, jobs)
-
-    if self.cache.hdfs:
-        _do_gc("hdfs", self.cache.hdfs, used, jobs)
-
-    if self.cache.azure:
-        _do_gc("azure", self.cache.azure, used, jobs)
-
-    if cloud:
-        _do_gc("remote", self.cloud.get_remote(remote, "gc -c"), used, jobs)
+    remote = self.cloud.get_remote(remote, "gc -c")
+    removed = remote.gc(set(used.scheme_keys(Schemes.LOCAL)), jobs=jobs)
+    if not removed:
+        logger.info("No unused cache to remove from remote.")

dvc/repo/get.py

@@ -18,7 +18,6 @@ class GetDVCFileError(DvcException):
         )
 
 
-@staticmethod
 def get(url, path, out=None, rev=None):
     from dvc.dvcfile import is_valid_filename
     from dvc.external_repo import external_repo

dvc/repo/get_url.py

@@ -5,7 +5,6 @@ import dvc.output as output
 from dvc.utils import resolve_output
 
 
-@staticmethod
 def get_url(url, out=None):
     out = resolve_output(url, out)
 
@@ -16,6 +15,7 @@ def get_url(url, out=None):
 
     (dep,) = dependency.loads_from(None, [url])
     (out,) = output.loads_from(None, [out], use_cache=False)
-    dep.save()
+    if not dep.exists:
+        raise dep.DoesNotExistError(dep)
     dep.download(out)
     out.save()

dvc/repo/graph.py

@@ -26,3 +26,97 @@ def get_pipelines(G):
     import networkx as nx
 
     return [G.subgraph(c).copy() for c in nx.weakly_connected_components(G)]
+
+
+def build_graph(stages, outs_trie=None):
+    """Generate a graph by using the given stages on the given directory
+
+    The nodes of the graph are the stage's path relative to the root.
+
+    Edges are created when the output of one stage is used as a
+    dependency in other stage.
+
+    The direction of the edges goes from the stage to its dependency:
+
+    For example, running the following:
+
+        $ dvc run -o A "echo A > A"
+        $ dvc run -d A -o B "echo B > B"
+        $ dvc run -d B -o C "echo C > C"
+
+    Will create the following graph:
+
+           ancestors <--
+                       |
+            C.dvc -> B.dvc -> A.dvc
+            |          |
+            |          --> descendants
+            |
+            ------- pipeline ------>
+                       |
+                       v
+          (weakly connected components)
+
+    Args:
+        stages (list): used to build a graph from
+
+    Raises:
+        OutputDuplicationError: two outputs with the same path
+        StagePathAsOutputError: stage inside an output directory
+        OverlappingOutputPathsError: output inside output directory
+        CyclicGraphError: resulting graph has cycles
+    """
+    import networkx as nx
+
+    from dvc.exceptions import StagePathAsOutputError
+
+    from ..path_info import PathInfo
+    from .trie import build_outs_trie
+
+    G = nx.DiGraph()
+
+    # Use trie to efficiently find overlapping outs and deps
+    outs_trie = outs_trie or build_outs_trie(stages)
+
+    for stage in stages:
+        out = outs_trie.shortest_prefix(PathInfo(stage.path).parts).value
+        if out:
+            raise StagePathAsOutputError(stage, str(out))
+
+    # Building graph
+    G.add_nodes_from(stages)
+    for stage in stages:
+        for dep in stage.deps:
+            if dep.path_info is None:
+                continue
+
+            dep_key = dep.path_info.parts
+            overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
+            if outs_trie.has_subtrie(dep_key):
+                overlapping.extend(outs_trie.values(prefix=dep_key))
+
+            G.add_edges_from((stage, out.stage) for out in overlapping)
+    check_acyclic(G)
+
+    return G
+
+
+# NOTE: using stage graph instead of just list of stages to make sure that it
+# has already passed all the sanity checks like cycles/overlapping outputs and
+# so on.
+def build_outs_graph(graph, outs_trie):
+    import networkx as nx
+
+    G = nx.DiGraph()
+
+    G.add_nodes_from(outs_trie.values())
+    for stage in graph.nodes():
+        for dep in stage.deps:
+            dep_key = dep.path_info.parts
+            overlapping = [n.value for n in outs_trie.prefixes(dep_key)]
+            if outs_trie.has_subtrie(dep_key):
+                overlapping.extend(outs_trie.values(prefix=dep_key))
+
+            for from_out in stage.outs:
+                G.add_edges_from((from_out, out) for out in overlapping)
+    return G

dvc/repo/imp.py

@@ -1,6 +1,8 @@
-def imp(self, url, path, out=None, rev=None):
+def imp(self, url, path, out=None, fname=None, rev=None, **kwargs):
     erepo = {"url": url}
     if rev is not None:
         erepo["rev"] = rev
 
-    return self.imp_url(path, out=out, erepo=erepo, frozen=True)
+    return self.imp_url(
+        path, out=out, fname=fname, erepo=erepo, frozen=True, **kwargs
+    )

dvc/repo/imp_url.py

@@ -11,7 +11,14 @@ from . import locked
 @locked
 @scm_context
 def imp_url(
-    self, url, out=None, fname=None, erepo=None, frozen=True, no_exec=False
+    self,
+    url,
+    out=None,
+    fname=None,
+    erepo=None,
+    frozen=True,
+    no_exec=False,
+    desc=None,
 ):
     from dvc.dvcfile import Dvcfile
     from dvc.stage import Stage, create_stage
@@ -40,6 +47,9 @@ def imp_url(
     if stage is None:
         return None
 
+    if desc:
+        stage.outs[0].desc = desc
+
     dvcfile = Dvcfile(self, stage.path)
     dvcfile.remove()
 

dvc/repo/init.py

@@ -1,49 +1,18 @@
 import logging
 import os
 
-import colorama
-
-from dvc import analytics
 from dvc.config import Config
 from dvc.exceptions import InitError, InvalidArgumentError
 from dvc.ignore import init as init_dvcignore
 from dvc.repo import Repo
 from dvc.scm import SCM
 from dvc.scm.base import SCMError
-from dvc.utils import boxify, relpath
+from dvc.utils import relpath
 from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
 
-def _welcome_message():
-    if analytics.is_enabled():
-        logger.info(
-            boxify(
-                "DVC has enabled anonymous aggregate usage analytics.\n"
-                "Read the analytics documentation (and how to opt-out) here:\n"
-                "{blue}https://dvc.org/doc/user-guide/analytics{nc}".format(
-                    blue=colorama.Fore.BLUE, nc=colorama.Fore.RESET
-                ),
-                border_color="red",
-            )
-        )
-
-    msg = (
-        "{yellow}What's next?{nc}\n"
-        "{yellow}------------{nc}\n"
-        "- Check out the documentation: {blue}https://dvc.org/doc{nc}\n"
-        "- Get help and share ideas: {blue}https://dvc.org/chat{nc}\n"
-        "- Star us on GitHub: {blue}https://github.com/iterative/dvc{nc}"
-    ).format(
-        yellow=colorama.Fore.YELLOW,
-        blue=colorama.Fore.BLUE,
-        nc=colorama.Fore.RESET,
-    )
-
-    logger.info(msg)
-
-
 def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):
     """
     Creates an empty repo on the given directory -- basically a
@@ -104,14 +73,14 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):
 
     proj = Repo(root_dir)
 
+    proj.plots.templates.init()
+
     scm.add(
-        [config.files["repo"], dvcignore, proj.plot_templates.templates_dir]
+        [config.files["repo"], dvcignore, proj.plots.templates.templates_dir]
     )
 
     if scm.ignore_file:
         scm.add([os.path.join(dvc_dir, scm.ignore_file)])
         logger.info("\nYou can now commit the changes to git.\n")
 
-    _welcome_message()
-
     return proj

dvc/repo/ls.py

@@ -4,7 +4,6 @@ from dvc.exceptions import PathMissingError
 from dvc.path_info import PathInfo
 
 
-@staticmethod
 def ls(
     url, path=None, rev=None, recursive=None, dvc_only=False,
 ):
@@ -38,7 +37,7 @@ def ls(
         if path:
             path_info /= path
 
-        ret = _ls(repo, path_info, recursive, dvc_only)
+        ret = _ls(repo.repo_tree, path_info, recursive, dvc_only)
 
         if path and not ret:
             raise PathMissingError(path, repo, dvc_only=dvc_only)
@@ -51,12 +50,10 @@ def ls(
         return ret_list
 
 
-def _ls(repo, path_info, recursive=None, dvc_only=False):
+def _ls(tree, path_info, recursive=None, dvc_only=False):
     def onerror(exc):
         raise exc
 
-    tree = repo.repo_tree
-
     infos = []
     try:
         for root, dirs, files in tree.walk(

dvc/repo/metrics/show.py

@@ -1,38 +1,28 @@
 import logging
-import os
 
-from dvc.exceptions import NoMetricsError
-from dvc.path_info import PathInfo
+from dvc.exceptions import NoMetricsFoundError, NoMetricsParsedError
 from dvc.repo import locked
+from dvc.repo.collect import collect
+from dvc.scm.base import SCMError
 from dvc.tree.repo import RepoTree
 from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml
 
 logger = logging.getLogger(__name__)
 
 
-def _collect_metrics(repo, targets, recursive):
-    metrics = set()
+def _is_metric(out):
+    return bool(out.metric)
 
-    for stage in repo.stages:
-        for out in stage.outs:
-            if not out.metric:
-                continue
 
-            metrics.add(out.path_info)
-
-    if not targets:
-        return list(metrics)
-
-    target_infos = [PathInfo(os.path.abspath(target)) for target in targets]
-
-    def _filter(path_info):
-        for info in target_infos:
-            func = path_info.isin_or_eq if recursive else path_info.__eq__
-            if func(info):
-                return True
-        return False
-
-    return list(filter(_filter, metrics))
+def _collect_metrics(repo, targets, revision, recursive):
+    metrics, path_infos = collect(
+        repo,
+        targets=targets,
+        output_filter=_is_metric,
+        recursive=recursive,
+        rev=revision,
+    )
+    return [m.path_info for m in metrics] + list(path_infos)
 
 
 def _extract_metrics(metrics, path, rev):
@@ -45,7 +35,7 @@ def _extract_metrics(metrics, path, rev):
     ret = {}
     for key, val in metrics.items():
         m = _extract_metrics(val, path, rev)
-        if m:
+        if m not in (None, {}):
             ret[key] = m
         else:
             logger.debug(
@@ -77,7 +67,7 @@ def _read_metrics(repo, metrics, rev):
             continue
 
         val = _extract_metrics(val, metric, rev)
-        if val:
+        if val not in (None, {}):
             res[str(metric)] = val
 
     return res
@@ -102,7 +92,7 @@ def show(
         all_tags=all_tags,
         all_commits=all_commits,
     ):
-        metrics = _collect_metrics(repo, targets, recursive)
+        metrics = _collect_metrics(repo, targets, rev, recursive)
 
         if not metrics_found and metrics:
             metrics_found = True
@@ -114,22 +104,17 @@ def show(
 
     if not res:
         if metrics_found:
-            msg = (
-                "Could not parse metric files. Use `-v` option to see more "
-                "details."
-            )
+            raise NoMetricsParsedError("metrics")
         else:
-            msg = (
-                "no metric files in this repository. Use `-m/-M` options for "
-                "`dvc run` to mark stage outputs as  metrics."
-            )
-        raise NoMetricsError(msg)
+            raise NoMetricsFoundError("metrics", "-m/-M")
 
     # Hide workspace metrics if they are the same as in the active branch
     try:
         active_branch = repo.scm.active_branch()
-    except TypeError:
-        pass  # Detached head
+    except (TypeError, SCMError):
+        # TypeError - detached head
+        # SCMError - no repo case
+        pass
     else:
         if res.get("workspace") == res.get(active_branch):
             res.pop("workspace", None)

dvc/repo/params/show.py

@@ -1,11 +1,17 @@
 import logging
+from typing import TYPE_CHECKING, List
 
 from dvc.dependency.param import ParamsDependency
 from dvc.exceptions import DvcException
 from dvc.path_info import PathInfo
 from dvc.repo import locked
+from dvc.repo.collect import collect
 from dvc.utils.serialize import LOADERS, ParseError
 
+if TYPE_CHECKING:
+    from dvc.output.base import BaseOutput
+    from dvc.repo import Repo
+
 logger = logging.getLogger(__name__)
 
 
@@ -13,15 +19,14 @@ class NoParamsError(DvcException):
     pass
 
 
-def _collect_configs(repo):
-    configs = set()
-    configs.add(PathInfo(repo.root_dir) / ParamsDependency.DEFAULT_PARAMS_FILE)
-    for stage in repo.stages:
-        for dep in stage.deps:
-            if not isinstance(dep, ParamsDependency):
-                continue
+def _is_params(dep: "BaseOutput"):
+    return isinstance(dep, ParamsDependency)
 
-            configs.add(dep.path_info)
+
+def _collect_configs(repo: "Repo", rev) -> List[PathInfo]:
+    params, _ = collect(repo, deps=True, output_filter=_is_params, rev=rev)
+    configs = {p.path_info for p in params}
+    configs.add(PathInfo(repo.root_dir) / ParamsDependency.DEFAULT_PARAMS_FILE)
     return list(configs)
 
 
@@ -49,7 +54,7 @@ def show(repo, revs=None):
     res = {}
 
     for branch in repo.brancher(revs=revs):
-        configs = _collect_configs(repo)
+        configs = _collect_configs(repo, branch)
         params = _read_params(repo, configs, branch)
 
         if params:

dvc/repo/plots/__init__.py

@@ -1,8 +1,14 @@
 import logging
 
-from funcy import first, project
+from funcy import cached_property, first, project
 
-from dvc.exceptions import DvcException, NoPlotsError, OutputNotFoundError
+from dvc.exceptions import (
+    DvcException,
+    NoMetricsFoundError,
+    NoMetricsParsedError,
+)
+from dvc.repo.collect import collect
+from dvc.repo.plots.data import PlotParsingError
 from dvc.schema import PLOT_PROPS
 from dvc.tree.repo import RepoTree
 from dvc.utils import relpath
@@ -70,10 +76,24 @@ class Plots:
         # Merge data by plot file and apply overriding props
         plots = _prepare_plots(data, revs, props)
 
-        return {
-            datafile: _render(datafile, desc["data"], desc["props"], templates)
-            for datafile, desc in plots.items()
-        }
+        result = {}
+        for datafile, desc in plots.items():
+            try:
+                result[datafile] = _render(
+                    datafile, desc["data"], desc["props"], templates
+                )
+            except PlotParsingError as e:
+                logger.debug(
+                    "failed to read '%s' on '%s'",
+                    e.path,
+                    e.revision,
+                    exc_info=True,
+                )
+
+        if not any(result.values()):
+            raise NoMetricsParsedError("plots")
+
+        return result
 
     def show(self, targets=None, revs=None, props=None, templates=None):
         from .data import NoMetricInHistoryError
@@ -83,15 +103,16 @@ class Plots:
         # If any mentioned plot doesn't have any data then that's an error
         targets = [targets] if isinstance(targets, str) else targets or []
         for target in targets:
-            if not any("data" in d[target] for d in data.values()):
+            rpath = relpath(target, self.repo.root_dir)
+            if not any("data" in d[rpath] for d in data.values()):
                 raise NoMetricInHistoryError(target)
 
         # No data at all is a special error with a special message
         if not data:
-            raise NoPlotsError()
+            raise NoMetricsFoundError("plots", "--plots/--plots-no-cache")
 
         if templates is None:
-            templates = self.repo.plot_templates
+            templates = self.templates
         return self.render(data, revs, props, templates)
 
     def diff(self, *args, **kwargs):
@@ -116,7 +137,7 @@ class Plots:
         props = props or {}
         template = props.get("template")
         if template:
-            self.repo.plot_templates.get_template(template)
+            self.templates.get_template(template)
 
         (out,) = self.repo.find_outs_by_path(path)
         if not out.plot and unset is not None:
@@ -140,25 +161,24 @@ class Plots:
         dvcfile = Dvcfile(self.repo, out.stage.path)
         dvcfile.dump(out.stage, update_lock=False)
 
+    @cached_property
+    def templates(self):
+        from .template import PlotTemplates
+
+        return PlotTemplates(self.repo.dvc_dir)
+
+
+def _is_plot(out):
+    return bool(out.plot)
+
 
 def _collect_plots(repo, targets=None, rev=None):
-    def _targets_to_outs(targets):
-        for t in targets:
-            try:
-                (out,) = repo.find_outs_by_path(t)
-                yield out
-            except OutputNotFoundError:
-                logger.warning(
-                    "File '{}' was not found at: '{}'. It will not be "
-                    "plotted.".format(t, rev)
-                )
-
-    if targets:
-        outs = _targets_to_outs(targets)
-    else:
-        outs = (out for stage in repo.stages for out in stage.outs if out.plot)
-
-    return {out.path_info: _plot_props(out) for out in outs}
+    plots, path_infos = collect(
+        repo, output_filter=_is_plot, targets=targets, rev=rev
+    )
+    result = {plot.path_info: _plot_props(plot) for plot in plots}
+    result.update({path_info: {} for path_info in path_infos})
+    return result
 
 
 def _plot_props(out):

dvc/repo/plots/data.py

@@ -1,14 +1,13 @@
 import csv
 import io
-import json
 import os
 from collections import OrderedDict
 from copy import copy
 
-from funcy import first
+from funcy import first, reraise
 
 from dvc.exceptions import DvcException
-from dvc.utils.serialize import loads_yaml
+from dvc.utils.serialize import ParseError, parse_json, parse_yaml
 
 
 class PlotMetricTypeError(DvcException):
@@ -32,6 +31,14 @@ class NoMetricInHistoryError(DvcException):
         super().__init__(f"Could not find '{path}'.")
 
 
+class PlotParsingError(ParseError):
+    def __init__(self, path, revision):
+        self.path = path
+        self.revision = revision
+
+        super().__init__(path, f"revision: {revision}")
+
+
 def plot_data(filename, revision, content):
     _, extension = os.path.splitext(filename.lower())
     if extension == ".json":
@@ -159,7 +166,11 @@ class PlotData:
         return [_filter_fields, _append_index, _append_revision]
 
     def to_datapoints(self, **kwargs):
-        data = self.raw(**kwargs)
+        with reraise(
+            [ParseError, csv.Error],
+            PlotParsingError(self.filename, self.revision),
+        ):
+            data = self.raw(**kwargs)
 
         for data_proc in self._processors():
             data = data_proc(
@@ -170,7 +181,9 @@ class PlotData:
 
 class JSONPlotData(PlotData):
     def raw(self, **kwargs):
-        return json.loads(self.content, object_pairs_hook=OrderedDict)
+        return parse_json(
+            self.content, self.filename, object_pairs_hook=OrderedDict
+        )
 
     def _processors(self):
         parent_processors = super()._processors()
@@ -207,7 +220,7 @@ class CSVPlotData(PlotData):
 
 class YAMLPlotData(PlotData):
     def raw(self, **kwargs):
-        return loads_yaml(self.content, typ="rt")
+        return parse_yaml(self.content, self.filename, typ="rt")
 
     def _processors(self):
         parent_processors = super()._processors()

dvc/repo/plots/diff.py

@@ -14,7 +14,7 @@ def _revisions(repo, revs, experiment):
 def diff(repo, *args, revs=None, experiment=False, **kwargs):
     if experiment:
         # use experiments repo brancher with templates from the main repo
-        kwargs["templates"] = repo.plot_templates
+        kwargs["templates"] = repo.plots.templates
         plots_repo = repo.experiments.exp_dvc
     else:
         plots_repo = repo

dvc/repo/plots/template.py

@@ -1,5 +1,6 @@
 import json
 import os
+from typing import Any, Dict, Optional
 
 from funcy import cached_property
 
@@ -29,11 +30,22 @@ class Template:
     EXTENSION = ".json"
     ANCHOR = "<DVC_METRIC_{}>"
 
-    DEFAULT_CONTENT = None
-    DEFAULT_NAME = None
+    DEFAULT_CONTENT: Optional[Dict[str, Any]] = None
+    DEFAULT_NAME: Optional[str] = None
 
     def __init__(self, content=None, name=None):
-        self.content = self.DEFAULT_CONTENT if content is None else content
+        if content:
+            self.content = content
+        else:
+            self.content = (
+                json.dumps(
+                    self.DEFAULT_CONTENT,
+                    indent=self.INDENT,
+                    separators=self.SEPARATORS,
+                )
+                + "\n"
+            )
+
         self.name = name or self.DEFAULT_NAME
         assert self.content and self.name
         self.filename = self.name + self.EXTENSION
@@ -126,22 +138,164 @@ class DefaultConfusionTemplate(Template):
         "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
         "data": {"values": Template.anchor("data")},
         "title": Template.anchor("title"),
-        "mark": "rect",
-        "encoding": {
-            "x": {
-                "field": Template.anchor("x"),
-                "type": "nominal",
-                "sort": "ascending",
-                "title": Template.anchor("x_label"),
+        "facet": {"field": "rev", "type": "nominal"},
+        "spec": {
+            "transform": [
+                {
+                    "aggregate": [{"op": "count", "as": "xy_count"}],
+                    "groupby": [Template.anchor("y"), Template.anchor("x")],
+                },
+                {
+                    "impute": "xy_count",
+                    "groupby": ["rev", Template.anchor("y")],
+                    "key": Template.anchor("x"),
+                    "value": 0,
+                },
+                {
+                    "impute": "xy_count",
+                    "groupby": ["rev", Template.anchor("x")],
+                    "key": Template.anchor("y"),
+                    "value": 0,
+                },
+                {
+                    "joinaggregate": [
+                        {"op": "max", "field": "xy_count", "as": "max_count"}
+                    ],
+                    "groupby": [],
+                },
+                {
+                    "calculate": "datum.xy_count / datum.max_count",
+                    "as": "percent_of_max",
+                },
+            ],
+            "encoding": {
+                "x": {
+                    "field": Template.anchor("x"),
+                    "type": "nominal",
+                    "sort": "ascending",
+                    "title": Template.anchor("x_label"),
+                },
+                "y": {
+                    "field": Template.anchor("y"),
+                    "type": "nominal",
+                    "sort": "ascending",
+                    "title": Template.anchor("y_label"),
+                },
             },
-            "y": {
-                "field": Template.anchor("y"),
-                "type": "nominal",
-                "sort": "ascending",
-                "title": Template.anchor("y_label"),
+            "layer": [
+                {
+                    "mark": "rect",
+                    "width": 300,
+                    "height": 300,
+                    "encoding": {
+                        "color": {
+                            "field": "xy_count",
+                            "type": "quantitative",
+                            "title": "",
+                            "scale": {"domainMin": 0, "nice": True},
+                        }
+                    },
+                },
+                {
+                    "mark": "text",
+                    "encoding": {
+                        "text": {"field": "xy_count", "type": "quantitative"},
+                        "color": {
+                            "condition": {
+                                "test": "datum.percent_of_max > 0.5",
+                                "value": "white",
+                            },
+                            "value": "black",
+                        },
+                    },
+                },
+            ],
+        },
+    }
+
+
+class NormalizedConfusionTemplate(Template):
+    DEFAULT_NAME = "confusion_normalized"
+    DEFAULT_CONTENT = {
+        "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+        "data": {"values": Template.anchor("data")},
+        "title": Template.anchor("title"),
+        "facet": {"field": "rev", "type": "nominal"},
+        "spec": {
+            "transform": [
+                {
+                    "aggregate": [{"op": "count", "as": "xy_count"}],
+                    "groupby": [Template.anchor("y"), Template.anchor("x")],
+                },
+                {
+                    "impute": "xy_count",
+                    "groupby": ["rev", Template.anchor("y")],
+                    "key": Template.anchor("x"),
+                    "value": 0,
+                },
+                {
+                    "impute": "xy_count",
+                    "groupby": ["rev", Template.anchor("x")],
+                    "key": Template.anchor("y"),
+                    "value": 0,
+                },
+                {
+                    "joinaggregate": [
+                        {"op": "sum", "field": "xy_count", "as": "sum_y"}
+                    ],
+                    "groupby": [Template.anchor("y")],
+                },
+                {
+                    "calculate": "datum.xy_count / datum.sum_y",
+                    "as": "percent_of_y",
+                },
+            ],
+            "encoding": {
+                "x": {
+                    "field": Template.anchor("x"),
+                    "type": "nominal",
+                    "sort": "ascending",
+                    "title": Template.anchor("x_label"),
+                },
+                "y": {
+                    "field": Template.anchor("y"),
+                    "type": "nominal",
+                    "sort": "ascending",
+                    "title": Template.anchor("y_label"),
+                },
             },
-            "color": {"aggregate": "count", "type": "quantitative"},
-            "facet": {"field": "rev", "type": "nominal"},
+            "layer": [
+                {
+                    "mark": "rect",
+                    "width": 300,
+                    "height": 300,
+                    "encoding": {
+                        "color": {
+                            "field": "percent_of_y",
+                            "type": "quantitative",
+                            "title": "",
+                            "scale": {"domain": [0, 1]},
+                        }
+                    },
+                },
+                {
+                    "mark": "text",
+                    "encoding": {
+                        "text": {
+                            "field": "percent_of_y",
+                            "type": "quantitative",
+                            "format": ".2f",
+                        },
+                        "color": {
+                            "condition": {
+                                "test": "datum.percent_of_y > 0.5",
+                                "value": "white",
+                            },
+                            "value": "black",
+                        },
+                    },
+                },
+            ],
         },
     }
 
@@ -208,6 +362,7 @@ class PlotTemplates:
     TEMPLATES = [
         DefaultLinearTemplate,
         DefaultConfusionTemplate,
+        NormalizedConfusionTemplate,
         DefaultScatterTemplate,
         SmoothLinearTemplate,
     ]
@@ -216,60 +371,54 @@ class PlotTemplates:
     def templates_dir(self):
         return os.path.join(self.dvc_dir, self.TEMPLATES_DIR)
 
-    @cached_property
-    def default_template(self):
-        default_plot_path = os.path.join(self.templates_dir, "default.json")
-        if not os.path.exists(default_plot_path):
-            raise TemplateNotFoundError(os.path.relpath(default_plot_path))
-        return default_plot_path
-
     def get_template(self, path):
         if os.path.exists(path):
             return path
 
-        t_path = os.path.join(self.templates_dir, path)
-        if os.path.exists(t_path):
-            return t_path
+        if self.dvc_dir and os.path.exists(self.dvc_dir):
+            t_path = os.path.join(self.templates_dir, path)
+            if os.path.exists(t_path):
+                return t_path
 
-        all_templates = [
-            os.path.join(root, file)
-            for root, _, files in os.walk(self.templates_dir)
-            for file in files
-        ]
-        matches = [
-            template
-            for template in all_templates
-            if os.path.splitext(template)[0] == t_path
-        ]
-        if matches:
-            assert len(matches) == 1
-            return matches[0]
+            all_templates = [
+                os.path.join(root, file)
+                for root, _, files in os.walk(self.templates_dir)
+                for file in files
+            ]
+            matches = [
+                template
+                for template in all_templates
+                if os.path.splitext(template)[0] == t_path
+            ]
+            if matches:
+                assert len(matches) == 1
+                return matches[0]
 
         raise TemplateNotFoundError(path)
 
     def __init__(self, dvc_dir):
         self.dvc_dir = dvc_dir
 
-        if not os.path.exists(self.templates_dir):
-            makedirs(self.templates_dir, exist_ok=True)
-            for t in self.TEMPLATES:
-                self.dump(t())
+    def init(self):
+        makedirs(self.templates_dir, exist_ok=True)
+        for t in self.TEMPLATES:
+            self._dump(t())
 
-    def dump(self, template):
+    def _dump(self, template):
         path = os.path.join(self.templates_dir, template.filename)
         with open(path, "w") as fd:
-            json.dump(
-                template.content,
-                fd,
-                indent=template.INDENT,
-                separators=template.SEPARATORS,
-            )
-            fd.write("\n")
+            fd.write(template.content)
 
     def load(self, name):
-        path = self.get_template(name)
+        try:
+            path = self.get_template(name)
 
-        with open(path) as fd:
-            content = fd.read()
+            with open(path) as fd:
+                content = fd.read()
 
-        return Template(content, name=name)
+            return Template(content, name=name)
+        except TemplateNotFoundError:
+            for template in self.TEMPLATES:
+                if template.DEFAULT_NAME == name:
+                    return template()
+            raise

dvc/repo/reproduce.py

@@ -1,8 +1,9 @@
 import logging
+from functools import partial
 
 from dvc.exceptions import InvalidArgumentError, ReproductionError
-from dvc.repo.experiments import UnchangedExperimentError
 from dvc.repo.scm_context import scm_context
+from dvc.stage.run import CheckpointKilledError
 
 from . import locked
 from .graph import get_pipeline, get_pipelines
@@ -11,6 +12,21 @@ logger = logging.getLogger(__name__)
 
 
 def _reproduce_stage(stage, **kwargs):
+    def _run_callback(repro_callback):
+        _dump_stage(stage)
+        repro_callback([stage])
+
+    checkpoint_func = kwargs.pop("checkpoint_func", None)
+    if stage.is_checkpoint:
+        if checkpoint_func:
+            kwargs["checkpoint_func"] = partial(_run_callback, checkpoint_func)
+        else:
+            logger.warning(
+                "Checkpoint stages are not fully supported in 'dvc repro'. "
+                "Checkpoint stages should be reproduced with 'dvc exp run' "
+                "or 'dvc exp resume'."
+            )
+
     if stage.frozen and not stage.is_import:
         logger.warning(
             "{} is frozen. Its dependencies are"
@@ -22,14 +38,18 @@ def _reproduce_stage(stage, **kwargs):
         return []
 
     if not kwargs.get("dry", False):
-        from ..dvcfile import Dvcfile
-
-        dvcfile = Dvcfile(stage.repo, stage.path)
-        dvcfile.dump(stage, update_pipeline=False)
+        _dump_stage(stage)
 
     return [stage]
 
 
+def _dump_stage(stage):
+    from ..dvcfile import Dvcfile
+
+    dvcfile = Dvcfile(stage.repo, stage.path)
+    dvcfile.dump(stage, update_pipeline=False)
+
+
 def _get_active_graph(G):
     import networkx as nx
 
@@ -70,28 +90,6 @@ def reproduce(
             "Neither `target` nor `--all-pipelines` are specified."
         )
 
-    experiment = kwargs.pop("experiment", False)
-    params = _parse_params(kwargs.pop("params", []))
-    queue = kwargs.pop("queue", False)
-    run_all = kwargs.pop("run_all", False)
-    jobs = kwargs.pop("jobs", 1)
-    if (experiment or run_all) and self.experiments:
-        try:
-            return _reproduce_experiments(
-                self,
-                target=target,
-                recursive=recursive,
-                all_pipelines=all_pipelines,
-                params=params,
-                queue=queue,
-                run_all=run_all,
-                jobs=jobs,
-                **kwargs,
-            )
-        except UnchangedExperimentError:
-            # If experiment contains no changes, just run regular repro
-            pass
-
     interactive = kwargs.get("interactive", False)
     if not interactive:
         kwargs["interactive"] = self.config["core"].get("interactive", False)
@@ -118,39 +116,6 @@ def reproduce(
     return _reproduce_stages(active_graph, targets, **kwargs)
 
 
-def _parse_params(path_params):
-    from ruamel.yaml import YAMLError
-
-    from dvc.dependency.param import ParamsDependency
-    from dvc.utils.flatten import unflatten
-    from dvc.utils.serialize import loads_yaml
-
-    ret = {}
-    for path_param in path_params:
-        path, _, params_str = path_param.rpartition(":")
-        # remove empty strings from params, on condition such as `-p "file1:"`
-        params = {}
-        for param_str in filter(bool, params_str.split(",")):
-            try:
-                # interpret value strings using YAML rules
-                key, value = param_str.split("=")
-                params[key] = loads_yaml(value)
-            except (ValueError, YAMLError):
-                raise InvalidArgumentError(
-                    f"Invalid param/value pair '{param_str}'"
-                )
-        if not path:
-            path = ParamsDependency.DEFAULT_PARAMS_FILE
-        ret[path] = unflatten(params)
-    return ret
-
-
-def _reproduce_experiments(repo, run_all=False, jobs=1, **kwargs):
-    if run_all:
-        return repo.experiments.reproduce_queued(jobs=jobs)
-    return repo.experiments.reproduce_one(**kwargs)
-
-
 def _reproduce_stages(
     G, stages, downstream=False, single_item=False, on_unchanged=None, **kwargs
 ):
@@ -189,6 +154,49 @@ def _reproduce_stages(
 
     The derived evaluation of _downstream_ B would be: [B, D, E]
     """
+    pipeline = _get_pipeline(G, stages, downstream, single_item)
+
+    force_downstream = kwargs.pop("force_downstream", False)
+    result = []
+    unchanged = []
+    # `ret` is used to add a cosmetic newline.
+    ret = []
+    checkpoint_func = kwargs.pop("checkpoint_func", None)
+    for stage in pipeline:
+        if ret:
+            logger.info("")
+
+        if checkpoint_func:
+            kwargs["checkpoint_func"] = partial(
+                _repro_callback, checkpoint_func, unchanged
+            )
+
+        try:
+            ret = _reproduce_stage(stage, **kwargs)
+
+            if len(ret) == 0:
+                unchanged.extend([stage])
+            elif force_downstream:
+                # NOTE: we are walking our pipeline from the top to the
+                # bottom. If one stage is changed, it will be reproduced,
+                # which tells us that we should force reproducing all of
+                # the other stages down below, even if their direct
+                # dependencies didn't change.
+                kwargs["force"] = True
+
+            if ret:
+                result.extend(ret)
+        except CheckpointKilledError:
+            raise
+        except Exception as exc:
+            raise ReproductionError(stage.relpath) from exc
+
+    if on_unchanged is not None:
+        on_unchanged(unchanged)
+    return result
+
+
+def _get_pipeline(G, stages, downstream, single_item):
     import networkx as nx
 
     if single_item:
@@ -217,30 +225,8 @@ def _reproduce_stages(
         if stage not in pipeline:
             pipeline.append(stage)
 
-    force_downstream = kwargs.pop("force_downstream", False)
-    result = []
-    # `ret` is used to add a cosmetic newline.
-    ret = []
-    for stage in pipeline:
-        if ret:
-            logger.info("")
+    return pipeline
 
-        try:
-            ret = _reproduce_stage(stage, **kwargs)
 
-            if len(ret) != 0 and force_downstream:
-                # NOTE: we are walking our pipeline from the top to the
-                # bottom. If one stage is changed, it will be reproduced,
-                # which tells us that we should force reproducing all of
-                # the other stages down below, even if their direct
-                # dependencies didn't change.
-                kwargs["force"] = True
-
-            if ret:
-                result.extend(ret)
-            elif on_unchanged is not None:
-                on_unchanged(stage)
-        except Exception as exc:
-            raise ReproductionError(stage.relpath) from exc
-
-    return result
+def _repro_callback(experiments_callback, unchanged, stages):
+    experiments_callback(unchanged, stages)

dvc/repo/run.py

@@ -48,6 +48,7 @@ def _get_file_path(kwargs):
             kwargs.get("plots_no_cache", []),
             kwargs.get("outs_persist", []),
             kwargs.get("outs_persist_no_cache", []),
+            kwargs.get("checkpoints", []),
         )
     )
 

dvc/repo/scm_context.py

@@ -3,7 +3,11 @@ def scm_context(method):
         try:
             result = method(repo, *args, **kw)
             repo.scm.reset_ignores()
-            repo.scm.remind_to_track()
+            autostage = repo.config.get("core", {}).get("autostage", False)
+            if autostage:
+                repo.scm.track_changed_files()
+            else:
+                repo.scm.remind_to_track()
             repo.scm.reset_tracked_files()
             return result
         except Exception:

dvc/repo/status.py

@@ -74,7 +74,7 @@ def _cloud_status(
 
             { "bar": "deleted" }
     """
-    import dvc.cache.base as cloud
+    import dvc.remote.base as cloud
 
     used = self.used_cache(
         targets,

dvc/repo/trie.py

@@ -0,0 +1,41 @@
+from funcy import first
+from pygtrie import Trie
+
+from dvc.exceptions import OutputDuplicationError, OverlappingOutputPathsError
+
+
+def build_outs_trie(stages):
+    outs = Trie()
+
+    for stage in filter(bool, stages):  # bug? not using it later
+        for out in stage.outs:
+            out_key = out.path_info.parts
+
+            # Check for dup outs
+            if out_key in outs:
+                dup_stages = [stage, outs[out_key].stage]
+                raise OutputDuplicationError(str(out), dup_stages)
+
+            # Check for overlapping outs
+            if outs.has_subtrie(out_key):
+                parent = out
+                overlapping = first(outs.values(prefix=out_key))
+            else:
+                parent = outs.shortest_prefix(out_key).value
+                overlapping = out
+            if parent and overlapping:
+                msg = (
+                    "Paths for outs:\n'{}'('{}')\n'{}'('{}')\n"
+                    "overlap. To avoid unpredictable behaviour, "
+                    "rerun command with non overlapping outs paths."
+                ).format(
+                    str(parent),
+                    parent.stage.addressing,
+                    str(overlapping),
+                    overlapping.stage.addressing,
+                )
+                raise OverlappingOutputPathsError(parent, overlapping, msg)
+
+            outs[out_key] = out
+
+    return outs

dvc/schema.py

@@ -1,7 +1,9 @@
 from voluptuous import Any, Optional, Required, Schema
 
 from dvc import dependency, output
+from dvc.hash_info import HashInfo
 from dvc.output import CHECKSUMS_SCHEMA, BaseOutput
+from dvc.parsing import DO_KWD, FOREACH_KWD, SET_KWD, VARS_KWD
 from dvc.stage.params import StageParams
 
 STAGES = "stages"
@@ -15,9 +17,15 @@ SINGLE_STAGE_SCHEMA = {
     StageParams.PARAM_FROZEN: bool,
     StageParams.PARAM_META: object,
     StageParams.PARAM_ALWAYS_CHANGED: bool,
+    StageParams.PARAM_DESC: str,
 }
 
-DATA_SCHEMA = {**CHECKSUMS_SCHEMA, Required("path"): str}
+DATA_SCHEMA = {
+    **CHECKSUMS_SCHEMA,
+    Required("path"): str,
+    HashInfo.PARAM_SIZE: int,
+    HashInfo.PARAM_NFILES: int,
+}
 LOCK_FILE_STAGE_SCHEMA = {
     Required(StageParams.PARAM_CMD): str,
     StageParams.PARAM_DEPS: [DATA_SCHEMA],
@@ -27,7 +35,12 @@ LOCK_FILE_STAGE_SCHEMA = {
 LOCKFILE_SCHEMA = {str: LOCK_FILE_STAGE_SCHEMA}
 
 OUT_PSTAGE_DETAILED_SCHEMA = {
-    str: {BaseOutput.PARAM_CACHE: bool, BaseOutput.PARAM_PERSIST: bool}
+    str: {
+        BaseOutput.PARAM_CACHE: bool,
+        BaseOutput.PARAM_PERSIST: bool,
+        BaseOutput.PARAM_CHECKPOINT: bool,
+        BaseOutput.PARAM_DESC: str,
+    }
 }
 
 PLOT_PROPS = {
@@ -47,27 +60,36 @@ PLOT_PSTAGE_SCHEMA = {str: Any(PLOT_PROPS_SCHEMA, [PLOT_PROPS_SCHEMA])}
 
 PARAM_PSTAGE_NON_DEFAULT_SCHEMA = {str: [str]}
 
-SINGLE_PIPELINE_STAGE_SCHEMA = {
-    str: {
-        StageParams.PARAM_CMD: str,
-        Optional(StageParams.PARAM_WDIR): str,
-        Optional(StageParams.PARAM_DEPS): [str],
-        Optional(StageParams.PARAM_PARAMS): [
-            Any(str, PARAM_PSTAGE_NON_DEFAULT_SCHEMA)
-        ],
-        Optional(StageParams.PARAM_FROZEN): bool,
-        Optional(StageParams.PARAM_META): object,
-        Optional(StageParams.PARAM_ALWAYS_CHANGED): bool,
-        Optional(StageParams.PARAM_OUTS): [
-            Any(str, OUT_PSTAGE_DETAILED_SCHEMA)
-        ],
-        Optional(StageParams.PARAM_METRICS): [
-            Any(str, OUT_PSTAGE_DETAILED_SCHEMA)
-        ],
-        Optional(StageParams.PARAM_PLOTS): [Any(str, PLOT_PSTAGE_SCHEMA)],
-    }
+VARS_SCHEMA = [str, dict]
+
+STAGE_DEFINITION = {
+    StageParams.PARAM_CMD: str,
+    Optional(SET_KWD): dict,
+    Optional(StageParams.PARAM_WDIR): str,
+    Optional(StageParams.PARAM_DEPS): [str],
+    Optional(StageParams.PARAM_PARAMS): [Any(str, dict)],
+    Optional(VARS_KWD): VARS_SCHEMA,
+    Optional(StageParams.PARAM_FROZEN): bool,
+    Optional(StageParams.PARAM_META): object,
+    Optional(StageParams.PARAM_DESC): str,
+    Optional(StageParams.PARAM_ALWAYS_CHANGED): bool,
+    Optional(StageParams.PARAM_OUTS): [Any(str, OUT_PSTAGE_DETAILED_SCHEMA)],
+    Optional(StageParams.PARAM_METRICS): [
+        Any(str, OUT_PSTAGE_DETAILED_SCHEMA)
+    ],
+    Optional(StageParams.PARAM_PLOTS): [Any(str, PLOT_PSTAGE_SCHEMA)],
+}
+
+FOREACH_IN = {
+    Optional(SET_KWD): dict,
+    Required(FOREACH_KWD): Any(dict, list, str),
+    Required(DO_KWD): STAGE_DEFINITION,
+}
+SINGLE_PIPELINE_STAGE_SCHEMA = {str: Any(STAGE_DEFINITION, FOREACH_IN)}
+MULTI_STAGE_SCHEMA = {
+    STAGES: SINGLE_PIPELINE_STAGE_SCHEMA,
+    VARS_KWD: VARS_SCHEMA,
 }
-MULTI_STAGE_SCHEMA = {STAGES: SINGLE_PIPELINE_STAGE_SCHEMA}
 
 COMPILED_SINGLE_STAGE_SCHEMA = Schema(SINGLE_STAGE_SCHEMA)
 COMPILED_MULTI_STAGE_SCHEMA = Schema(MULTI_STAGE_SCHEMA)

dvc/scheme.py

@@ -1,6 +1,7 @@
 class Schemes:
     SSH = "ssh"
     HDFS = "hdfs"
+    WEBHDFS = "webhdfs"
     S3 = "s3"
     AZURE = "azure"
     HTTP = "http"

dvc/scm/base.py

@@ -41,7 +41,7 @@ class Base:
         self._root_dir = os.path.realpath(root_dir)
 
     @property
-    def root_dir(self):
+    def root_dir(self) -> str:
         return self._root_dir
 
     def __repr__(self):
@@ -154,6 +154,11 @@ class Base:
         Method to remind user to track newly created files handled by scm
         """
 
+    def track_changed_files(self):
+        """
+        Method to stage files that have changed
+        """
+
     def track_file(self, path):
         """
         Method to add file to mechanism that will remind user

dvc/scm/git.py

@@ -4,8 +4,9 @@ import logging
 import os
 import shlex
 from functools import partial
+from typing import Optional
 
-from funcy import cached_property
+from funcy import cached_property, first
 from pathspec.patterns import GitWildMatchPattern
 
 from dvc.exceptions import GitHookAlreadyExistsError
@@ -89,7 +90,7 @@ class Git(Base):
         self.files_to_track = set()
 
     @property
-    def root_dir(self):
+    def root_dir(self) -> str:
         return self.repo.working_tree_dir
 
     @staticmethod
@@ -177,19 +178,18 @@ class Git(Base):
 
         gitignore = os.path.join(ignore_file_dir, self.GITIGNORE)
 
-        if not path_isin(gitignore, os.path.realpath(self.root_dir)):
+        if not path_isin(os.path.realpath(gitignore), self.root_dir):
             raise FileNotInRepoError(path)
 
         return entry, gitignore
 
     def _ignored(self, path):
-        from git.exc import GitCommandError
+        from dulwich import ignore
+        from dulwich.repo import Repo
 
-        try:
-            self.repo.git.check_ignore(path)
-            return True
-        except GitCommandError:
-            return False
+        repo = Repo(self.root_dir)
+        manager = ignore.IgnoreFilterManager.from_repo(repo)
+        return manager.is_ignored(relpath(path, self.root_dir))
 
     def ignore(self, path):
         entry, gitignore = self._get_gitignore(path)
@@ -210,6 +210,7 @@ class Git(Base):
         entry = GitWildMatchPattern.escape(entry)
 
         with open(gitignore, "a+", encoding="utf-8") as fobj:
+            unique_lines = set(fobj.readlines())
             fobj.seek(0, os.SEEK_END)
             if fobj.tell() == 0:
                 # Empty file
@@ -218,7 +219,9 @@ class Git(Base):
                 fobj.seek(fobj.tell() - 1, os.SEEK_SET)
                 last = fobj.read(1)
                 prefix = "" if last == "\n" else "\n"
-            fobj.write(f"{prefix}{entry}\n")
+            new_entry = f"{prefix}{entry}\n"
+            if new_entry not in unique_lines:
+                fobj.write(new_entry)
 
     def ignore_remove(self, path):
         entry, gitignore = self._get_gitignore(path)
@@ -285,8 +288,8 @@ class Git(Base):
     def is_tracked(self, path):
         return bool(self.repo.git.ls_files(path))
 
-    def is_dirty(self):
-        return self.repo.is_dirty()
+    def is_dirty(self, **kwargs):
+        return self.repo.is_dirty(**kwargs)
 
     def active_branch(self):
         return self.repo.active_branch.name
@@ -381,6 +384,12 @@ class Git(Base):
             "\tgit add {files}".format(files=files)
         )
 
+    def track_changed_files(self):
+        if not self.files_to_track:
+            return
+
+        self.add(self.files_to_track)
+
     def track_file(self, path):
         self.files_to_track.add(path)
 
@@ -460,3 +469,25 @@ class Git(Base):
     @property
     def no_commits(self):
         return not self.list_all_commits()
+
+    def branch_revs(self, branch: str, end_rev: Optional[str] = None):
+        """Iterate over revisions in a given branch (from newest to oldest).
+
+        If end_rev is set, iterator will stop when the specified revision is
+        reached.
+        """
+        commit = self.resolve_commit(branch)
+        while commit is not None:
+            yield commit.hexsha
+            commit = first(commit.parents)
+            if commit and commit.hexsha == end_rev:
+                return
+
+    def resolve_commit(self, rev):
+        """Return Commit object for the specified revision."""
+        from git.objects.tag import TagObject
+
+        commit = self.repo.rev_parse(rev)
+        if isinstance(commit, TagObject):
+            commit = commit.object
+        return commit

dvc/stage/__init__.py

@@ -2,12 +2,20 @@ import logging
 import os
 import string
 from collections import defaultdict
+from dataclasses import dataclass
+from typing import Optional
 
 from funcy import cached_property, project
 
 import dvc.dependency as dependency
 import dvc.prompt as prompt
-from dvc.exceptions import CheckoutError, DvcException, MergeError
+from dvc.exceptions import (
+    CacheLinkError,
+    CheckoutError,
+    DvcException,
+    MergeError,
+)
+from dvc.output.base import OutputDoesNotExistError
 from dvc.utils import relpath
 
 from . import params
@@ -46,6 +54,7 @@ def loads_from(cls, repo, path, wdir, data):
                 Stage.PARAM_FROZEN,
                 Stage.PARAM_ALWAYS_CHANGED,
                 Stage.PARAM_MD5,
+                Stage.PARAM_DESC,
                 "name",
             ],
         ),
@@ -53,6 +62,12 @@ def loads_from(cls, repo, path, wdir, data):
     return cls(**kw)
 
 
+@dataclass
+class RawData:
+    parametrized: bool = False
+    generated_from: Optional[str] = None
+
+
 def create_stage(cls, repo, path, external=False, **kwargs):
     from dvc.dvcfile import check_dvc_filename
 
@@ -105,6 +120,7 @@ class Stage(params.StageParams):
         always_changed=False,
         stage_text=None,
         dvcfile=None,
+        desc=None,
     ):
         if deps is None:
             deps = []
@@ -122,13 +138,15 @@ class Stage(params.StageParams):
         self.always_changed = always_changed
         self._stage_text = stage_text
         self._dvcfile = dvcfile
+        self.desc = desc
+        self.raw_data = RawData()
 
     @property
-    def path(self):
+    def path(self) -> str:
         return self._path
 
     @path.setter
-    def path(self, path):
+    def path(self, path: str):
         self._path = path
         self.__dict__.pop("path_in_repo", None)
         self.__dict__.pop("relpath", None)
@@ -160,7 +178,7 @@ class Stage(params.StageParams):
         return f"stage: '{self.addressing}'"
 
     @property
-    def addressing(self):
+    def addressing(self) -> str:
         """
         Useful for alternative presentations where we don't need
         `Stage:` prefix.
@@ -210,6 +228,14 @@ class Stage(params.StageParams):
 
         return isinstance(self.deps[0], dependency.RepoDependency)
 
+    @property
+    def is_checkpoint(self):
+        """
+        A stage containing checkpoint outs is always considered as changed
+        since the checkpoint out is a circular dependency.
+        """
+        return any(out.checkpoint for out in self.outs)
+
     def changed_deps(self):
         if self.frozen:
             return False
@@ -223,7 +249,7 @@ class Stage(params.StageParams):
             )
             return True
 
-        if self.always_changed:
+        if self.always_changed or self.is_checkpoint:
             return True
 
         return self._changed_deps()
@@ -278,7 +304,7 @@ class Stage(params.StageParams):
     def remove_outs(self, ignore_remove=False, force=False):
         """Used mainly for `dvc remove --outs` and :func:`Stage.reproduce`."""
         for out in self.outs:
-            if out.persist and not force:
+            if (out.persist or out.checkpoint) and not force:
                 out.unprotect()
                 continue
 
@@ -380,9 +406,9 @@ class Stage(params.StageParams):
         logger.debug(f"Computed {self} md5: '{m}'")
         return m
 
-    def save(self):
+    def save(self, allow_missing=False):
         self.save_deps()
-        self.save_outs()
+        self.save_outs(allow_missing=allow_missing)
         self.md5 = self.compute_md5()
 
         self.repo.stage_cache.save(self)
@@ -391,9 +417,13 @@ class Stage(params.StageParams):
         for dep in self.deps:
             dep.save()
 
-    def save_outs(self):
+    def save_outs(self, allow_missing=False):
         for out in self.outs:
-            out.save()
+            try:
+                out.save()
+            except OutputDoesNotExistError:
+                if not (allow_missing or out.checkpoint):
+                    raise
 
     def ignore_outs(self):
         for out in self.outs:
@@ -416,19 +446,35 @@ class Stage(params.StageParams):
         )
 
     @rwlocked(write=["outs"])
-    def commit(self):
+    def commit(self, allow_missing=False):
+        link_failures = []
         for out in self.outs:
-            out.commit()
+            try:
+                out.commit()
+            except OutputDoesNotExistError:
+                if not (allow_missing or out.checkpoint):
+                    raise
+            except CacheLinkError:
+                link_failures.append(out.path_info)
+        if link_failures:
+            raise CacheLinkError(link_failures)
 
     @rwlocked(read=["deps"], write=["outs"])
-    def run(self, dry=False, no_commit=False, force=False, run_cache=True):
+    def run(
+        self,
+        dry=False,
+        no_commit=False,
+        force=False,
+        allow_missing=False,
+        **kwargs,
+    ):
         if (self.cmd or self.is_import) and not self.frozen and not dry:
             self.remove_outs(ignore_remove=False, force=False)
 
         if not self.frozen and self.is_import:
             sync_import(self, dry, force)
         elif not self.frozen and self.cmd:
-            run_stage(self, dry, force, run_cache)
+            run_stage(self, dry, force, **kwargs)
         else:
             args = (
                 ("outputs", "frozen ") if self.frozen else ("data sources", "")
@@ -438,9 +484,9 @@ class Stage(params.StageParams):
                 check_missing_outputs(self)
 
         if not dry:
-            self.save()
+            self.save(allow_missing=allow_missing)
             if not no_commit:
-                self.commit()
+                self.commit(allow_missing=allow_missing)
 
     def filter_outs(self, path_info):
         def _func(o):
@@ -451,6 +497,7 @@ class Stage(params.StageParams):
     @rwlocked(write=["outs"])
     def checkout(self, **kwargs):
         stats = defaultdict(list)
+        kwargs["allow_missing"] = self.is_checkpoint
         for out in self.filter_outs(kwargs.get("filter_info")):
             key, outs = self._checkout(out, **kwargs)
             if key:
@@ -501,7 +548,7 @@ class Stage(params.StageParams):
             ret.append({"changed outs": outs_status})
 
     def _status_always_changed(self, ret):
-        if self.is_callback or self.always_changed:
+        if self.is_callback or self.always_changed or self.is_checkpoint:
             ret.append("always changed")
 
     def _status_stage(self, ret):

dvc/stage/cache.py

@@ -1,15 +1,16 @@
 import logging
 import os
+import tempfile
 from contextlib import contextmanager
 
-from funcy import first
+from funcy import cached_property, first
 from voluptuous import Invalid
 
-from dvc.cache.local import _log_exceptions
 from dvc.exceptions import DvcException
+from dvc.path_info import PathInfo
+from dvc.remote.base import _log_exceptions
 from dvc.schema import COMPILED_LOCK_FILE_STAGE_SCHEMA
 from dvc.utils import dict_sha256, relpath
-from dvc.utils.fs import makedirs
 from dvc.utils.serialize import YAMLFileCorruptedError, dump_yaml, load_yaml
 
 from .loader import StageLoader
@@ -24,9 +25,13 @@ class RunCacheNotFoundError(DvcException):
 
 
 def _get_cache_hash(cache, key=False):
+    from dvc.hash_info import HashInfo
+
     if key:
         cache["outs"] = [out["path"] for out in cache.get("outs", [])]
-    return dict_sha256(cache)
+    return dict_sha256(
+        cache, exclude=[HashInfo.PARAM_SIZE, HashInfo.PARAM_NFILES]
+    )
 
 
 def _get_stage_hash(stage):
@@ -47,7 +52,14 @@ def _get_stage_hash(stage):
 class StageCache:
     def __init__(self, repo):
         self.repo = repo
-        self.cache_dir = os.path.join(repo.cache.local.cache_dir, "runs")
+
+    @cached_property
+    def cache_dir(self):
+        return os.path.join(self.repo.cache.local.cache_dir, "runs")
+
+    @property
+    def tree(self):
+        return self.repo.cache.local.tree
 
     def _get_cache_dir(self, key):
         return os.path.join(self.cache_dir, key[:2], key)
@@ -124,6 +136,9 @@ class StageCache:
                     yield out
 
     def save(self, stage):
+        if stage.is_callback or stage.always_changed:
+            return
+
         cache_key = _get_stage_hash(stage)
         if not cache_key:
             return
@@ -143,38 +158,45 @@ class StageCache:
         # sanity check
         COMPILED_LOCK_FILE_STAGE_SCHEMA(cache)
 
-        path = self._get_cache_path(cache_key, cache_value)
-        dpath = os.path.dirname(path)
-        makedirs(dpath, exist_ok=True)
-        dump_yaml(path, cache)
+        path = PathInfo(self._get_cache_path(cache_key, cache_value))
+        self.tree.makedirs(path.parent)
+        tmp = tempfile.NamedTemporaryFile(delete=False, dir=path.parent).name
+        assert os.path.exists(path.parent)
+        assert os.path.isdir(path.parent)
+        dump_yaml(tmp, cache)
+        self.tree.move(PathInfo(tmp), path)
 
-    def _restore(self, stage):
-        stage.save_deps()
-        cache = self._load(stage)
-        if not cache:
-            raise RunCacheNotFoundError(stage)
-
-        StageLoader.fill_from_lock(stage, cache)
-        for out in self._uncached_outs(stage, cache):
-            out.checkout()
-
-        if not stage.outs_cached():
-            raise RunCacheNotFoundError(stage)
-
-    def restore(self, stage, run_cache=True):
+    def restore(self, stage, run_cache=True, pull=False):
         if stage.is_callback or stage.always_changed:
             raise RunCacheNotFoundError(stage)
 
-        if not stage.already_cached():
+        if (
+            not stage.changed_stage()
+            and stage.deps_cached()
+            and all(bool(out.hash_info) for out in stage.outs)
+        ):
+            cache = to_single_stage_lockfile(stage)
+        else:
             if not run_cache:  # backward compatibility
                 raise RunCacheNotFoundError(stage)
-            self._restore(stage)
+            stage.save_deps()
+            cache = self._load(stage)
+            if not cache:
+                raise RunCacheNotFoundError(stage)
+
+        cached_stage = self._create_stage(cache, wdir=stage.wdir)
+
+        if pull:
+            self.repo.cloud.pull(cached_stage.get_used_cache())
+
+        if not cached_stage.outs_cached():
+            raise RunCacheNotFoundError(stage)
 
         logger.info(
             "Stage '%s' is cached - skipping run, checking out outputs",
             stage.addressing,
         )
-        stage.checkout()
+        cached_stage.checkout()
 
     @staticmethod
     def _transfer(func, from_remote, to_remote):

dvc/stage/decorators.py

@@ -50,3 +50,19 @@ def unlocked_repo(f):
         return ret
 
     return wrapper
+
+
+def relock_repo(f):
+    @wraps(f)
+    def wrapper(stage, *args, **kwargs):
+        stage.repo.lock.lock()
+        stage.repo.state.load()
+        try:
+            ret = f(stage, *args, **kwargs)
+        finally:
+            stage.repo.state.dump()
+            stage.repo.lock.unlock()
+            stage.repo._reset()  # pylint: disable=protected-access
+        return ret
+
+    return wrapper

dvc/stage/loader.py

@@ -3,10 +3,12 @@ from collections.abc import Mapping
 from copy import deepcopy
 from itertools import chain
 
-from funcy import get_in, lcat, project
+from funcy import cached_property, get_in, lcat, log_durations, project
 
 from dvc import dependency, output
 from dvc.hash_info import HashInfo
+from dvc.parsing import JOIN, DataResolver
+from dvc.path_info import PathInfo
 
 from . import PipelineStage, Stage, loads_from
 from .exceptions import StageNameUnspecified, StageNotFound
@@ -17,11 +19,26 @@ logger = logging.getLogger(__name__)
 
 
 class StageLoader(Mapping):
-    def __init__(self, dvcfile, stages_data, lockfile_data=None):
+    def __init__(self, dvcfile, data, lockfile_data=None):
         self.dvcfile = dvcfile
-        self.stages_data = stages_data or {}
+        self.data = data or {}
+        self.stages_data = self.data.get("stages", {})
+        self.repo = self.dvcfile.repo
+        self._enable_parametrization = self.repo.config["feature"][
+            "parametrization"
+        ]
         self.lockfile_data = lockfile_data or {}
 
+    @cached_property
+    def resolved_data(self):
+        data = self.data
+        if self._enable_parametrization:
+            wdir = PathInfo(self.dvcfile.path).parent
+            with log_durations(logger.debug, "resolving values"):
+                resolver = DataResolver(self.repo, wdir, data)
+                data = resolver.resolve()
+        return data.get("stages", {})
+
     @staticmethod
     def fill_from_lock(stage, lock_data=None):
         """Fill values for params, checksums for outs and deps from lock."""
@@ -58,6 +75,7 @@ class StageLoader(Mapping):
         )
         stage = loads_from(PipelineStage, dvcfile.repo, path, wdir, stage_data)
         stage.name = name
+        stage.desc = stage_data.get(Stage.PARAM_DESC)
 
         deps = project(stage_data, [stage.PARAM_DEPS, stage.PARAM_PARAMS])
         fill_stage_dependencies(stage, **deps)
@@ -89,21 +107,31 @@ class StageLoader(Mapping):
                 "No lock entry found for '%s:%s'", self.dvcfile.relpath, name,
             )
 
-        return self.load_stage(
+        stage = self.load_stage(
             self.dvcfile,
             name,
-            self.stages_data[name],
+            self.resolved_data[name],
             self.lockfile_data.get(name, {}),
         )
 
+        group, *keys = name.rsplit(JOIN, maxsplit=1)
+        if group and keys and name not in self.stages_data:
+            stage.raw_data.generated_from = group
+
+        stage.raw_data.parametrized = (
+            self.stages_data.get(name, {}) != self.resolved_data[name]
+        )
+
+        return stage
+
     def __iter__(self):
-        return iter(self.stages_data)
+        return iter(self.resolved_data)
 
     def __len__(self):
-        return len(self.stages_data)
+        return len(self.resolved_data)
 
     def __contains__(self, name):
-        return name in self.stages_data
+        return name in self.resolved_data
 
 
 class SingleStageLoader(Mapping):

dvc/stage/params.py

@@ -11,3 +11,4 @@ class StageParams:
     PARAM_PARAMS = "params"
     PARAM_METRICS = "metrics"
     PARAM_PLOTS = "plots"
+    PARAM_DESC = "desc"