Update packages.yaml

* experiments: account for cloning detached-head parent repo

* add test for detached-head parent

.github/release-drafter.yml

@@ -1,5 +1,48 @@
 # Config for https://github.com/apps/release-drafter
-branches:
-  - master
+name-template: "$RESOLVED_VERSION 🦉 "
+tag-template: "$RESOLVED_VERSION"
+categories:
+  - title: "🚀 New Features and Enhancements "
+    labels:
+      - "feature"
+      - "enhancement"
+  - title: "💣 Breaking Changes "
+    labels:
+      - "breaking-change"
+  - title: "🏇 Optimizations "
+    labels:
+      - "optimize"
+      - "performance"
+  - title: "🐛 Bug Fixes "
+    labels:
+      - "bugfix"
+      - "bug"
+  - title: "🔨 Maintenance "
+    labels:
+      - "maintenance"
+      - "refactoring"
+      - "chore"
+      - "build"
+      - "ci"
+change-template: "- $TITLE (#$NUMBER) @$AUTHOR"
+version-resolver:
+  major:
+    labels:
+      - "major"
+  minor:
+    labels:
+      - "minor"
+  patch:
+    labels:
+      - "patch"
+  default: patch
+exclude-labels:
+  - "skip-changelog"
 template: |
+  > Refer to https://dvc.org/doc/install for installation instructions.
+
+  ## Changes
+
   $CHANGES
+
+  Thanks again to $CONTRIBUTORS for the contributions! 🎉

.github/workflows/check-patch.yaml

@@ -1,19 +0,0 @@
-name: Lint Python
-
-on: [push, pull_request]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.8
-    - name: Install requirements
-      run: bash ./scripts/ci/install.sh
-    - name: Check Patch
-      run: ./scripts/ci/check_patch.sh

.github/workflows/packages.yaml

@@ -0,0 +1,134 @@
+name: Build packages
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+  release:
+    types: [released]
+jobs:
+  linux:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Build
+      run: ./scripts/build_posix.sh
+    - name: Publish deb
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_content_type: binary/octet-stream
+    - name: Publish rpm
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_content_type: binary/octet-stream
+  mac:
+    runs-on: macos-10.15
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Build
+      run: ./scripts/build_posix.sh
+    - name: Publish
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_name: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_content_type: binary/octet-stream
+  windows:
+    runs-on: windows-2019
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Build
+      run: scripts\build_windows.cmd
+      shell: cmd
+    - name: Publish
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc-${{ github.event.release.tag_name }}.exe
+        asset_name: dvc-${{ github.event.release.tag_name }}.exe
+        asset_content_type: binary/octet-stream
+  python:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Install
+      run: pip install ".[all,tests]"
+    - name: Build packages
+      run: ./scripts/build_package.sh
+    - name: Publish packages to PyPI
+      if: github.event_name == 'release'
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_TOKEN }}
+  snap:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build the package
+      uses: snapcore/action-build@v1
+      id: snapcraft
+    - name: Test the package
+      run: |
+        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
+        dvc version
+        sudo snap remove dvc
+    - name: Publish to the edge channel
+      uses: snapcore/action-publish@v1
+      if: github.event_name == 'push'
+      with:
+        store_login: ${{ secrets.STORE_LOGIN }}
+        snap: ${{ steps.snapcraft.outputs.snap }}
+        release: edge
+    - name: Publish to the stable channel
+      uses: snapcore/action-publish@v1
+      if: github.event_name == 'release'
+      with:
+        store_login: ${{ secrets.STORE_LOGIN }}
+        snap: ${{ steps.snapcraft.outputs.snap }}
+        release: stable

.github/workflows/tests.yaml

@@ -0,0 +1,57 @@
+name: Tests build
+
+on: [push, pull_request]
+
+env:
+  DVC_TEST: "true"
+  HOMEBREW_NO_AUTO_UPDATE: 1
+  SHELL: /bin/bash
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: Install requirements
+      run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
+    - name: Check README
+      run: python setup.py checkdocs
+    - name: Check formatting
+      run: pre-commit run --all-files
+  tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, windows-2019, macos-10.15]
+        pyv: [3.6, 3.7, 3.8]
+        exclude:
+          - os: windows-2019
+            pyv: 3.8
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python
+      uses: actions/[email protected]
+      with:
+        python-version: ${{ matrix.pyv }}
+    - name: install
+      run: |
+        pip install ".[all,tests]"
+    - name: setup git
+      run: |
+        git config --global user.email "[email protected]"
+        git config --global user.name "DVC Tester"
+    - name: run tests
+      run: python -m tests --cov-report=xml
+    - name: upload coverage report
+      if: github.event.name == 'push'
+      uses: codecov/[email protected]
+      with:
+        file: ./coverage.xml
+        fail_ci_if_error: true
+
+

.gitignore

@@ -1,12 +1,11 @@
 neatlynx.conf
 neatlynx/__pycache__
 .idea
-cache
+/cache
 *.pyc
 .env/
 .env2.7/
 
-cache
 .dvc.conf.lock
 .DS_Store
 build/

.pre-commit-config.yaml

@@ -4,15 +4,11 @@ repos:
         language_version: python3
     repo: https://github.com/ambv/black
     rev: 19.10b0
-  - repo: https://github.com/asottile/seed-isort-config
-    rev: v2.1.1
-    hooks:
-      - id: seed-isort-config
   - hooks:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 4.3.21
+    rev: 5.4.2
   - hooks:
       - id: flake8
         language_version: python3
@@ -22,7 +18,7 @@ repos:
           - flake8-debugger
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
-    rev: 3.7.9
+    rev: 3.8.3
   - repo: local
     hooks:
     - id: pylint

.restyled.yaml

@@ -10,7 +10,7 @@ restylers:
     interpreters:
     - python
   - name: isort
-    image: restyled/restyler-isort:v4.3.21
+    image: restyled/restyler-isort:v5.4.2
     command:
     - isort
     arguments: []

.travis.yml

@@ -1,162 +0,0 @@
-cache:
-  directories:
-    - $HOME/.cache/pip
-    - $HOME/.pyenv
-dist: xenial
-env:
-  global:
-    - HOMEBREW_NO_AUTO_UPDATE=1
-    - PYENV_ROOT="$HOME/.pyenv"
-    - PATH="$PYENV_ROOT/bin:$PATH"
-    - PIP_CACHE_DIR="$HOME/.cache/pip"  # unify pip cache location for all platforms
-stages:
-  - check
-  - test
-  - build
-# Travis doesn't support testing python on Mac yet, so we need
-# to workaround it by installing it directly with brew.
-#
-# Note that we specify language as ruby-2.4.3 to help us satisfy
-# fpm dependencies, as travis runs el capitan with an old ruby
-# version, that causes fpm to crash.
-#
-# Each job will use the previous job's `stage` tag if not specified.
-jobs:
-  include:
-  # patch formatting checks
-  - name: "Check patch formatting"
-    stage: check
-    os: linux
-    language: python
-    python: 3.7
-    before_install: bash ./scripts/ci/install.sh
-    install:
-    script: ./scripts/ci/check_patch.sh
-  # test jobs
-  - name: "3.7 on Windows"
-    stage: test
-    os: windows           # Windows 10.0.17134 N/A Build 17134
-    language: shell       # 'language: python' is an error on Travis CI Windows
-    env:
-      - PYTHON_VERSION=3.7.5
-      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
-  - name: "3.8 on Windows"
-    os: windows           # Windows 10.0.17134 N/A Build 17134
-    language: shell       # 'language: python' is an error on Travis CI Windows
-    env:
-      - PYTHON_VERSION=3.8.0
-      - PATH=/c/Python38:/c/Python38/Scripts:$PATH
-  - name: "3.6 on Linux"
-    language: python
-    python: 3.6
-  - name: "3.7 on Linux"
-    language: python
-    python: 3.7
-  - name: "3.8 on Linux"
-    language: python
-    python: 3.8
-  - name: "3.7 on Mac"
-    os: osx
-    osx_image: xcode11.3
-  # build jobs
-  - name: "Mac pkg"
-    stage: build
-    os: osx
-    osx_image: xcode9.4
-    language: ruby
-    rvm: 2.4.3
-    install:
-    script: ./scripts/build_posix.sh
-  - name: "Windows pkg"
-    os: windows
-    language: shell
-    install:
-    script: ./scripts/build_windows.cmd
-    env:
-      - PYTHON_VERSION=3.7.5
-      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
-  - name: "Linux pkgs"
-    language: python
-    python: 3.7
-    before_install:
-    install:
-    script: ./scripts/build_posix.sh
-  - name: "PyPI pkgs"
-    language: python
-    python: 3.8
-    script: ./scripts/build_package.sh
-  - name: Snapcraft snap
-    language: python
-    python: 3.8
-    addons:
-     snaps:
-     - name: snapcraft
-       channel: stable
-       confinement: classic
-     - name: review-tools
-       channel: stable
-     - name: lxd
-       channel: stable
-    env:
-    - SNAPCRAFT_IMAGE_INFO: |
-        '{"build_url": "$TRAVIS_JOB_WEB_URL"}'
-    - SNAPCRAFT_BUILD_ENVIRONMENT: lxd
-    - SNAPCRAFT_BUILD_INFO: 1  # https://snapcraft.io/blog/introducing-developer-notifications-for-snap-security-updates
-    install:
-    - sudo usermod --append --groups lxd $USER
-    - sudo /snap/bin/lxd.migrate -yes
-    - sudo /snap/bin/lxd waitready
-    - sudo /snap/bin/lxd init --auto
-    script:
-    - ./scripts/build_snap.sh
-    after_failure:
-    - sudo journalctl -u snapd
-
-before_install:
-  - bash ./scripts/ci/before_install.sh
-  - source env.sh
-  - bash ./scripts/ci/check_python.sh
-install:
-  - bash ./scripts/ci/install.sh
-script:
-  - bash ./scripts/ci/script.sh
-deploy:
-  - provider: releases
-    api_key:
-      secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=
-    file_glob: true
-    file:
-      - dvc*.rpm
-      - dvc*.deb
-      - dvc*.pkg
-      - dvc*.exe
-    skip_cleanup: true
-    on:
-      condition: "$(./scripts/ci/deploy_condition.sh dvc*.rpm dvc*.deb dvc*.pkg dvc*.exe)"
-      tags: true
-      repo: iterative/dvc
-      stage: build
-  - provider: pypi
-    user:
-      secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
-    password:
-      secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
-    skip_existing: true
-    file_glob: true
-    file:
-      - dist/dvc*.whl
-      - dist/dvc*.tar.gz
-    skip_cleanup: true
-    on:
-      condition: "$(./scripts/ci/deploy_condition.sh dist/dvc*.whl dist/dvc*.tar.gz)"
-      tags: true
-      repo: iterative/dvc
-      stage: build
-  - provider: script
-    skip_cleanup: true
-    script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out"
-    on:
-      all_branches: true
-      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL"
-      repo: iterative/dvc
-      stage: build

dvc/api.py

@@ -2,35 +2,41 @@ import os
 from contextlib import _GeneratorContextManager as GCM
 from contextlib import contextmanager
 
-from dvc.exceptions import DvcException, NotDvcRepoError
+from funcy import reraise
+
+from dvc.exceptions import (
+    NotDvcRepoError,
+    OutputNotFoundError,
+    PathMissingError,
+)
 from dvc.external_repo import external_repo
+from dvc.path_info import PathInfo
 from dvc.repo import Repo
 
 
-class UrlNotDvcRepoError(DvcException):
-    """Thrown if the given URL is not a DVC repository."""
-
-    def __init__(self, url):
-        super().__init__(f"'{url}' is not a DVC repository.")
-
-
 def get_url(path, repo=None, rev=None, remote=None):
     """
     Returns the URL to the storage location of a data file or directory tracked
     in a DVC repo. For Git repos, HEAD is used unless a rev argument is
     supplied. The default remote is tried unless a remote argument is supplied.
 
-    Raises UrlNotDvcRepoError if repo is not a DVC project.
+    Raises OutputNotFoundError if the file is not a dvc-tracked file.
 
     NOTE: This function does not check for the actual existence of the file or
     directory in the remote storage.
     """
     with _make_repo(repo, rev=rev) as _repo:
-        if not isinstance(_repo, Repo):
-            raise UrlNotDvcRepoError(_repo.url)  # pylint: disable=no-member
-        out = _repo.find_out_by_relpath(path)
-        remote_obj = _repo.cloud.get_remote(remote)
-        return str(remote_obj.tree.hash_to_path_info(out.checksum))
+        path_info = PathInfo(_repo.root_dir) / path
+        with reraise(FileNotFoundError, PathMissingError(path, repo)):
+            metadata = _repo.repo_tree.metadata(path_info)
+
+        if not metadata.is_dvc:
+            raise OutputNotFoundError(path, repo)
+
+        cloud = metadata.repo.cloud
+        with _repo.state:
+            hash_info = _repo.repo_tree.get_hash(path_info)
+        return cloud.get_url_for(remote, checksum=hash_info.value)
 
 
 def open(  # noqa, pylint: disable=redefined-builtin
@@ -97,7 +103,7 @@ def _make_repo(repo_url=None, rev=None):
     repo_url = repo_url or os.getcwd()
     if rev is None and os.path.exists(repo_url):
         try:
-            yield Repo(repo_url)
+            yield Repo(repo_url, subrepos=True)
             return
         except NotDvcRepoError:
             pass  # fallthrough to external_repo

dvc/cache/__init__.py

@@ -63,7 +63,12 @@ class Cache:
         if local:
             settings = {"name": local}
         else:
-            settings = {**config, "url": config["dir"]}
+            from ..config import LOCAL_COMMON
+
+            settings = {"url": config["dir"]}
+            for opt in LOCAL_COMMON.keys():
+                if opt in config:
+                    settings[str(opt)] = config.get(opt)
 
         tree = get_cloud_tree(repo, **settings)
         self.local = LocalCache(tree)

dvc/cache/base.py

@@ -1,11 +1,18 @@
 import json
 import logging
 from copy import copy
+from operator import itemgetter
 
 from shortuuid import uuid
 
 import dvc.prompt as prompt
-from dvc.exceptions import CheckoutError, ConfirmRemoveError, DvcException
+from dvc.exceptions import (
+    CheckoutError,
+    ConfirmRemoveError,
+    DvcException,
+    MergeError,
+)
+from dvc.hash_info import HashInfo
 from dvc.path_info import WindowsPathInfo
 from dvc.progress import Tqdm
 from dvc.remote.slow_link_detection import slow_link_guard
@@ -51,29 +58,29 @@ class CloudCache:
         self.cache_type_confirmed = False
         self._dir_info = {}
 
-    def get_dir_cache(self, hash_):
-        assert hash_
+    def get_dir_cache(self, hash_info):
+        assert hash_info
 
-        dir_info = self._dir_info.get(hash_)
+        dir_info = self._dir_info.get(hash_info.value)
         if dir_info:
             return dir_info
 
         try:
-            dir_info = self.load_dir_cache(hash_)
+            dir_info = self.load_dir_cache(hash_info)
         except DirCacheError:
             dir_info = []
 
-        self._dir_info[hash_] = dir_info
+        self._dir_info[hash_info.value] = dir_info
         return dir_info
 
-    def load_dir_cache(self, hash_):
-        path_info = self.tree.hash_to_path_info(hash_)
+    def load_dir_cache(self, hash_info):
+        path_info = self.tree.hash_to_path_info(hash_info.value)
 
         try:
             with self.tree.open(path_info, "r") as fobj:
                 d = json.load(fobj)
         except (ValueError, FileNotFoundError) as exc:
-            raise DirCacheError(hash_) from exc
+            raise DirCacheError(hash_info) from exc
 
         if not isinstance(d, list):
             logger.error(
@@ -118,20 +125,21 @@ class CloudCache:
             logger.debug("'%s' doesn't exist.", path_info)
             return True
 
-        hash_ = hash_info.get(self.tree.PARAM_CHECKSUM)
-        if hash_ is None:
+        if not hash_info:
             logger.debug("hash value for '%s' is missing.", path_info)
             return True
 
-        if self.changed_cache(hash_):
-            logger.debug("cache for '%s'('%s') has changed.", path_info, hash_)
+        if self.changed_cache(hash_info):
+            logger.debug(
+                "cache for '%s'('%s') has changed.", path_info, hash_info
+            )
             return True
 
         actual = self.tree.get_hash(path_info)
-        if hash_ != actual:
+        if hash_info != actual:
             logger.debug(
                 "hash value '%s' for '%s' has changed (actual '%s').",
-                hash_,
+                hash_info,
                 actual,
                 path_info,
             )
@@ -188,12 +196,12 @@ class CloudCache:
             "Created '%s': %s -> %s", self.cache_types[0], from_info, to_info,
         )
 
-    def _save_file(self, path_info, tree, hash_, save_link=True, **kwargs):
-        assert hash_
+    def _save_file(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        assert hash_info
 
-        cache_info = self.tree.hash_to_path_info(hash_)
+        cache_info = self.tree.hash_to_path_info(hash_info.value)
         if tree == self.tree:
-            if self.changed_cache(hash_):
+            if self.changed_cache(hash_info):
                 self.tree.move(path_info, cache_info, mode=self.CACHE_MODE)
                 self.link(cache_info, path_info)
             elif self.tree.iscopy(path_info) and self._cache_is_copy(
@@ -210,9 +218,9 @@ class CloudCache:
             # we need to update path and cache, since in case of reflink,
             # or copy cache type moving original file results in updates on
             # next executed command, which causes md5 recalculation
-            self.tree.state.save(path_info, hash_)
+            self.tree.state.save(path_info, hash_info.value)
         else:
-            if self.changed_cache(hash_):
+            if self.changed_cache(hash_info):
                 with tree.open(path_info, mode="rb") as fobj:
                     # if tree has fetch enabled, DVC out will be fetched on
                     # open and we do not need to read/copy any data
@@ -224,8 +232,7 @@ class CloudCache:
                 if callback:
                     callback(1)
 
-        self.tree.state.save(cache_info, hash_)
-        return {self.tree.PARAM_CHECKSUM: hash_}
+        self.tree.state.save(cache_info, hash_info.value)
 
     def _cache_is_copy(self, path_info):
         """Checks whether cache uses copies."""
@@ -249,13 +256,58 @@ class CloudCache:
         self.cache_type_confirmed = True
         return self.cache_types[0] == "copy"
 
-    def _save_dir(self, path_info, tree, hash_, save_link=True, **kwargs):
-        dir_info = self.get_dir_cache(hash_)
+    def _get_dir_info_hash(self, dir_info):
+        import tempfile
+
+        from dvc.path_info import PathInfo
+        from dvc.utils import tmp_fname
+
+        # Sorting the list by path to ensure reproducibility
+        dir_info = sorted(dir_info, key=itemgetter(self.tree.PARAM_RELPATH))
+
+        tmp = tempfile.NamedTemporaryFile(delete=False).name
+        with open(tmp, "w+") as fobj:
+            json.dump(dir_info, fobj, sort_keys=True)
+
+        from_info = PathInfo(tmp)
+        to_info = self.tree.path_info / tmp_fname("")
+        self.tree.upload(from_info, to_info, no_progress_bar=True)
+
+        hash_info = self.tree.get_file_hash(to_info)
+        hash_info.value += self.tree.CHECKSUM_DIR_SUFFIX
+        hash_info.dir_info = dir_info
+
+        return hash_info, to_info
+
+    def save_dir_info(self, dir_info, hash_info=None):
+        if (
+            hash_info
+            and hash_info.name == self.tree.PARAM_CHECKSUM
+            and not self.changed_cache_file(hash_info)
+        ):
+            return hash_info
+
+        hi, tmp_info = self._get_dir_info_hash(dir_info)
+        new_info = self.tree.hash_to_path_info(hi.value)
+        if self.changed_cache_file(hi):
+            self.tree.makedirs(new_info.parent)
+            self.tree.move(tmp_info, new_info, mode=self.CACHE_MODE)
+
+        self.tree.state.save(new_info, hi.value)
+
+        return hi
+
+    def _save_dir(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        if not hash_info.dir_info:
+            hash_info.dir_info = tree.cache.get_dir_cache(hash_info)
+        hi = self.save_dir_info(hash_info.dir_info, hash_info)
         for entry in Tqdm(
-            dir_info, desc="Saving " + path_info.name, unit="file"
+            hi.dir_info, desc="Saving " + path_info.name, unit="file"
         ):
             entry_info = path_info / entry[self.tree.PARAM_RELPATH]
-            entry_hash = entry[self.tree.PARAM_CHECKSUM]
+            entry_hash = HashInfo(
+                self.tree.PARAM_CHECKSUM, entry[self.tree.PARAM_CHECKSUM]
+            )
             self._save_file(
                 entry_info, tree, entry_hash, save_link=False, **kwargs
             )
@@ -263,11 +315,10 @@ class CloudCache:
         if save_link:
             self.tree.state.save_link(path_info)
         if self.tree.exists(path_info):
-            self.tree.state.save(path_info, hash_)
+            self.tree.state.save(path_info, hi.value)
 
-        cache_info = self.tree.hash_to_path_info(hash_)
-        self.tree.state.save(cache_info, hash_)
-        return {self.tree.PARAM_CHECKSUM: hash_}
+        cache_info = self.tree.hash_to_path_info(hi.value)
+        self.tree.state.save(cache_info, hi.value)
 
     def save(self, path_info, tree, hash_info, save_link=True, **kwargs):
         if path_info.scheme != self.tree.scheme:
@@ -276,25 +327,23 @@ class CloudCache:
                 self.tree.scheme,
             )
 
-        if not hash_info:
-            hash_info = self.tree.save_info(path_info, tree=tree, **kwargs)
-        hash_ = hash_info[self.tree.PARAM_CHECKSUM]
-        return self._save(path_info, tree, hash_, save_link, **kwargs)
+        self._save(path_info, tree, hash_info, save_link, **kwargs)
 
-    def _save(self, path_info, tree, hash_, save_link=True, **kwargs):
-        to_info = self.tree.hash_to_path_info(hash_)
+    def _save(self, path_info, tree, hash_info, save_link=True, **kwargs):
+        to_info = self.tree.hash_to_path_info(hash_info.value)
         logger.debug("Saving '%s' to '%s'.", path_info, to_info)
 
         if tree.isdir(path_info):
-            return self._save_dir(path_info, tree, hash_, save_link, **kwargs)
-        return self._save_file(path_info, tree, hash_, save_link, **kwargs)
+            self._save_dir(path_info, tree, hash_info, save_link, **kwargs)
+        else:
+            self._save_file(path_info, tree, hash_info, save_link, **kwargs)
 
     # Override to return path as a string instead of PathInfo for clouds
     # which support string paths (see local)
     def hash_to_path(self, hash_):
         return self.tree.hash_to_path_info(hash_)
 
-    def changed_cache_file(self, hash_):
+    def changed_cache_file(self, hash_info):
         """Compare the given hash with the (corresponding) actual one.
 
         - Use `State` as a cache for computed hashes
@@ -307,7 +356,7 @@ class CloudCache:
         - Remove the file from cache if it doesn't match the actual hash
         """
         # Prefer string path over PathInfo when possible due to performance
-        cache_info = self.hash_to_path(hash_)
+        cache_info = self.hash_to_path(hash_info.value)
         if self.tree.is_protected(cache_info):
             logger.debug(
                 "Assuming '%s' is unchanged since it is read-only", cache_info
@@ -317,13 +366,16 @@ class CloudCache:
         actual = self.tree.get_hash(cache_info)
 
         logger.debug(
-            "cache '%s' expected '%s' actual '%s'", cache_info, hash_, actual,
+            "cache '%s' expected '%s' actual '%s'",
+            cache_info,
+            hash_info,
+            actual,
         )
 
-        if not hash_ or not actual:
+        if not hash_info or not actual:
             return True
 
-        if actual.split(".")[0] == hash_.split(".")[0]:
+        if actual.value.split(".")[0] == hash_info.value.split(".")[0]:
             # making cache file read-only so we don't need to check it
             # next time
             self.tree.protect(cache_info)
@@ -335,12 +387,14 @@ class CloudCache:
 
         return True
 
-    def _changed_dir_cache(self, hash_, path_info=None, filter_info=None):
-        if self.changed_cache_file(hash_):
+    def _changed_dir_cache(self, hash_info, path_info=None, filter_info=None):
+        if self.changed_cache_file(hash_info):
             return True
 
-        for entry in self.get_dir_cache(hash_):
-            entry_hash = entry[self.tree.PARAM_CHECKSUM]
+        for entry in self.get_dir_cache(hash_info):
+            entry_hash = HashInfo(
+                self.tree.PARAM_CHECKSUM, entry[self.tree.PARAM_CHECKSUM]
+            )
 
             if path_info and filter_info:
                 entry_info = path_info / entry[self.tree.PARAM_RELPATH]
@@ -352,12 +406,12 @@ class CloudCache:
 
         return False
 
-    def changed_cache(self, hash_, path_info=None, filter_info=None):
-        if self.tree.is_dir_hash(hash_):
+    def changed_cache(self, hash_info, path_info=None, filter_info=None):
+        if hash_info.isdir:
             return self._changed_dir_cache(
-                hash_, path_info=path_info, filter_info=filter_info
+                hash_info, path_info=path_info, filter_info=filter_info
             )
-        return self.changed_cache_file(hash_)
+        return self.changed_cache_file(hash_info)
 
     def already_cached(self, path_info):
         current = self.tree.get_hash(path_info)
@@ -383,11 +437,11 @@ class CloudCache:
         self.tree.remove(path_info)
 
     def _checkout_file(
-        self, path_info, hash_, force, progress_callback=None, relink=False
+        self, path_info, hash_info, force, progress_callback=None, relink=False
     ):
         """The file is changed we need to checkout a new copy"""
         added, modified = True, False
-        cache_info = self.tree.hash_to_path_info(hash_)
+        cache_info = self.tree.hash_to_path_info(hash_info.value)
         if self.tree.exists(path_info):
             logger.debug("data '%s' will be replaced.", path_info)
             self.safe_remove(path_info, force=force)
@@ -395,7 +449,7 @@ class CloudCache:
 
         self.link(cache_info, path_info)
         self.tree.state.save_link(path_info)
-        self.tree.state.save(path_info, hash_)
+        self.tree.state.save(path_info, hash_info.value)
         if progress_callback:
             progress_callback(str(path_info))
 
@@ -404,7 +458,7 @@ class CloudCache:
     def _checkout_dir(
         self,
         path_info,
-        hash_,
+        hash_info,
         force,
         progress_callback=None,
         relink=False,
@@ -417,7 +471,7 @@ class CloudCache:
             added = True
             self.tree.makedirs(path_info)
 
-        dir_info = self.get_dir_cache(hash_)
+        dir_info = self.get_dir_cache(hash_info)
 
         logger.debug("Linking directory '%s'.", path_info)
 
@@ -430,7 +484,7 @@ class CloudCache:
             if filter_info and not entry_info.isin_or_eq(filter_info):
                 continue
 
-            entry_hash_info = {self.tree.PARAM_CHECKSUM: entry_hash}
+            entry_hash_info = HashInfo(self.tree.PARAM_CHECKSUM, entry_hash)
             if relink or self.changed(entry_info, entry_hash_info):
                 modified = True
                 self.safe_remove(entry_info, force=force)
@@ -445,7 +499,7 @@ class CloudCache:
         )
 
         self.tree.state.save_link(path_info)
-        self.tree.state.save(path_info, hash_)
+        self.tree.state.save(path_info, hash_info.value)
 
         # relink is not modified, assume it as nochange
         return added, not added and modified and not relink
@@ -474,10 +528,9 @@ class CloudCache:
         if path_info.scheme not in ["local", self.tree.scheme]:
             raise NotImplementedError
 
-        hash_ = hash_info.get(self.tree.PARAM_CHECKSUM)
         failed = None
         skip = False
-        if not hash_:
+        if not hash_info:
             logger.warning(
                 "No file hash info found for '%s'. " "It won't be created.",
                 path_info,
@@ -490,11 +543,11 @@ class CloudCache:
             skip = True
 
         elif self.changed_cache(
-            hash_, path_info=path_info, filter_info=filter_info
+            hash_info, path_info=path_info, filter_info=filter_info
         ):
             logger.warning(
                 "Cache '%s' not found. File '%s' won't be created.",
-                hash_,
+                hash_info,
                 path_info,
             )
             self.safe_remove(path_info, force=force)
@@ -505,50 +558,137 @@ class CloudCache:
                 progress_callback(
                     str(path_info),
                     self.get_files_number(
-                        self.tree.path_info, hash_, filter_info
+                        self.tree.path_info, hash_info, filter_info
                     ),
                 )
             if failed:
                 raise CheckoutError([failed])
             return
 
-        logger.debug("Checking out '%s' with cache '%s'.", path_info, hash_)
+        logger.debug(
+            "Checking out '%s' with cache '%s'.", path_info, hash_info
+        )
 
         return self._checkout(
-            path_info, hash_, force, progress_callback, relink, filter_info,
+            path_info,
+            hash_info,
+            force,
+            progress_callback,
+            relink,
+            filter_info,
         )
 
     def _checkout(
         self,
         path_info,
-        hash_,
+        hash_info,
         force=False,
         progress_callback=None,
         relink=False,
         filter_info=None,
     ):
-        if not self.tree.is_dir_hash(hash_):
+        if not hash_info.isdir:
             return self._checkout_file(
-                path_info, hash_, force, progress_callback, relink
+                path_info, hash_info, force, progress_callback, relink
             )
 
         return self._checkout_dir(
-            path_info, hash_, force, progress_callback, relink, filter_info
+            path_info, hash_info, force, progress_callback, relink, filter_info
         )
 
-    def get_files_number(self, path_info, hash_, filter_info):
+    def get_files_number(self, path_info, hash_info, filter_info):
         from funcy.py3 import ilen
 
-        if not hash_:
+        if not hash_info:
             return 0
 
-        if not self.tree.is_dir_hash(hash_):
+        if not hash_info.isdir:
             return 1
 
         if not filter_info:
-            return len(self.get_dir_cache(hash_))
+            return len(self.get_dir_cache(hash_info))
 
         return ilen(
             filter_info.isin_or_eq(path_info / entry[self.tree.PARAM_CHECKSUM])
-            for entry in self.get_dir_cache(hash_)
+            for entry in self.get_dir_cache(hash_info)
         )
+
+    def _to_dict(self, dir_info):
+        return {
+            entry[self.tree.PARAM_RELPATH]: entry[self.tree.PARAM_CHECKSUM]
+            for entry in dir_info
+        }
+
+    def _from_dict(self, dir_dict):
+        return [
+            {
+                self.tree.PARAM_RELPATH: relpath,
+                self.tree.PARAM_CHECKSUM: checksum,
+            }
+            for relpath, checksum in dir_dict.items()
+        ]
+
+    @staticmethod
+    def _diff(ancestor, other, allow_removed=False):
+        from dictdiffer import diff
+
+        allowed = ["add"]
+        if allow_removed:
+            allowed.append("remove")
+
+        result = list(diff(ancestor, other))
+        for typ, _, _ in result:
+            if typ not in allowed:
+                raise MergeError(
+                    "unable to auto-merge directories with diff that contains "
+                    f"'{typ}'ed files"
+                )
+        return result
+
+    def _merge_dirs(self, ancestor_info, our_info, their_info):
+        from dictdiffer import patch
+
+        ancestor = self._to_dict(ancestor_info)
+        our = self._to_dict(our_info)
+        their = self._to_dict(their_info)
+
+        our_diff = self._diff(ancestor, our)
+        if not our_diff:
+            return self._from_dict(their)
+
+        their_diff = self._diff(ancestor, their)
+        if not their_diff:
+            return self._from_dict(our)
+
+        # make sure there are no conflicting files
+        self._diff(our, their, allow_removed=True)
+
+        merged = patch(our_diff + their_diff, ancestor, in_place=True)
+
+        # Sorting the list by path to ensure reproducibility
+        return sorted(
+            self._from_dict(merged), key=itemgetter(self.tree.PARAM_RELPATH)
+        )
+
+    def merge(self, ancestor_info, our_info, their_info):
+        assert our_info
+        assert their_info
+
+        if ancestor_info:
+            ancestor = self.get_dir_cache(ancestor_info)
+        else:
+            ancestor = []
+
+        our = self.get_dir_cache(our_info)
+        their = self.get_dir_cache(their_info)
+
+        merged = self._merge_dirs(ancestor, our, their)
+        return self.save_dir_info(merged)
+
+    def get_hash(self, tree, path_info):
+        hash_info = tree.get_hash(path_info)
+        if not hash_info.isdir:
+            assert hash_info.name == self.tree.PARAM_CHECKSUM
+            return hash_info
+
+        return self.save_dir_info(hash_info.dir_info, hash_info)

dvc/cache/local.py

@@ -7,6 +7,7 @@ from functools import partial, wraps
 from funcy import cached_property, concat
 
 from dvc.exceptions import DownloadError, UploadError
+from dvc.hash_info import HashInfo
 from dvc.path_info import PathInfo
 from dvc.progress import Tqdm
 
@@ -85,18 +86,15 @@ class LocalCache(CloudCache):
                 desc="Querying "
                 + ("cache in " + name if name else "local cache"),
             )
-            if not self.changed_cache_file(hash_)
+            if not self.changed_cache_file(
+                HashInfo(self.tree.PARAM_CHECKSUM, hash_)
+            )
         ]
 
     def already_cached(self, path_info):
         assert path_info.scheme in ["", "local"]
 
-        current_md5 = self.tree.get_hash(path_info)
-
-        if not current_md5:
-            return False
-
-        return not self.changed_cache(current_md5)
+        return super().already_cached(path_info)
 
     def _verify_link(self, path_info, link_type):
         if link_type == "hardlink" and self.tree.getsize(path_info) == 0:
@@ -112,6 +110,7 @@ class LocalCache(CloudCache):
         jobs=None,
         show_checksums=False,
         download=False,
+        log_missing=True,
     ):
         # Return flattened dict containing all status info
         dir_status, file_status, _ = self._status(
@@ -120,6 +119,7 @@ class LocalCache(CloudCache):
             jobs=jobs,
             show_checksums=show_checksums,
             download=download,
+            log_missing=log_missing,
         )
         return dict(dir_status, **file_status)
 
@@ -130,6 +130,7 @@ class LocalCache(CloudCache):
         jobs=None,
         show_checksums=False,
         download=False,
+        log_missing=True,
     ):
         """Return a tuple of (dir_status_info, file_status_info, dir_contents).
 
@@ -170,11 +171,20 @@ class LocalCache(CloudCache):
                     )
                 )
         return self._make_status(
-            named_cache, show_checksums, local_exists, remote_exists
+            named_cache,
+            show_checksums,
+            local_exists,
+            remote_exists,
+            log_missing,
         )
 
     def _make_status(
-        self, named_cache, show_checksums, local_exists, remote_exists
+        self,
+        named_cache,
+        show_checksums,
+        local_exists,
+        remote_exists,
+        log_missing,
     ):
         def make_names(hash_, names):
             return {"name": hash_ if show_checksums else " ".join(names)}
@@ -197,7 +207,8 @@ class LocalCache(CloudCache):
         self._fill_statuses(dir_status, local_exists, remote_exists)
         self._fill_statuses(file_status, local_exists, remote_exists)
 
-        self._log_missing_caches(dict(dir_status, **file_status))
+        if log_missing:
+            self._log_missing_caches(dict(dir_status, **file_status))
 
         return dir_status, file_status, dir_contents
 
@@ -255,6 +266,7 @@ class LocalCache(CloudCache):
         path_infos = []
         names = []
         hashes = []
+        missing = []
         for md5, info in Tqdm(
             status_info.items(), desc="Analysing status", unit="file"
         ):
@@ -263,6 +275,8 @@ class LocalCache(CloudCache):
                 path_infos.append(remote.tree.hash_to_path_info(md5))
                 names.append(info["name"])
                 hashes.append(md5)
+            elif info["status"] == STATUS_MISSING:
+                missing.append(md5)
 
         if download:
             to_infos = cache
@@ -271,7 +285,7 @@ class LocalCache(CloudCache):
             to_infos = path_infos
             from_infos = cache
 
-        return from_infos, to_infos, names, hashes
+        return (from_infos, to_infos, names, hashes), missing
 
     def _process(
         self,
@@ -312,8 +326,10 @@ class LocalCache(CloudCache):
             download=download,
         )
 
-        dir_plans = self._get_plans(download, remote, dir_status, status)
-        file_plans = self._get_plans(download, remote, file_status, status)
+        dir_plans, _ = self._get_plans(download, remote, dir_status, status)
+        file_plans, missing_files = self._get_plans(
+            download, remote, file_status, status
+        )
 
         total = len(dir_plans[0]) + len(file_plans[0])
         if total == 0:
@@ -339,19 +355,32 @@ class LocalCache(CloudCache):
                         )
                     dir_futures = {}
                     for from_info, to_info, name, dir_hash in zip(*dir_plans):
-                        wait_futures = {
-                            future
-                            for file_hash, future in file_futures.items()
-                            if file_hash in dir_contents[dir_hash]
-                        }
-                        dir_futures[dir_hash] = executor.submit(
-                            self._dir_upload,
-                            func,
-                            wait_futures,
-                            from_info,
-                            to_info,
-                            name,
-                        )
+                        # if for some reason a file contained in this dir is
+                        # missing both locally and in the remote, we want to
+                        # push whatever file content we have, but should not
+                        # push .dir file
+                        for file_hash in missing_files:
+                            if file_hash in dir_contents[dir_hash]:
+                                logger.debug(
+                                    "directory '%s' contains missing files,"
+                                    "skipping .dir file upload",
+                                    name,
+                                )
+                                break
+                        else:
+                            wait_futures = {
+                                future
+                                for file_hash, future in file_futures.items()
+                                if file_hash in dir_contents[dir_hash]
+                            }
+                            dir_futures[dir_hash] = executor.submit(
+                                self._dir_upload,
+                                func,
+                                wait_futures,
+                                from_info,
+                                to_info,
+                                name,
+                            )
                     fails = sum(
                         future.result()
                         for future in concat(
@@ -404,7 +433,7 @@ class LocalCache(CloudCache):
 
     @index_locked
     def pull(self, named_cache, remote, jobs=None, show_checksums=False):
-        return self._process(
+        ret = self._process(
             named_cache,
             remote,
             jobs=jobs,
@@ -412,6 +441,18 @@ class LocalCache(CloudCache):
             download=True,
         )
 
+        if not remote.tree.verify:
+            for checksum in named_cache.scheme_keys("local"):
+                cache_file = self.tree.hash_to_path_info(checksum)
+                if self.tree.exists(cache_file):
+                    # We can safely save here, as existing corrupted files will
+                    # be removed upon status, while files corrupted during
+                    # download will not be moved from tmp_file
+                    # (see `BaseTree.download()`)
+                    self.tree.state.save(cache_file, checksum)
+
+        return ret
+
     @staticmethod
     def _log_missing_caches(hash_info_dict):
         missing_caches = [

dvc/cli.py

@@ -6,6 +6,7 @@ import sys
 from .command import (
     add,
     cache,
+    check_ignore,
     checkout,
     commit,
     completion,
@@ -79,6 +80,7 @@ COMMANDS = [
     git_hook,
     plots,
     experiments,
+    check_ignore,
 ]
 
 
@@ -135,6 +137,14 @@ def get_parent_parser():
     """
     parent_parser = argparse.ArgumentParser(add_help=False)
 
+    parent_parser.add_argument(
+        "--cprofile",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    parent_parser.add_argument("--cprofile-dump", help=argparse.SUPPRESS)
+
     log_level_group = parent_parser.add_mutually_exclusive_group()
     log_level_group.add_argument(
         "-q", "--quiet", action="count", default=0, help="Be quiet."

dvc/command/check_ignore.py

@@ -0,0 +1,126 @@
+import argparse
+import logging
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
+
+logger = logging.getLogger(__name__)
+
+
+class CmdCheckIgnore(CmdBase):
+    def __init__(self, args):
+        super().__init__(args)
+        self.ignore_filter = self.repo.tree.dvcignore
+
+    def _show_results(self, result):
+        if not result.match and not self.args.non_matching:
+            return
+
+        if self.args.details:
+            patterns = result.patterns
+            if not self.args.all:
+                patterns = patterns[-1:]
+
+            for pattern in patterns:
+                logger.info("{}\t{}".format(pattern, result.file))
+        else:
+            logger.info(result.file)
+
+    def _check_one_file(self, target):
+        result = self.ignore_filter.check_ignore(target)
+        self._show_results(result)
+        if result.match:
+            return 0
+        return 1
+
+    def _interactive_mode(self):
+        ret = 1
+        while True:
+            try:
+                target = input()
+            except (KeyboardInterrupt, EOFError):
+                break
+            if target == "":
+                break
+            if not self._check_one_file(target):
+                ret = 0
+        return ret
+
+    def _normal_mode(self):
+        ret = 1
+        for target in self.args.targets:
+            if not self._check_one_file(target):
+                ret = 0
+        return ret
+
+    def _check_args(self):
+        if not self.args.stdin and not self.args.targets:
+            raise DvcException("`targets` or `--stdin` needed")
+
+        if self.args.stdin and self.args.targets:
+            raise DvcException("cannot have both `targets` and `--stdin`")
+
+        if self.args.non_matching and not self.args.details:
+            raise DvcException(
+                "`--non-matching` is only valid with `--details`"
+            )
+
+        if self.args.all and not self.args.details:
+            raise DvcException("`--all` is only valid with `--details`")
+
+        if self.args.quiet and self.args.details:
+            raise DvcException("cannot use both `--details` and `--quiet`")
+
+    def run(self):
+        self._check_args()
+        if self.args.stdin:
+            return self._interactive_mode()
+        return self._normal_mode()
+
+
+def add_parser(subparsers, parent_parser):
+    ADD_HELP = (
+        "Check whether files or directories are excluded due to `.dvcignore`."
+    )
+
+    parser = subparsers.add_parser(
+        "check-ignore",
+        parents=[parent_parser],
+        description=append_doc_link(ADD_HELP, "check-ignore"),
+        help=ADD_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "-d",
+        "--details",
+        action="store_true",
+        default=False,
+        help="Show the exclude patterns along with each target path.",
+    )
+    parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        default=False,
+        help="Include the target paths which don’t match any pattern "
+        "in the `--details` list.",
+    )
+    parser.add_argument(
+        "-n",
+        "--non-matching",
+        action="store_true",
+        default=False,
+        help="Include the target paths which don’t match any pattern "
+        "in the `--details` list.",
+    )
+    parser.add_argument(
+        "--stdin",
+        action="store_true",
+        default=False,
+        help="Read paths from standard input instead of providing `targets`.",
+    )
+    parser.add_argument(
+        "targets", nargs="*", help="File or directory paths to check",
+    ).complete = completion.FILE
+    parser.set_defaults(func=CmdCheckIgnore)

dvc/command/daemon.py

@@ -9,8 +9,9 @@ class CmdDaemonBase(CmdBaseNoRepo):
 class CmdDaemonUpdater(CmdDaemonBase):
     def run(self):
         import os
-        from dvc.repo import Repo
+
         from dvc.config import Config
+        from dvc.repo import Repo
         from dvc.updater import Updater
 
         root_dir = Repo.find_root()

dvc/command/dag.py

@@ -8,8 +8,8 @@ logger = logging.getLogger(__name__)
 
 
 def _show_ascii(G):
-    from dvc.repo.graph import get_pipelines
     from dvc.dagascii import draw
+    from dvc.repo.graph import get_pipelines
 
     pipelines = get_pipelines(G)
 
@@ -22,6 +22,7 @@ def _show_ascii(G):
 
 def _show_dot(G):
     import io
+
     from networkx.drawing.nx_pydot import write_dot
 
     dot_file = io.StringIO()
@@ -31,6 +32,7 @@ def _show_dot(G):
 
 def _build(G, target=None, full=False):
     import networkx as nx
+
     from dvc.repo.graph import get_pipeline, get_pipelines
 
     if target:

dvc/command/data_sync.py

@@ -97,7 +97,11 @@ def shared_parent_parser():
         "-j",
         "--jobs",
         type=int,
-        help="Number of jobs to run simultaneously.",
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
         metavar="<number>",
     )
     parent_parser.add_argument(

dvc/command/destroy.py

@@ -25,7 +25,7 @@ class CmdDestroy(CmdBase):
                 )
 
             self.repo.destroy()
-        except Exception:  # noqa, pylint: disable=broad-except
+        except DvcException:
             logger.exception("failed to destroy DVC")
             return 1
         return 0

dvc/command/diff.py

@@ -11,30 +11,43 @@ from dvc.exceptions import DvcException
 logger = logging.getLogger(__name__)
 
 
-def _show_md(diff):
+def _digest(checksum):
+    if isinstance(checksum, str):
+        return checksum[0:8]
+    return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
+
+
+def _show_md(diff, show_hash=False, hide_missing=False):
     from dvc.utils.diff import table
 
+    header = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
     rows = []
-    for status in ["added", "deleted", "modified"]:
+    statuses = ["added", "deleted", "modified"]
+    if not hide_missing:
+        statuses.append("not in cache")
+    for status in statuses:
         entries = diff.get(status, [])
         if not entries:
             continue
-        paths = sorted(entry["path"] for entry in entries)
-        for path in paths:
-            rows.append([status, path])
+        for entry in entries:
+            path = entry["path"]
+            if show_hash:
+                check_sum = _digest(entry.get("hash", ""))
+                rows.append([status, check_sum, path])
+            else:
+                rows.append([status, path])
 
-    return table(["Status", "Path"], rows, True)
+    return table(header, rows, True)
 
 
 class CmdDiff(CmdBase):
     @staticmethod
-    def _format(diff):
+    def _format(diff, hide_missing=False):
         """
         Given a diff structure, generate a string of paths separated
         by new lines and grouped together by their state.
 
-        A group's header is colored and its entries are sorted to enhance
-        readability, for example:
+        A group's header is colored to enhance readability, for example:
 
             Added:
                 another_file.txt
@@ -55,21 +68,20 @@ class CmdDiff(CmdBase):
         At the bottom, include a summary with the number of files per state.
         """
 
-        def _digest(checksum):
-            if isinstance(checksum, str):
-                return checksum[0:8]
-            return "{}..{}".format(checksum["old"][0:8], checksum["new"][0:8])
-
         colors = {
             "added": colorama.Fore.GREEN,
             "modified": colorama.Fore.YELLOW,
             "deleted": colorama.Fore.RED,
+            "not in cache": colorama.Fore.YELLOW,
         }
 
         summary = {}
         groups = []
 
-        for state in ["added", "deleted", "modified"]:
+        states = ["added", "deleted", "modified"]
+        if not hide_missing:
+            states.append("not in cache")
+        for state in states:
             summary[state] = 0
             entries = diff[state]
 
@@ -100,28 +112,42 @@ class CmdDiff(CmdBase):
                 )
             )
 
-        groups.append(
+        if not sum(summary.values()):
+            return None
+
+        fmt = (
             "files summary: {added} added, {deleted} deleted,"
-            " {modified} modified".format_map(summary)
+            " {modified} modified"
         )
+        if not hide_missing:
+            fmt += ", {not in cache} not in cache"
+        groups.append(fmt.format_map(summary))
 
         return "\n\n".join(groups)
 
     def run(self):
         try:
             diff = self.repo.diff(self.args.a_rev, self.args.b_rev)
+            show_hash = self.args.show_hash
+            hide_missing = self.args.b_rev or self.args.hide_missing
+            if hide_missing:
+                del diff["not in cache"]
 
-            if not self.args.show_hash:
-                for _, entries in diff.items():
+            for key, entries in diff.items():
+                entries = sorted(entries, key=lambda entry: entry["path"])
+                if not show_hash:
                     for entry in entries:
                         del entry["hash"]
+                diff[key] = entries
 
             if self.args.show_json:
                 logger.info(json.dumps(diff))
             elif self.args.show_md:
-                logger.info(_show_md(diff))
+                logger.info(_show_md(diff, show_hash, hide_missing))
             elif diff:
-                logger.info(self._format(diff))
+                output = self._format(diff, hide_missing)
+                if output:
+                    logger.info(output)
 
         except DvcException:
             logger.exception("failed to get diff")
@@ -170,4 +196,9 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         default=False,
     )
+    diff_parser.add_argument(
+        "--hide-missing",
+        help="Hide missing cache file status.",
+        action="store_true",
+    )
     diff_parser.set_defaults(func=CmdDiff)

dvc/command/experiments.py

@@ -2,26 +2,69 @@ import argparse
 import io
 import logging
 from collections import OrderedDict
+from datetime import date
+from itertools import groupby
+from typing import Iterable, Optional
 
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.command.metrics import DEFAULT_PRECISION
-from dvc.exceptions import DvcException
+from dvc.exceptions import DvcException, InvalidArgumentError
+from dvc.utils.flatten import flatten
 
 logger = logging.getLogger(__name__)
 
 
-def _update_names(names, items):
-    from flatten_json import flatten
+def _filter_names(
+    names: Iterable,
+    label: str,
+    include: Optional[Iterable],
+    exclude: Optional[Iterable],
+):
+    if include and exclude:
+        intersection = set(include) & set(exclude)
+        if intersection:
+            values = ", ".join(intersection)
+            raise InvalidArgumentError(
+                f"'{values}' specified in both --include-{label} and"
+                f" --exclude-{label}"
+            )
 
+    names = [tuple(name.split(".")) for name in names]
+
+    def _filter(filters, update_func):
+        filters = [tuple(name.split(".")) for name in filters]
+        for length, groups in groupby(filters, len):
+            for group in groups:
+                matches = [name for name in names if name[:length] == group]
+                if not matches:
+                    name = ".".join(group)
+                    raise InvalidArgumentError(
+                        f"'{name}' does not match any known {label}"
+                    )
+                update_func({match: None for match in matches})
+
+    if include:
+        ret = OrderedDict()
+        _filter(include, ret.update)
+    else:
+        ret = OrderedDict({name: None for name in names})
+
+    if exclude:
+        _filter(exclude, ret.difference_update)
+
+    return [".".join(name) for name in ret]
+
+
+def _update_names(names, items):
     for name, item in items:
         if isinstance(item, dict):
-            item = flatten(item, ".")
+            item = flatten(item)
             names.update(item.keys())
         else:
-            names.add(name)
+            names[name] = None
 
 
-def _collect_names(all_experiments):
+def _collect_names(all_experiments, **kwargs):
     metric_names = set()
     param_names = set()
 
@@ -30,62 +73,159 @@ def _collect_names(all_experiments):
             _update_names(metric_names, exp.get("metrics", {}).items())
             _update_names(param_names, exp.get("params", {}).items())
 
-    return sorted(metric_names), sorted(param_names)
+    metric_names = _filter_names(
+        sorted(metric_names),
+        "metrics",
+        kwargs.get("include_metrics"),
+        kwargs.get("exclude_metrics"),
+    )
+    param_names = _filter_names(
+        sorted(param_names),
+        "params",
+        kwargs.get("include_params"),
+        kwargs.get("exclude_params"),
+    )
+
+    return metric_names, param_names
 
 
 def _collect_rows(
-    base_rev, experiments, metric_names, param_names, precision=None
+    base_rev,
+    experiments,
+    metric_names,
+    param_names,
+    precision=DEFAULT_PRECISION,
+    no_timestamp=False,
+    sort_by=None,
+    sort_order=None,
 ):
-    from flatten_json import flatten
+    if sort_by:
+        if sort_by in metric_names:
+            sort_type = "metrics"
+        elif sort_by in param_names:
+            sort_type = "params"
+        else:
+            raise InvalidArgumentError(f"Unknown sort column '{sort_by}'")
+        reverse = sort_order == "desc"
+        experiments = _sort_exp(experiments, sort_by, sort_type, reverse)
 
-    if precision is None:
-        precision = DEFAULT_PRECISION
+    for i, (rev, exp) in enumerate(experiments.items()):
+        row = []
+        style = None
+        queued = "*" if exp.get("queued", False) else ""
 
+        if rev == "baseline":
+            name = exp.get("name", base_rev)
+            row.append(f"{name}")
+            style = "bold"
+        elif i < len(experiments) - 1:
+            row.append(f"├── {queued}{rev[:7]}")
+        else:
+            row.append(f"└── {queued}{rev[:7]}")
+
+        if not no_timestamp:
+            row.append(_format_time(exp.get("timestamp")))
+
+        _extend_row(
+            row, metric_names, exp.get("metrics", {}).items(), precision
+        )
+        _extend_row(row, param_names, exp.get("params", {}).items(), precision)
+
+        yield row, style
+
+
+def _sort_exp(experiments, sort_by, typ, reverse):
+    if "baseline" in experiments:
+        ret = OrderedDict({"baseline": experiments.pop("baseline")})
+    else:
+        ret = OrderedDict()
+
+    def _sort(item):
+        _, exp = item
+        for fname, item in exp.get(typ, {}).items():
+            if isinstance(item, dict):
+                item = flatten(item)
+            else:
+                item = {fname: item}
+            if sort_by in item:
+                val = item[sort_by]
+                return (val is None, val)
+        return (True, None)
+
+    ret.update(sorted(experiments.items(), key=_sort, reverse=reverse))
+    return ret
+
+
+def _format_time(timestamp):
+    if timestamp is None:
+        return "-"
+    if timestamp.date() == date.today():
+        fmt = "%I:%M %p"
+    else:
+        fmt = "%b %d, %Y"
+    return timestamp.strftime(fmt)
+
+
+def _extend_row(row, names, items, precision):
     def _round(val):
         if isinstance(val, float):
             return round(val, precision)
 
         return val
 
-    def _extend(row, names, items):
-        for fname, item in items:
-            if isinstance(item, dict):
-                item = flatten(item, ".")
-            else:
-                item = {fname: item}
-            for name in names:
-                if name in item:
-                    row.append(str(_round(item[name])))
-                else:
-                    row.append("-")
+    if not items:
+        row.extend(["-"] * len(names))
+        return
 
-    for i, (rev, exp) in enumerate(experiments.items()):
-        row = []
-        style = None
-        if rev == "baseline":
-            row.append(f"{base_rev}")
-            style = "bold"
-        elif i < len(experiments) - 1:
-            row.append(f"├── {rev[:7]}")
+    for fname, item in items:
+        if isinstance(item, dict):
+            item = flatten(item)
         else:
-            row.append(f"└── {rev[:7]}")
-
-        _extend(row, metric_names, exp.get("metrics", {}).items())
-        _extend(row, param_names, exp.get("params", {}).items())
-
-        yield row, style
+            item = {fname: item}
+        for name in names:
+            if name in item:
+                value = item[name]
+                text = str(_round(value)) if value is not None else "-"
+                row.append(text)
+            else:
+                row.append("-")
 
 
-def _show_experiments(all_experiments, console, precision=None):
+def _parse_list(param_list):
+    ret = []
+    for param_str in param_list:
+        # we don't care about filename prefixes for show, silently
+        # ignore it if provided to keep usage consistent with other
+        # metric/param list command options
+        _, _, param_str = param_str.rpartition(":")
+        ret.extend(param_str.split(","))
+    return ret
+
+
+def _show_experiments(all_experiments, console, **kwargs):
     from rich.table import Table
+
     from dvc.scm.git import Git
 
-    metric_names, param_names = _collect_names(all_experiments)
+    include_metrics = _parse_list(kwargs.pop("include_metrics", []))
+    exclude_metrics = _parse_list(kwargs.pop("exclude_metrics", []))
+    include_params = _parse_list(kwargs.pop("include_params", []))
+    exclude_params = _parse_list(kwargs.pop("exclude_params", []))
+
+    metric_names, param_names = _collect_names(
+        all_experiments,
+        include_metrics=include_metrics,
+        exclude_metrics=exclude_metrics,
+        include_params=include_params,
+        exclude_params=exclude_params,
+    )
 
     table = Table()
-    table.add_column("Experiment")
+    table.add_column("Experiment", no_wrap=True)
+    if not kwargs.get("no_timestamp", False):
+        table.add_column("Created")
     for name in metric_names:
-        table.add_column(name, justify="right")
+        table.add_column(name, justify="right", no_wrap=True)
     for name in param_names:
         table.add_column(name, justify="left")
 
@@ -94,11 +234,7 @@ def _show_experiments(all_experiments, console, precision=None):
             base_rev = base_rev[:7]
 
         for row, _, in _collect_rows(
-            base_rev,
-            experiments,
-            metric_names,
-            param_names,
-            precision=precision,
+            base_rev, experiments, metric_names, param_names, **kwargs,
         ):
             table.add_row(*row)
 
@@ -108,6 +244,7 @@ def _show_experiments(all_experiments, console, precision=None):
 class CmdExperimentsShow(CmdBase):
     def run(self):
         from rich.console import Console
+
         from dvc.utils.pager import pager
 
         if not self.repo.experiments:
@@ -118,17 +255,32 @@ class CmdExperimentsShow(CmdBase):
                 all_branches=self.args.all_branches,
                 all_tags=self.args.all_tags,
                 all_commits=self.args.all_commits,
+                sha_only=self.args.sha,
             )
 
-            # Note: rich does not currently include a native way to force
-            # infinite width for use with a pager
-            console = Console(
-                file=io.StringIO(), force_terminal=True, width=9999
+            if self.args.no_pager:
+                console = Console()
+            else:
+                # Note: rich does not currently include a native way to force
+                # infinite width for use with a pager
+                console = Console(
+                    file=io.StringIO(), force_terminal=True, width=9999
+                )
+
+            _show_experiments(
+                all_experiments,
+                console,
+                include_metrics=self.args.include_metrics,
+                exclude_metrics=self.args.exclude_metrics,
+                include_params=self.args.include_params,
+                exclude_params=self.args.exclude_params,
+                no_timestamp=self.args.no_timestamp,
+                sort_by=self.args.sort_by,
+                sort_order=self.args.sort_order,
             )
 
-            _show_experiments(all_experiments, console)
-
-            pager(console.file.getvalue())
+            if not self.args.no_pager:
+                pager(console.file.getvalue())
         except DvcException:
             logger.exception("failed to show experiments")
             return 1
@@ -141,9 +293,7 @@ class CmdExperimentsCheckout(CmdBase):
         if not self.repo.experiments:
             return 0
 
-        self.repo.experiments.checkout(
-            self.args.experiment, force=self.args.force
-        )
+        self.repo.experiments.checkout(self.args.experiment)
 
         return 0
 
@@ -229,6 +379,7 @@ def add_parser(subparsers, parent_parser):
     experiments_parser = subparsers.add_parser(
         "experiments",
         parents=[parent_parser],
+        aliases=["exp"],
         description=append_doc_link(EXPERIMENTS_HELP, "experiments"),
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -269,6 +420,63 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show metrics for all commits.",
     )
+    experiments_show_parser.add_argument(
+        "--no-pager",
+        action="store_true",
+        default=False,
+        help="Do not pipe output into a pager.",
+    )
+    experiments_show_parser.add_argument(
+        "--include-metrics",
+        action="append",
+        default=[],
+        help="Include the specified metrics in output table.",
+        metavar="<metrics_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--exclude-metrics",
+        action="append",
+        default=[],
+        help="Exclude the specified metrics from output table.",
+        metavar="<metrics_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--include-params",
+        action="append",
+        default=[],
+        help="Include the specified params in output table.",
+        metavar="<params_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--exclude-params",
+        action="append",
+        default=[],
+        help="Exclude the specified params from output table.",
+        metavar="<params_list>",
+    )
+    experiments_show_parser.add_argument(
+        "--sort-by",
+        help="Sort related experiments by the specified metric or param.",
+        metavar="<metric/param>",
+    )
+    experiments_show_parser.add_argument(
+        "--sort-order",
+        help="Sort order to use with --sort-by.",
+        choices=("asc", "desc"),
+        default="asc",
+    )
+    experiments_show_parser.add_argument(
+        "--no-timestamp",
+        action="store_true",
+        default=False,
+        help="Do not show experiment timestamps.",
+    )
+    experiments_show_parser.add_argument(
+        "--sha",
+        action="store_true",
+        default=False,
+        help="Always show git commit SHAs instead of branch/tag names.",
+    )
     experiments_show_parser.set_defaults(func=CmdExperimentsShow)
 
     EXPERIMENTS_CHECKOUT_HELP = "Checkout experiments."
@@ -281,14 +489,6 @@ def add_parser(subparsers, parent_parser):
         help=EXPERIMENTS_CHECKOUT_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    experiments_checkout_parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Overwrite your current workspace with changes from the "
-        "experiment.",
-    )
     experiments_checkout_parser.add_argument(
         "experiment", help="Checkout this experiment.",
     )

dvc/command/gc.py

@@ -123,7 +123,11 @@ def add_parser(subparsers, parent_parser):
         "-j",
         "--jobs",
         type=int,
-        help="Number of jobs to run simultaneously.",
+        help=(
+            "Number of jobs to run simultaneously. "
+            "The default value is 4 * cpu_count(). "
+            "For SSH remotes, the default is 4. "
+        ),
         metavar="<number>",
     )
     gc_parser.add_argument(

dvc/command/git_hook.py

@@ -61,6 +61,26 @@ class CmdPrePush(CmdHookBase):
         return main(["push"])
 
 
+class CmdMergeDriver(CmdHookBase):
+    def _run(self):
+        from dvc.dvcfile import Dvcfile
+        from dvc.repo import Repo
+
+        dvc = Repo()
+
+        try:
+            with dvc.state:
+                ancestor = Dvcfile(dvc, self.args.ancestor, verify=False)
+                our = Dvcfile(dvc, self.args.our, verify=False)
+                their = Dvcfile(dvc, self.args.their, verify=False)
+
+                our.merge(ancestor, their)
+
+                return 0
+        finally:
+            dvc.close()
+
+
 def add_parser(subparsers, parent_parser):
     GIT_HOOK_HELP = "Run GIT hook."
 
@@ -113,3 +133,27 @@ def add_parser(subparsers, parent_parser):
         "args", nargs="*", help="Arguments passed by GIT or pre-commit tool.",
     )
     pre_push_parser.set_defaults(func=CmdPrePush)
+
+    MERGE_DRIVER_HELP = "Run GIT merge driver."
+    merge_driver_parser = git_hook_subparsers.add_parser(
+        "merge-driver",
+        parents=[parent_parser],
+        description=MERGE_DRIVER_HELP,
+        help=MERGE_DRIVER_HELP,
+    )
+    merge_driver_parser.add_argument(
+        "--ancestor",
+        required=True,
+        help="Ancestor's version of the conflicting file.",
+    )
+    merge_driver_parser.add_argument(
+        "--our",
+        required=True,
+        help="Current version of the conflicting file.",
+    )
+    merge_driver_parser.add_argument(
+        "--their",
+        required=True,
+        help="Other branch's version of the conflicting file.",
+    )
+    merge_driver_parser.set_defaults(func=CmdMergeDriver)

dvc/command/imp_url.py

@@ -42,20 +42,16 @@ def add_parser(subparsers, parent_parser):
     )
     import_parser.add_argument(
         "url",
-        help="Supported urls:\n"
-        "/path/to/file\n"
-        "/path/to/directory\n"
-        "C:\\\\path\\to\\file\n"
-        "C:\\\\path\\to\\directory\n"
+        help="Location of the data to download. Supported URLs:\n"
+        "/absolute/path/to/file/or/dir\n"
+        "relative/path/to/file/or/dir\n"
+        "C:\\\\path\\to\\file\\or\\dir\n"
         "https://example.com/path/to/file\n"
-        "s3://bucket/path/to/file\n"
-        "s3://bucket/path/to/directory\n"
-        "gs://bucket/path/to/file\n"
-        "gs://bucket/path/to/directory\n"
+        "s3://bucket/key/path\n"
+        "gs://bucket/path/to/file/or/dir\n"
         "hdfs://example.com/path/to/file\n"
-        "ssh://example.com:/path/to/file\n"
-        "ssh://example.com:/path/to/directory\n"
-        "remote://myremote/path/to/file (see `dvc remote`)",
+        "ssh://example.com/absolute/path/to/file/or/dir\n"
+        "remote://remote_name/path/to/file/or/dir (see `dvc remote`)",
     )
     import_parser.add_argument(
         "out", nargs="?", help="Destination path to put files to.",

dvc/command/init.py

@@ -8,8 +8,8 @@ logger = logging.getLogger(__name__)
 
 class CmdInit(CmdBaseNoRepo):
     def run(self):
-        from dvc.repo import Repo
         from dvc.exceptions import InitError
+        from dvc.repo import Repo
 
         try:
             self.repo = Repo.init(

dvc/command/install.py

@@ -2,6 +2,7 @@ import argparse
 import logging
 
 from dvc.command.base import CmdBase, append_doc_link
+from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
@@ -10,7 +11,7 @@ class CmdInstall(CmdBase):
     def run(self):
         try:
             self.repo.install(self.args.use_pre_commit_tool)
-        except Exception:  # noqa, pylint:disable=broad-except
+        except DvcException:
             logger.exception("failed to install DVC Git hooks")
             return 1
         return 0

dvc/command/metrics.py

@@ -14,8 +14,8 @@ DEFAULT_PRECISION = 5
 def show_metrics(
     metrics, all_branches=False, all_tags=False, all_commits=False
 ):
-    from flatten_json import flatten
     from dvc.utils.diff import format_dict
+    from dvc.utils.flatten import flatten
 
     # When `metrics` contains a `None` key, it means that some files
     # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
@@ -31,7 +31,7 @@ def show_metrics(
                 continue
 
             logger.info(f"\t{fname}:")
-            for key, value in flatten(format_dict(metric), ".").items():
+            for key, value in flatten(format_dict(metric)).items():
                 logger.info(f"\t\t{key}: {value}")
 
     if missing:

dvc/command/plots.py

@@ -86,7 +86,12 @@ class CmdPlotsShow(CmdPlots):
 
 class CmdPlotsDiff(CmdPlots):
     def _func(self, *args, **kwargs):
-        return self.repo.plots.diff(*args, revs=self.args.revisions, **kwargs)
+        return self.repo.plots.diff(
+            *args,
+            revs=self.args.revisions,
+            experiment=self.args.experiment,
+            **kwargs,
+        )
 
 
 class CmdPlotsModify(CmdPlots):
@@ -151,6 +156,13 @@ def add_parser(subparsers, parent_parser):
         help="Plots file to visualize. Shows all plots by default.",
         metavar="<path>",
     ).complete = completion.FILE
+    plots_diff_parser.add_argument(
+        "-e",
+        "--experiment",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
     plots_diff_parser.add_argument(
         "revisions", nargs="*", default=None, help="Git commits to plot from",
     )

dvc/command/remote.py

@@ -3,7 +3,7 @@ import logging
 
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
-from dvc.config import ConfigError
+from dvc.config import ConfigError, merge
 from dvc.utils import format_link
 
 logger = logging.getLogger(__name__)
@@ -18,7 +18,7 @@ class CmdRemote(CmdConfig):
 
     def _check_exists(self, conf):
         if self.args.name not in conf["remote"]:
-            raise ConfigError(f"remote '{self.args.name}' doesn't exists.")
+            raise ConfigError(f"remote '{self.args.name}' doesn't exist.")
 
 
 class CmdRemoteAdd(CmdRemote):
@@ -61,7 +61,12 @@ class CmdRemoteRemove(CmdRemote):
 class CmdRemoteModify(CmdRemote):
     def run(self):
         with self.config.edit(self.args.level) as conf:
-            self._check_exists(conf)
+            merged = self.config.load_config_to_level(self.args.level)
+            merge(merged, conf)
+            self._check_exists(merged)
+
+            if self.args.name not in conf["remote"]:
+                conf["remote"][self.args.name] = {}
             section = conf["remote"][self.args.name]
             if self.args.unset:
                 section.pop(self.args.option, None)
@@ -167,7 +172,7 @@ def add_parser(subparsers, parent_parser):
     remote_add_parser.add_argument("name", help="Name of the remote")
     remote_add_parser.add_argument(
         "url",
-        help="URL. See full list of supported urls at {}".format(
+        help="Remote location. See full list of supported URLs at {}".format(
             format_link("https://man.dvc.org/remote")
         ),
     )
@@ -196,7 +201,7 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     remote_default_parser.add_argument(
-        "name", nargs="?", help="Name of the remote."
+        "name", nargs="?", help="Name of the remote"
     )
     remote_default_parser.add_argument(
         "-u",
@@ -215,7 +220,7 @@ def add_parser(subparsers, parent_parser):
         help=REMOTE_MODIFY_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
-    remote_modify_parser.add_argument("name", help="Name of the remote.")
+    remote_modify_parser.add_argument("name", help="Name of the remote")
     remote_modify_parser.add_argument(
         "option", help="Name of the option to modify."
     )

dvc/command/repro.py

@@ -41,6 +41,10 @@ class CmdRepro(CmdBase):
                     recursive=self.args.recursive,
                     force_downstream=self.args.force_downstream,
                     experiment=self.args.experiment,
+                    queue=self.args.queue,
+                    run_all=self.args.run_all,
+                    jobs=self.args.jobs,
+                    params=self.args.params,
                 )
 
                 if len(stages) == 0:
@@ -174,4 +178,23 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help=argparse.SUPPRESS,
     )
+    repro_parser.add_argument(
+        "--params",
+        action="append",
+        default=[],
+        help=argparse.SUPPRESS,
+        metavar="[<filename>:]<params_list>",
+    )
+    repro_parser.add_argument(
+        "--queue", action="store_true", default=False, help=argparse.SUPPRESS
+    )
+    repro_parser.add_argument(
+        "--run-all",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    repro_parser.add_argument(
+        "-j", "--jobs", type=int, help=argparse.SUPPRESS, metavar="<number>"
+    )
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/status.py

@@ -1,6 +1,7 @@
 import logging
 
 from dvc.command.data_sync import CmdDataBase
+from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
 
@@ -63,7 +64,7 @@ class CmdDataStatus(CmdDataBase):
             else:
                 logger.info(self.UP_TO_DATE_MSG)
 
-        except Exception:  # noqa, pylint: disable=broad-except
+        except DvcException:
             logger.exception("failed to obtain data status")
             return 1
         return 0

dvc/command/version.py

@@ -1,159 +1,18 @@
 import argparse
-import itertools
 import logging
-import os
-import pathlib
-import platform
-import uuid
 
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
-from dvc.exceptions import DvcException, NotDvcRepoError
-from dvc.scm.base import SCMError
-from dvc.system import System
-from dvc.utils import relpath
-from dvc.utils.pkg import PKG
-from dvc.version import __version__
-
-try:
-    import psutil
-except ImportError:
-    psutil = None
-
+from dvc.info import get_dvc_info
 
 logger = logging.getLogger(__name__)
 
 
 class CmdVersion(CmdBaseNoRepo):
     def run(self):
-        from dvc.repo import Repo
-
-        package = PKG
-        if PKG is None:
-            package = ""
-        else:
-            package = f"({PKG})"
-
-        info = [
-            f"DVC version: {__version__} {package}",
-            "---------------------------------",
-            f"Platform: Python {platform.python_version()} on "
-            f"{platform.platform()}",
-            f"Supports: {self.get_supported_remotes()}",
-        ]
-
-        try:
-            repo = Repo()
-            root_directory = repo.root_dir
-
-            # cache_dir might not exist yet (e.g. after `dvc init`), and we
-            # can't auto-create it, as it might cause issues if the user
-            # later decides to enable shared cache mode with
-            # `dvc config cache.shared group`.
-            if os.path.exists(repo.cache.local.cache_dir):
-                info.append(
-                    "Cache types: {}".format(
-                        self.get_linktype_support_info(repo)
-                    )
-                )
-                if psutil:
-                    fs_type = self.get_fs_type(repo.cache.local.cache_dir)
-                    info.append(f"Cache directory: {fs_type}")
-            else:
-                logger.warning(
-                    "Unable to detect supported link types, as cache "
-                    "directory '{}' doesn't exist. It is usually auto-created "
-                    "by commands such as `dvc add/fetch/pull/run/import`, "
-                    "but you could create it manually to enable this "
-                    "check.".format(relpath(repo.cache.local.cache_dir))
-                )
-
-        except NotDvcRepoError:
-            root_directory = os.getcwd()
-        except SCMError:
-            root_directory = os.getcwd()
-            info.append("Repo: dvc, git (broken)")
-        else:
-            if psutil:
-                fs_root = self.get_fs_type(os.path.abspath(root_directory))
-                info.append(f"Workspace directory: {fs_root}")
-
-            info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
-
-        logger.info("\n".join(info))
+        dvc_info = get_dvc_info()
+        logger.info(dvc_info)
         return 0
 
-    @staticmethod
-    def get_fs_type(path):
-        partition = {
-            pathlib.Path(part.mountpoint): (part.fstype + " on " + part.device)
-            for part in psutil.disk_partitions(all=True)
-        }
-
-        path = pathlib.Path(path)
-
-        for parent in itertools.chain([path], path.parents):
-            if parent in partition:
-                return partition[parent]
-        return ("unknown", "none")
-
-    @staticmethod
-    def get_linktype_support_info(repo):
-        links = {
-            "reflink": (System.reflink, None),
-            "hardlink": (System.hardlink, System.is_hardlink),
-            "symlink": (System.symlink, System.is_symlink),
-        }
-
-        fname = "." + str(uuid.uuid4())
-        src = os.path.join(repo.cache.local.cache_dir, fname)
-        open(src, "w").close()
-        dst = os.path.join(repo.root_dir, fname)
-
-        cache = []
-
-        for name, (link, is_link) in links.items():
-            try:
-                link(src, dst)
-                status = "supported"
-                if is_link and not is_link(dst):
-                    status = "broken"
-                os.unlink(dst)
-            except DvcException:
-                status = "not supported"
-
-            if status == "supported":
-                cache.append(name)
-        os.remove(src)
-
-        return ", ".join(cache)
-
-    @staticmethod
-    def get_supported_remotes():
-        from dvc.tree import TREES
-
-        supported_remotes = []
-        for tree_cls in TREES:
-            if not tree_cls.get_missing_deps():
-                supported_remotes.append(tree_cls.scheme)
-
-        if len(supported_remotes) == len(TREES):
-            return "All remotes"
-
-        if len(supported_remotes) == 1:
-            return supported_remotes
-
-        return ", ".join(supported_remotes)
-
-
-def _get_dvc_repo_info(repo):
-    if repo.config.get("core", {}).get("no_scm", False):
-        return "dvc (no_scm)"
-
-    if repo.root_dir != repo.scm.root_dir:
-        return "dvc (subdir), git"
-
-    return "dvc, git"
-
 
 def add_parser(subparsers, parent_parser):
     VERSION_HELP = (

dvc/config.py

@@ -118,7 +118,19 @@ HTTP_COMMON = {
     "user": str,
     "password": str,
     "ask_password": Bool,
+    "ssl_verify": Bool,
+    "method": str,
 }
+WEBDAV_COMMON = {
+    "user": str,
+    "password": str,
+    "ask_password": Bool,
+    "token": str,
+    "cert_path": str,
+    "key_path": str,
+    "timeout": Coerce(int),
+}
+
 SCHEMA = {
     "core": {
         "remote": Lower,
@@ -127,7 +139,9 @@ SCHEMA = {
         Optional("analytics", default=True): Bool,
         Optional("hardlink_lock", default=False): Bool,
         Optional("no_scm", default=False): Bool,
+        Optional("autostage", default=False): Bool,
         Optional("experiments", default=False): Bool,
+        Optional("check_update", default=True): Bool,
     },
     "cache": {
         "local": str,
@@ -151,7 +165,7 @@ SCHEMA = {
                     "endpointurl": str,
                     "access_key_id": str,
                     "secret_access_key": str,
-                    Optional("listobjects", default=False): Bool,
+                    Optional("listobjects", default=False): Bool,  # obsoleted
                     Optional("use_ssl", default=True): Bool,
                     "sse": str,
                     "sse_kms_key_id": str,
@@ -199,6 +213,8 @@ SCHEMA = {
                 },
                 "http": {**HTTP_COMMON, **REMOTE_COMMON},
                 "https": {**HTTP_COMMON, **REMOTE_COMMON},
+                "webdav": {**WEBDAV_COMMON, **REMOTE_COMMON},
+                "webdavs": {**WEBDAV_COMMON, **REMOTE_COMMON},
                 "remote": {str: object},  # Any of the above options are valid
             }
         )
@@ -258,7 +274,7 @@ class Config(dict):
 
     @classmethod
     def get_dir(cls, level):
-        from appdirs import user_config_dir, site_config_dir
+        from appdirs import site_config_dir, user_config_dir
 
         assert level in ("global", "system")
 
@@ -312,11 +328,17 @@ class Config(dict):
         if not self["cache"].get("dir") and self.dvc_dir:
             self["cache"]["dir"] = os.path.join(self.dvc_dir, "cache")
 
+    def _get_tree(self, level):
+        # NOTE: this might be a GitTree, which doesn't see things outside of
+        # the repo.
+        return self.tree if level == "repo" else self.wtree
+
     def _load_config(self, level):
         filename = self.files[level]
+        tree = self._get_tree(level)
 
-        if self.tree.exists(filename, use_dvcignore=False):
-            with self.tree.open(filename) as fobj:
+        if tree.exists(filename, use_dvcignore=False):
+            with tree.open(filename) as fobj:
                 conf_obj = configobj.ConfigObj(fobj)
         else:
             conf_obj = configobj.ConfigObj()
@@ -324,13 +346,14 @@ class Config(dict):
 
     def _save_config(self, level, conf_dict):
         filename = self.files[level]
+        tree = self._get_tree(level)
 
         logger.debug(f"Writing '{filename}'.")
 
-        self.tree.makedirs(os.path.dirname(filename))
+        tree.makedirs(os.path.dirname(filename))
 
         config = configobj.ConfigObj(_pack_remotes(conf_dict))
-        with self.tree.open(filename, "wb") as fobj:
+        with tree.open(filename, "wb") as fobj:
             config.write(fobj)
         config.filename = filename
 
@@ -388,7 +411,7 @@ class Config(dict):
             if merge_level == level:
                 break
             if merge_level in self.files:
-                _merge(merged_conf, self.load_one(merge_level))
+                merge(merged_conf, self.load_one(merge_level))
         return merged_conf
 
     @contextmanager
@@ -402,7 +425,7 @@ class Config(dict):
         conf = self._save_paths(conf, self.files[level])
 
         merged_conf = self.load_config_to_level(level)
-        _merge(merged_conf, conf)
+        merge(merged_conf, conf)
         self.validate(merged_conf)
 
         self._save_config(level, conf)
@@ -441,11 +464,11 @@ def _pack_remotes(conf):
     return result
 
 
-def _merge(into, update):
+def merge(into, update):
     """Merges second dict into first recursively"""
     for key, val in update.items():
         if isinstance(into.get(key), dict) and isinstance(val, dict):
-            _merge(into[key], val)
+            merge(into[key], val)
         else:
             into[key] = val
 

dvc/daemon.py

@@ -25,7 +25,7 @@ def _popen(cmd, **kwargs):
 
 
 def _spawn_windows(cmd, env):
-    from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
+    from subprocess import STARTF_USESHOWWINDOW, STARTUPINFO
 
     creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
 

dvc/data_cloud.py

@@ -81,26 +81,18 @@ class DataCloud:
         """
         remote = self.get_remote(remote, "pull")
 
-        downloaded_items_num = self.repo.cache.local.pull(
+        return self.repo.cache.local.pull(
             cache, jobs=jobs, remote=remote, show_checksums=show_checksums
         )
 
-        if not remote.tree.verify:
-            self._save_pulled_checksums(cache)
-
-        return downloaded_items_num
-
-    def _save_pulled_checksums(self, cache):
-        for checksum in cache.scheme_keys("local"):
-            cache_file = self.repo.cache.local.tree.hash_to_path_info(checksum)
-            if self.repo.cache.local.tree.exists(cache_file):
-                # We can safely save here, as existing corrupted files will
-                # be removed upon status, while files corrupted during
-                # download will not be moved from tmp_file
-                # (see `RemoteBASE.download()`)
-                self.repo.state.save(cache_file, checksum)
-
-    def status(self, cache, jobs=None, remote=None, show_checksums=False):
+    def status(
+        self,
+        cache,
+        jobs=None,
+        remote=None,
+        show_checksums=False,
+        log_missing=True,
+    ):
         """Check status of data items in a cloud-agnostic way.
 
         Args:
@@ -111,8 +103,18 @@ class DataCloud:
                 is used.
             show_checksums (bool): show checksums instead of file names in
                 information messages.
+            log_missing (bool): log warning messages if file doesn't exist
+                neither in cache, neither in cloud.
         """
         remote = self.get_remote(remote, "status")
         return self.repo.cache.local.status(
-            cache, jobs=jobs, remote=remote, show_checksums=show_checksums
+            cache,
+            jobs=jobs,
+            remote=remote,
+            show_checksums=show_checksums,
+            log_missing=log_missing,
         )
+
+    def get_url_for(self, remote, checksum):
+        remote = self.get_remote(remote)
+        return str(remote.tree.hash_to_path_info(checksum))

dvc/dependency/param.py

@@ -2,12 +2,12 @@ import os
 from collections import defaultdict
 
 import dpath.util
-import toml
-import yaml
 from voluptuous import Any
 
 from dvc.dependency.local import LocalDependency
 from dvc.exceptions import DvcException
+from dvc.hash_info import HashInfo
+from dvc.utils.serialize import LOADERS, ParseError
 
 
 class MissingParamsError(DvcException):
@@ -22,8 +22,6 @@ class ParamsDependency(LocalDependency):
     PARAM_PARAMS = "params"
     PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
     DEFAULT_PARAMS_FILE = "params.yaml"
-    PARAMS_FILE_LOADERS = defaultdict(lambda: yaml.safe_load)
-    PARAMS_FILE_LOADERS.update({".toml": toml.load})
 
     def __init__(self, stage, path, params):
         info = {}
@@ -34,7 +32,7 @@ class ParamsDependency(LocalDependency):
             else:
                 assert isinstance(params, dict)
                 self.params = list(params.keys())
-                info = params
+                info = {self.PARAM_PARAMS: params}
 
         super().__init__(
             stage,
@@ -43,60 +41,61 @@ class ParamsDependency(LocalDependency):
             info=info,
         )
 
+    def dumpd(self):
+        ret = super().dumpd()
+        if not self.hash_info:
+            ret[self.PARAM_PARAMS] = self.params
+        return ret
+
     def fill_values(self, values=None):
         """Load params values dynamically."""
         if not values:
             return
+        info = {}
         for param in self.params:
             if param in values:
-                self.info[param] = values[param]
+                info[param] = values[param]
+        self.hash_info = HashInfo(self.PARAM_PARAMS, info)
 
-    def save(self):
-        super().save()
-        self.info = self.save_info()
+    def workspace_status(self):
+        status = super().workspace_status()
 
-    def status(self):
-        status = super().status()
-
-        if status[str(self)] == "deleted":
+        if status.get(str(self)) == "deleted":
             return status
 
         status = defaultdict(dict)
-        info = self.read_params()
+        info = self.hash_info.value if self.hash_info else {}
+        actual = self.read_params()
         for param in self.params:
-            if param not in info.keys():
+            if param not in actual.keys():
                 st = "deleted"
-            elif param not in self.info:
+            elif param not in info:
                 st = "new"
-            elif info[param] != self.info[param]:
+            elif actual[param] != info[param]:
                 st = "modified"
             else:
-                assert info[param] == self.info[param]
+                assert actual[param] == info[param]
                 continue
 
             status[str(self)][param] = st
 
         return status
 
-    def dumpd(self):
-        return {
-            self.PARAM_PATH: self.def_path,
-            self.PARAM_PARAMS: self.info or self.params,
-        }
+    def status(self):
+        return self.workspace_status()
 
     def read_params(self):
         if not self.exists:
             return {}
 
-        with self.repo.tree.open(self.path_info, "r") as fobj:
-            try:
-                config = self.PARAMS_FILE_LOADERS[
-                    self.path_info.suffix.lower()
-                ](fobj)
-            except (yaml.YAMLError, toml.TomlDecodeError) as exc:
-                raise BadParamFileError(
-                    f"Unable to read parameters from '{self}'"
-                ) from exc
+        suffix = self.path_info.suffix.lower()
+        loader = LOADERS[suffix]
+        try:
+            config = loader(self.path_info, tree=self.repo.tree)
+        except ParseError as exc:
+            raise BadParamFileError(
+                f"Unable to read parameters from '{self}'"
+            ) from exc
 
         ret = {}
         for param in self.params:
@@ -106,7 +105,7 @@ class ParamsDependency(LocalDependency):
                 pass
         return ret
 
-    def save_info(self):
+    def get_hash(self):
         info = self.read_params()
 
         missing_params = set(self.params) - set(info.keys())
@@ -117,4 +116,4 @@ class ParamsDependency(LocalDependency):
                 )
             )
 
-        return info
+        return HashInfo(self.PARAM_PARAMS, info)

dvc/dependency/repo.py

@@ -1,10 +1,5 @@
-import os
-
 from voluptuous import Required
 
-from dvc.exceptions import OutputNotFoundError
-from dvc.path_info import PathInfo
-
 from .local import LocalDependency
 
 
@@ -42,40 +37,31 @@ class RepoDependency(LocalDependency):
     def __str__(self):
         return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
 
-    def _make_repo(self, *, locked=True):
+    def _make_repo(self, *, locked=True, **kwargs):
         from dvc.external_repo import external_repo
 
         d = self.def_repo
         rev = (d.get("rev_lock") if locked else None) or d.get("rev")
-        return external_repo(d["url"], rev=rev)
+        return external_repo(d["url"], rev=rev, **kwargs)
 
-    def _get_checksum(self, locked=True):
-        from dvc.repo.tree import RepoTree
+    def _get_hash(self, locked=True):
+        # we want stream but not fetch, so DVC out directories are
+        # walked, but dir contents is not fetched
+        with self._make_repo(locked=locked, fetch=False, stream=True) as repo:
+            return repo.get_checksum(self.def_path)
 
-        with self._make_repo(locked=locked) as repo:
-            try:
-                return repo.find_out_by_relpath(self.def_path).info["md5"]
-            except OutputNotFoundError:
-                path = PathInfo(os.path.join(repo.root_dir, self.def_path))
+    def workspace_status(self):
+        current = self._get_hash(locked=True)
+        updated = self._get_hash(locked=False)
 
-                # we want stream but not fetch, so DVC out directories are
-                # walked, but dir contents is not fetched
-                tree = RepoTree(repo, stream=True)
-
-                # We are polluting our repo cache with some dir listing here
-                if tree.isdir(path):
-                    return self.repo.cache.local.tree.get_hash(path, tree=tree)
-                return tree.get_file_hash(path)
-
-    def status(self):
-        current_checksum = self._get_checksum(locked=True)
-        updated_checksum = self._get_checksum(locked=False)
-
-        if current_checksum != updated_checksum:
+        if current != updated:
             return {str(self): "update available"}
 
         return {}
 
+    def status(self):
+        return self.workspace_status()
+
     def save(self):
         pass
 
@@ -83,14 +69,15 @@ class RepoDependency(LocalDependency):
         return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo}
 
     def download(self, to):
-        with self._make_repo() as repo:
+        cache = self.repo.cache.local
+
+        with self._make_repo(cache_dir=cache.cache_dir) as repo:
             if self.def_repo.get(self.PARAM_REV_LOCK) is None:
                 self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
 
-            cache = self.repo.cache.local
-            with repo.use_cache(cache):
-                _, _, cache_infos = repo.fetch_external([self.def_path])
-            cache.checkout(to.path_info, cache_infos[0])
+            _, _, cache_infos = repo.fetch_external([self.def_path])
+
+        cache.checkout(to.path_info, cache_infos[0])
 
     def update(self, rev=None):
         if rev:

dvc/dvcfile.py

@@ -16,7 +16,13 @@ from dvc.stage.exceptions import (
 from dvc.stage.loader import SingleStageLoader, StageLoader
 from dvc.utils import relpath
 from dvc.utils.collections import apply_diff
-from dvc.utils.yaml import dump_yaml, parse_yaml, parse_yaml_for_update
+from dvc.utils.serialize import (
+    dump_yaml,
+    load_yaml,
+    modify_yaml,
+    parse_yaml,
+    parse_yaml_for_update,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -56,9 +62,10 @@ def check_dvc_filename(path):
 class FileMixin:
     SCHEMA = None
 
-    def __init__(self, repo, path, **kwargs):
+    def __init__(self, repo, path, verify=True, **kwargs):
         self.repo = repo
         self.path = path
+        self.verify = verify
 
     def __repr__(self):
         return "{}: {}".format(
@@ -90,7 +97,8 @@ class FileMixin:
         # 3. path doesn't represent a regular file
         if not self.exists():
             raise StageFileDoesNotExistError(self.path)
-        check_dvc_filename(self.path)
+        if self.verify:
+            check_dvc_filename(self.path)
         if not self.repo.tree.isfile(self.path):
             raise StageFileIsNotDvcFileError(self.path)
 
@@ -115,6 +123,9 @@ class FileMixin:
     def dump(self, stage, **kwargs):
         raise NotImplementedError
 
+    def merge(self, ancestor, other):
+        raise NotImplementedError
+
 
 class SingleStageFile(FileMixin):
     from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
@@ -134,7 +145,8 @@ class SingleStageFile(FileMixin):
         from dvc.stage import PipelineStage
 
         assert not isinstance(stage, PipelineStage)
-        check_dvc_filename(self.path)
+        if self.verify:
+            check_dvc_filename(self.path)
         logger.debug(
             "Saving information to '{file}'.".format(file=relpath(self.path))
         )
@@ -144,6 +156,14 @@ class SingleStageFile(FileMixin):
     def remove_stage(self, stage):  # pylint: disable=unused-argument
         self.remove()
 
+    def merge(self, ancestor, other):
+        assert isinstance(ancestor, SingleStageFile)
+        assert isinstance(other, SingleStageFile)
+
+        stage = self.stage
+        stage.merge(ancestor.stage, other.stage)
+        self.dump(stage)
+
 
 class PipelineFile(FileMixin):
     """Abstraction for pipelines file, .yaml + .lock combined."""
@@ -155,48 +175,46 @@ class PipelineFile(FileMixin):
         return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
 
     def dump(
-        self, stage, update_pipeline=False, no_lock=False, **kwargs
+        self, stage, update_pipeline=True, update_lock=True, **kwargs
     ):  # pylint: disable=arguments-differ
         """Dumps given stage appropriately in the dvcfile."""
         from dvc.stage import PipelineStage
 
         assert isinstance(stage, PipelineStage)
-        check_dvc_filename(self.path)
+        if self.verify:
+            check_dvc_filename(self.path)
 
         if update_pipeline and not stage.is_data_source:
             self._dump_pipeline_file(stage)
 
-        if not no_lock:
+        if update_lock:
             self._dump_lockfile(stage)
 
     def _dump_lockfile(self, stage):
         self._lockfile.dump(stage)
 
     def _dump_pipeline_file(self, stage):
-        data = {}
-        if self.exists():
-            with open(self.path) as fd:
-                data = parse_yaml_for_update(fd.read(), self.path)
-        else:
-            logger.info("Creating '%s'", self.relpath)
-            open(self.path, "w+").close()
-
-        data["stages"] = data.get("stages", {})
         stage_data = serialize.to_pipeline_file(stage)
-        existing_entry = stage.name in data["stages"]
 
-        action = "Modifying" if existing_entry else "Adding"
-        logger.info("%s stage '%s' in '%s'", action, stage.name, self.relpath)
+        with modify_yaml(self.path, tree=self.repo.tree) as data:
+            if not data:
+                logger.info("Creating '%s'", self.relpath)
 
-        if existing_entry:
-            orig_stage_data = data["stages"][stage.name]
-            if "meta" in orig_stage_data:
-                stage_data[stage.name]["meta"] = orig_stage_data["meta"]
-            apply_diff(stage_data[stage.name], orig_stage_data)
-        else:
-            data["stages"].update(stage_data)
+            data["stages"] = data.get("stages", {})
+            existing_entry = stage.name in data["stages"]
+            action = "Modifying" if existing_entry else "Adding"
+            logger.info(
+                "%s stage '%s' in '%s'", action, stage.name, self.relpath
+            )
+
+            if existing_entry:
+                orig_stage_data = data["stages"][stage.name]
+                if "meta" in orig_stage_data:
+                    stage_data[stage.name]["meta"] = orig_stage_data["meta"]
+                apply_diff(stage_data[stage.name], orig_stage_data)
+            else:
+                data["stages"].update(stage_data)
 
-        dump_yaml(self.path, data)
         self.repo.scm.track_file(self.relpath)
 
     @property
@@ -239,6 +257,9 @@ class PipelineFile(FileMixin):
         else:
             super().remove()
 
+    def merge(self, ancestor, other):
+        raise NotImplementedError
+
 
 class Lockfile(FileMixin):
     from dvc.schema import COMPILED_LOCKFILE_SCHEMA as SCHEMA
@@ -246,8 +267,8 @@ class Lockfile(FileMixin):
     def load(self):
         if not self.exists():
             return {}
-        with self.repo.tree.open(self.path) as fd:
-            data = parse_yaml(fd.read(), self.path)
+
+        data = load_yaml(self.path, tree=self.repo.tree)
         try:
             self.validate(data, fname=self.relpath)
         except StageFileFormatError:
@@ -258,21 +279,18 @@ class Lockfile(FileMixin):
 
     def dump(self, stage, **kwargs):
         stage_data = serialize.to_lockfile(stage)
-        if not self.exists():
-            modified = True
-            logger.info("Generating lock file '%s'", self.relpath)
-            data = stage_data
-            open(self.path, "w+").close()
-        else:
-            with self.repo.tree.open(self.path, "r") as fd:
-                data = parse_yaml_for_update(fd.read(), self.path)
+
+        with modify_yaml(self.path, tree=self.repo.tree) as data:
+            if not data:
+                logger.info("Generating lock file '%s'", self.relpath)
+
             modified = data.get(stage.name, {}) != stage_data.get(
                 stage.name, {}
             )
             if modified:
                 logger.info("Updating lock file '%s'", self.relpath)
             data.update(stage_data)
-        dump_yaml(self.path, data)
+
         if modified:
             self.repo.scm.track_file(self.relpath)
 
@@ -295,6 +313,9 @@ class Lockfile(FileMixin):
         else:
             self.remove()
 
+    def merge(self, ancestor, other):
+        raise NotImplementedError
+
 
 class Dvcfile:
     def __new__(cls, repo, path, **kwargs):

dvc/exceptions.py

@@ -1,7 +1,7 @@
 """Exceptions raised by the dvc."""
 from funcy import first
 
-from dvc.utils import format_link, relpath
+from dvc.utils import error_link, format_link, relpath
 
 
 class DvcException(Exception):
@@ -171,11 +171,7 @@ class BadMetricError(DvcException):
 
 
 class NoMetricsError(DvcException):
-    def __init__(self):
-        super().__init__(
-            "no metric files in this repository. Use `-m/-M` options for "
-            "`dvc run` to mark stage outputs as  metrics."
-        )
+    pass
 
 
 class NoPlotsError(DvcException):
@@ -186,14 +182,6 @@ class NoPlotsError(DvcException):
         )
 
 
-class YAMLFileCorruptedError(DvcException):
-    def __init__(self, path):
-        path = relpath(path)
-        super().__init__(
-            f"unable to read: '{path}', YAML file structure is corrupted"
-        )
-
-
 class RecursiveAddingWhileUsingFilename(DvcException):
     def __init__(self):
         super().__init__(
@@ -268,8 +256,10 @@ class CheckoutError(DvcException):
         self.stats = stats
         targets = [str(t) for t in target_infos]
         m = (
-            "Checkout failed for following targets:\n{}\nDid you "
-            "forget to fetch?".format("\n".join(targets))
+            "Checkout failed for following targets:\n{}\nIs your "
+            "cache up to date?\n{}".format(
+                "\n".join(targets), error_link("missing-files"),
+            )
         )
         super().__init__(m)
 
@@ -344,3 +334,7 @@ class NoOutputOrStageError(DvcException):
             f"'{target}' "
             f"does not exist as an output or a stage name in '{file}'"
         )
+
+
+class MergeError(DvcException):
+    pass

dvc/external_repo.py

@@ -6,10 +6,12 @@ from contextlib import contextmanager
 from distutils.dir_util import copy_tree
 from typing import Iterable
 
-from funcy import cached_property, retry, wrap_with
+from funcy import cached_property, reraise, retry, wrap_with
 
+from dvc.cache import Cache
 from dvc.config import NoRemoteError, NotDvcRepoError
 from dvc.exceptions import (
+    DvcException,
     FileMissingError,
     NoOutputInExternalRepoError,
     NoRemoteInExternalRepoError,
@@ -18,39 +20,56 @@ from dvc.exceptions import (
 )
 from dvc.path_info import PathInfo
 from dvc.repo import Repo
-from dvc.repo.tree import RepoTree
 from dvc.scm.base import CloneError
 from dvc.scm.git import Git
+from dvc.state import StateNoop
 from dvc.tree.local import LocalTree
+from dvc.tree.repo import RepoTree
+from dvc.utils import relpath
 from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
 
+class IsADVCRepoError(DvcException):
+    """Raised when it is not expected to be a dvc repo."""
+
+
 @contextmanager
-def external_repo(url, rev=None, for_write=False):
+def external_repo(url, rev=None, for_write=False, **kwargs):
     logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
-    if not rev:
-        # Local HEAD points to the tip of whatever branch we first cloned from
-        # (which may not be the default branch), use origin/HEAD here to get
-        # the tip of the default branch
-        rev = "refs/remotes/origin/HEAD"
+    # Local HEAD points to the tip of whatever branch we first cloned from
+    # (which may not be the default branch), use origin/HEAD here to get
+    # the tip of the default branch
+    rev = rev or "refs/remotes/origin/HEAD"
+
+    root_dir = path if for_write else os.path.realpath(path)
+    conf = dict(
+        root_dir=root_dir,
+        url=url,
+        scm=None if for_write else Git(root_dir),
+        rev=None if for_write else rev,
+        for_write=for_write,
+        **kwargs,
+    )
     try:
-        repo = ExternalRepo(path, url, rev, for_write=for_write)
+        repo = ExternalRepo(**conf)
     except NotDvcRepoError:
-        repo = ExternalGitRepo(path, url, rev)
+        repo = ExternalGitRepo(**conf)
 
     try:
         yield repo
-    except NoRemoteError:
-        raise NoRemoteInExternalRepoError(url)
+    except NoRemoteError as exc:
+        raise NoRemoteInExternalRepoError(url) from exc
     except OutputNotFoundError as exc:
         if exc.repo is repo:
-            raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url)
+            raise NoOutputInExternalRepoError(
+                exc.output, repo.root_dir, url
+            ) from exc
         raise
     except FileMissingError as exc:
-        raise PathMissingError(exc.path, url)
+        raise PathMissingError(exc.path, url) from exc
     finally:
         repo.close()
         if for_write:
@@ -74,42 +93,37 @@ def clean_repos():
 class BaseExternalRepo:
     # pylint: disable=no-member
 
-    _local_cache = None
-
     def __str__(self):
         return self.url
 
-    @property
-    def local_cache(self):
-        if hasattr(self, "cache"):
-            return self.cache.local
-        return self._local_cache
-
-    @contextmanager
-    def use_cache(self, cache):
-        """Use the specified cache in place of default tmpdir cache for
-        download operations.
-        """
-        if hasattr(self, "cache"):
-            save_cache = self.cache.local
-            self.cache.local = cache
-        self._local_cache = cache
-
-        yield
-
-        if hasattr(self, "cache"):
-            self.cache.local = save_cache
-        self._local_cache = None
-
     @cached_property
     def repo_tree(self):
-        return RepoTree(self, fetch=True)
+        return self._get_tree_for(
+            self, subrepos=not self.for_write, repo_factory=self.make_repo
+        )
 
     def get_rev(self):
+        assert self.scm
         if isinstance(self.tree, LocalTree):
             return self.scm.get_rev()
         return self.tree.rev
 
+    def _fetch_to_cache(self, path_info, repo, callback):
+        # don't support subrepo traversal as it might fail due to difference
+        # in remotes
+        tree = self._get_tree_for(repo)
+        cache = repo.cache.local
+
+        hash_info = tree.get_hash(path_info, download_callback=callback)
+        cache.save(
+            path_info,
+            tree,
+            hash_info,
+            save_link=False,
+            download_callback=callback,
+        )
+        return hash_info
+
     def fetch_external(self, paths: Iterable, **kwargs):
         """Fetch specified external repo paths into cache.
 
@@ -120,114 +134,196 @@ class BaseExternalRepo:
         """
         download_results = []
         failed = 0
+        root = PathInfo(self.root_dir)
 
-        paths = [PathInfo(self.root_dir) / path for path in paths]
+        paths = [root / path for path in paths]
 
         def download_update(result):
             download_results.append(result)
 
-        save_infos = []
+        hash_infos = []
         for path in paths:
-            if not self.repo_tree.exists(path):
-                raise PathMissingError(path, self.url)
-            save_info = self.local_cache.save(
-                path,
-                self.repo_tree,
-                None,
-                save_link=False,
-                download_callback=download_update,
-            )
-            save_infos.append(save_info)
+            with reraise(FileNotFoundError, PathMissingError(path, self.url)):
+                metadata = self.repo_tree.metadata(path)
 
-        return sum(download_results), failed, save_infos
+            self._check_repo(path, metadata.repo)
+            repo = metadata.repo
+            hash_info = self._fetch_to_cache(path, repo, download_update)
+            hash_infos.append(hash_info)
+
+        return sum(download_results), failed, hash_infos
+
+    def _check_repo(self, path_info, repo):
+        if not repo:
+            return
+
+        repo_path = PathInfo(repo.root_dir)
+        if path_info == repo_path and isinstance(repo, Repo):
+            message = "Cannot fetch a complete DVC repository"
+            if repo.root_dir != self.root_dir:
+                rel = relpath(repo.root_dir, self.root_dir)
+                message += f" '{rel}'"
+            raise IsADVCRepoError(message)
 
     def get_external(self, path, dest):
         """Convenience wrapper for fetch_external and checkout."""
-        if self.local_cache:
+        path_info = PathInfo(self.root_dir) / path
+        with reraise(FileNotFoundError, PathMissingError(path, self.url)):
+            metadata = self.repo_tree.metadata(path_info)
+
+        self._check_repo(path_info, metadata.repo)
+        if metadata.output_exists:
+            repo = metadata.repo
+            cache = repo.cache.local
             # fetch DVC and git files to tmpdir cache, then checkout
-            _, _, save_infos = self.fetch_external([path])
-            self.local_cache.checkout(PathInfo(dest), save_infos[0])
+            save_info = self._fetch_to_cache(path_info, repo, None)
+            cache.checkout(PathInfo(dest), save_info)
         else:
-            # git-only erepo with no cache, just copy files directly
-            # to dest
-            path = PathInfo(self.root_dir) / path
-            if not self.repo_tree.exists(path):
-                raise PathMissingError(path, self.url)
-            self.repo_tree.copytree(path, dest)
+            # git-only folder, just copy files directly to dest
+            tree = self._get_tree_for(metadata.repo)  # ignore subrepos
+            tree.copytree(path_info, dest)
 
+    def _get_tree_for(self, repo, **kwargs):
+        """
+        Provides a combined tree of a single repo with dvc + git/local tree.
+        """
+        kw = {**self.tree_confs, **kwargs}
+        if "fetch" not in kw:
+            kw["fetch"] = True
+        return RepoTree(repo, **kw)
 
-class ExternalRepo(Repo, BaseExternalRepo):
-    def __init__(self, root_dir, url, rev, for_write=False):
-        if for_write:
-            super().__init__(root_dir)
-        else:
-            root_dir = os.path.realpath(root_dir)
-            super().__init__(root_dir, scm=Git(root_dir), rev=rev)
-        self.url = url
-        self._set_cache_dir()
-        self._fix_upstream()
+    def get_checksum(self, path):
+        path_info = PathInfo(self.root_dir) / path
+        with reraise(FileNotFoundError, PathMissingError(path, self.url)):
+            metadata = self.repo_tree.metadata(path_info)
 
-    @wrap_with(threading.Lock())
-    def _set_cache_dir(self):
-        try:
-            cache_dir = CACHE_DIRS[self.url]
-        except KeyError:
-            cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
+        # skip subrepos to check for
+        tree = self._get_tree_for(metadata.repo)
+        return tree.get_hash(path_info)
 
-        self.cache.local.cache_dir = cache_dir
-        self._local_cache = self.cache.local
-
-    def _fix_upstream(self):
-        if not os.path.isdir(self.url):
-            return
-
-        try:
-            src_repo = Repo(self.url)
-        except NotDvcRepoError:
-            # If ExternalRepo does not throw NotDvcRepoError and Repo does,
-            # the self.url might be a bare git repo.
-            # NOTE: This will fail to resolve remote with relative path,
-            # same as if it was a remote DVC repo.
-            return
-
-        try:
-            remote_name = self.config["core"].get("remote")
-            if remote_name:
-                self._fix_local_remote(src_repo, remote_name)
-            else:
-                self._add_upstream(src_repo)
-        finally:
-            src_repo.close()
-
-    def _fix_local_remote(self, src_repo, remote_name):
+    @staticmethod
+    def _fix_local_remote(orig_repo, src_repo, remote_name):
         # If a remote URL is relative to the source repo,
         # it will have changed upon config load and made
         # relative to this new repo. Restore the old one here.
-        new_remote = self.config["remote"][remote_name]
+        new_remote = orig_repo.config["remote"][remote_name]
         old_remote = src_repo.config["remote"][remote_name]
         if new_remote["url"] != old_remote["url"]:
             new_remote["url"] = old_remote["url"]
 
-    def _add_upstream(self, src_repo):
+    @staticmethod
+    def _add_upstream(orig_repo, src_repo):
         # Fill the empty upstream entry with a new remote pointing to the
         # original repo's cache location.
         cache_dir = src_repo.cache.local.cache_dir
-        self.config["remote"]["auto-generated-upstream"] = {"url": cache_dir}
-        self.config["core"]["remote"] = "auto-generated-upstream"
+        orig_repo.config["remote"]["auto-generated-upstream"] = {
+            "url": cache_dir
+        }
+        orig_repo.config["core"]["remote"] = "auto-generated-upstream"
+
+    def make_repo(self, path):
+        repo = Repo(path, scm=self.scm, rev=self.get_rev())
+
+        self._setup_cache(repo)
+        self._fix_upstream(repo)
+
+        return repo
+
+    def _setup_cache(self, repo):
+        repo.cache.local.cache_dir = self.cache_dir
+        if self.cache_types:
+            repo.cache.local.cache_types = self.cache_types
+
+    def _fix_upstream(self, repo):
+        if not os.path.isdir(self.url):
+            return
+
+        try:
+            rel_path = os.path.relpath(repo.root_dir, self.root_dir)
+            src_repo = Repo(PathInfo(self.url) / rel_path)
+        except NotDvcRepoError:
+            return
+
+        try:
+            remote_name = repo.config["core"].get("remote")
+            if remote_name:
+                self._fix_local_remote(repo, src_repo, remote_name)
+            else:
+                self._add_upstream(repo, src_repo)
+        finally:
+            src_repo.close()
+
+    @wrap_with(threading.Lock())
+    def _get_cache_dir(self):
+        try:
+            cache_dir = CACHE_DIRS[self.url]
+        except KeyError:
+            cache_dir = CACHE_DIRS[self.url] = tempfile.mkdtemp("dvc-cache")
+        return cache_dir
+
+
+class ExternalRepo(BaseExternalRepo, Repo):
+    def __init__(
+        self,
+        root_dir,
+        url,
+        scm=None,
+        rev=None,
+        for_write=False,
+        cache_dir=None,
+        cache_types=None,
+        **kwargs,
+    ):
+        super().__init__(root_dir, scm=scm, rev=rev)
+
+        self.url = url
+        self.for_write = for_write
+        self.cache_dir = cache_dir or self._get_cache_dir()
+        self.cache_types = cache_types
+
+        self._setup_cache(self)
+        self._fix_upstream(self)
+        self.tree_confs = kwargs
 
 
 class ExternalGitRepo(BaseExternalRepo):
-    def __init__(self, root_dir, url, rev):
+    def __init__(
+        self,
+        root_dir,
+        url,
+        scm=None,
+        rev=None,
+        for_write=False,
+        cache_dir=None,
+        cache_types=None,
+        **kwargs,
+    ):
         self.root_dir = os.path.realpath(root_dir)
+        self.scm = scm
+
         self.url = url
-        self.tree = self.scm.get_tree(rev)
+        self.for_write = for_write
+        self.cache_dir = cache_dir or self._get_cache_dir()
+        self.cache_types = cache_types
+
+        self.rev = rev
+        self.tree_confs = kwargs
+
+        self.config = {"cache": {"dir": self.cache_dir}}
+        self.cache = Cache(self)
+        if cache_types:
+            self.cache.local.cache_types = cache_types
+
+        self.state = StateNoop()
 
     @cached_property
-    def scm(self):
-        return Git(self.root_dir)
+    def tree(self):
+        if self.scm:
+            return self.scm.get_tree(self.rev)
+        return LocalTree(self, {"url": self.root_dir})
 
     def close(self):
-        if "scm" in self.__dict__:
+        if self.scm:
             self.scm.close()
 
     def find_out_by_relpath(self, path):
@@ -236,14 +332,14 @@ class ExternalGitRepo(BaseExternalRepo):
     @contextmanager
     def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
         """Opens a specified resource as a file object."""
-        tree = RepoTree(self)
+        path_info = PathInfo(self.root_dir) / path
         try:
-            with tree.open(
-                path, mode=mode, encoding=encoding, **kwargs
+            with self.repo_tree.open(
+                path_info, mode=mode, encoding=encoding, **kwargs
             ) as fobj:
                 yield fobj
-        except FileNotFoundError:
-            raise PathMissingError(path, self.url)
+        except FileNotFoundError as exc:
+            raise PathMissingError(path, self.url) from exc
 
 
 def _cached_clone(url, rev, for_write=False):
@@ -351,6 +447,11 @@ def _remove(path):
     if os.name == "nt":
         # git.exe may hang for a while not permitting to remove temp dir
         os_retry = retry(5, errors=OSError, timeout=0.1)
-        os_retry(remove)(path)
+        try:
+            os_retry(remove)(path)
+        except PermissionError:
+            logger.warning(
+                "Failed to remove '%s'", relpath(path), exc_info=True
+            )
     else:
         remove(path)

dvc/hash_info.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass, field
+
+HASH_DIR_SUFFIX = ".dir"
+
+
+@dataclass
+class HashInfo:
+    name: str
+    value: str
+    dir_info: list = field(default=None, compare=False)
+
+    def __bool__(self):
+        return bool(self.value)
+
+    @classmethod
+    def from_dict(cls, d):
+        if not d:
+            return cls(None, None)
+        ((name, value),) = d.items()
+        return cls(name, value)
+
+    def to_dict(self):
+        return {self.name: self.value} if self else {}
+
+    @property
+    def isdir(self):
+        if not self:
+            return False
+        return self.value.endswith(HASH_DIR_SUFFIX)

dvc/ignore.py

@@ -1,16 +1,17 @@
 import logging
 import os
 import re
+from collections import namedtuple
 from itertools import groupby, takewhile
 
 from pathspec.patterns import GitWildMatchPattern
 from pathspec.util import normalize_file
-from pygtrie import StringTrie
 
 from dvc.path_info import PathInfo
-from dvc.pathspec_math import merge_patterns
+from dvc.pathspec_math import PatternInfo, merge_patterns
 from dvc.system import System
 from dvc.utils import relpath
+from dvc.utils.collections import PathStringTrie
 
 logger = logging.getLogger(__name__)
 
@@ -24,18 +25,26 @@ class DvcIgnore:
 
 class DvcIgnorePatterns(DvcIgnore):
     def __init__(self, pattern_list, dirname):
+        if pattern_list:
+            if isinstance(pattern_list[0], str):
+                pattern_list = [
+                    PatternInfo(pattern, "") for pattern in pattern_list
+                ]
 
         self.pattern_list = pattern_list
         self.dirname = dirname
         self.prefix = self.dirname + os.sep
 
-        regex_pattern_list = map(
-            GitWildMatchPattern.pattern_to_regex, pattern_list
-        )
+        self.regex_pattern_list = [
+            GitWildMatchPattern.pattern_to_regex(pattern_info.patterns)
+            for pattern_info in pattern_list
+        ]
 
         self.ignore_spec = [
             (ignore, re.compile("|".join(item[0] for item in group)))
-            for ignore, group in groupby(regex_pattern_list, lambda x: x[1])
+            for ignore, group in groupby(
+                self.regex_pattern_list, lambda x: x[1]
+            )
             if ignore is not None
         ]
 
@@ -43,9 +52,19 @@ class DvcIgnorePatterns(DvcIgnore):
     def from_files(cls, ignore_file_path, tree):
         assert os.path.isabs(ignore_file_path)
         dirname = os.path.normpath(os.path.dirname(ignore_file_path))
+        ignore_file_rel_path = os.path.relpath(
+            ignore_file_path, tree.tree_root
+        )
         with tree.open(ignore_file_path, encoding="utf-8") as fobj:
             path_spec_lines = [
-                line for line in map(str.strip, fobj.readlines()) if line
+                PatternInfo(
+                    line,
+                    "{}:{}:{}".format(ignore_file_rel_path, line_no + 1, line),
+                )
+                for line_no, line in enumerate(
+                    map(str.strip, fobj.readlines())
+                )
+                if line and not (line.strip().startswith("#"))
             ]
 
         return cls(path_spec_lines, dirname)
@@ -56,7 +75,7 @@ class DvcIgnorePatterns(DvcIgnore):
 
         return dirs, files
 
-    def matches(self, dirname, basename, is_dir=False):
+    def _get_normalize_path(self, dirname, basename):
         # NOTE: `relpath` is too slow, so we have to assume that both
         # `dirname` and `self.dirname` are relative or absolute together.
         if dirname == self.dirname:
@@ -70,6 +89,12 @@ class DvcIgnorePatterns(DvcIgnore):
 
         if not System.is_unix():
             path = normalize_file(path)
+        return path
+
+    def matches(self, dirname, basename, is_dir=False):
+        path = self._get_normalize_path(dirname, basename)
+        if not path:
+            return False
         return self.ignore(path, is_dir)
 
     def ignore(self, path, is_dir):
@@ -85,25 +110,57 @@ class DvcIgnorePatterns(DvcIgnore):
                     result = ignore
         return result
 
+    def match_details(self, dirname, basename, is_dir=False):
+        path = self._get_normalize_path(dirname, basename)
+        if not path:
+            return False
+        return self._ignore_details(path, is_dir)
+
+    def _ignore_details(self, path, is_dir):
+        result = []
+        for ignore, pattern in zip(self.regex_pattern_list, self.pattern_list):
+            regex = re.compile(ignore[0])
+            # skip system pattern
+            if not pattern.file_info:
+                continue
+            if is_dir:
+                path_dir = f"{path}/"
+                if regex.match(path) or regex.match(path_dir):
+                    result.append(pattern.file_info)
+            else:
+                if regex.match(path):
+                    result.append(pattern.file_info)
+        return result
+
     def __hash__(self):
-        return hash(self.dirname + ":" + "\n".join(self.pattern_list))
+        return hash(self.dirname + ":" + str(self.pattern_list))
 
     def __eq__(self, other):
         if not isinstance(other, DvcIgnorePatterns):
             return NotImplemented
         return (self.dirname == other.dirname) & (
-            self.pattern_list == other.pattern_list
+            [pattern.patterns for pattern in self.pattern_list]
+            == [pattern.patterns for pattern in other.pattern_list]
         )
 
     def __bool__(self):
         return bool(self.pattern_list)
 
 
+CheckIgnoreResult = namedtuple(
+    "CheckIgnoreResult", ["file", "match", "patterns"]
+)
+
+
+def _no_match(path):
+    return CheckIgnoreResult(path, False, ["::"])
+
+
 class DvcIgnoreFilterNoop:
     def __init__(self, tree, root_dir):
         pass
 
-    def __call__(self, root, dirs, files):
+    def __call__(self, root, dirs, files, **kwargs):
         return dirs, files
 
     def is_ignored_dir(self, _):
@@ -112,6 +169,12 @@ class DvcIgnoreFilterNoop:
     def is_ignored_file(self, _):
         return False
 
+    def check_ignore(self, path):
+        return _no_match(path)
+
+    def is_ignored(self, _):
+        return False
+
 
 class DvcIgnoreFilter:
     @staticmethod
@@ -127,10 +190,11 @@ class DvcIgnoreFilter:
 
         self.tree = tree
         self.root_dir = root_dir
-        self.ignores_trie_tree = StringTrie(separator=os.sep)
+        self.ignores_trie_tree = PathStringTrie()
         self.ignores_trie_tree[root_dir] = DvcIgnorePatterns(
             default_ignore_patterns, root_dir
         )
+        self._ignored_subrepos = PathStringTrie()
         self._update(self.root_dir)
 
     def _update(self, dirname):
@@ -166,27 +230,33 @@ class DvcIgnoreFilter:
     def _update_sub_repo(self, root, dirs):
         for d in dirs:
             if self._is_dvc_repo(root, d):
+                self._ignored_subrepos[root] = self._ignored_subrepos.get(
+                    root, set()
+                ) | {d}
+                pattern_info = PatternInfo(
+                    f"/{d}/", "in sub_repo:{}".format(d)
+                )
+                new_pattern = DvcIgnorePatterns([pattern_info], root)
                 old_pattern = self.ignores_trie_tree.longest_prefix(root).value
                 if old_pattern:
                     self.ignores_trie_tree[root] = DvcIgnorePatterns(
                         *merge_patterns(
                             old_pattern.pattern_list,
                             old_pattern.dirname,
-                            ["/{}/".format(d)],
-                            root,
+                            new_pattern.pattern_list,
+                            new_pattern.dirname,
                         )
                     )
                 else:
-                    self.ignores_trie_tree[root] = DvcIgnorePatterns(
-                        ["/{}/".format(d)], root
-                    )
+                    self.ignores_trie_tree[root] = new_pattern
 
-    def __call__(self, root, dirs, files):
+    def __call__(self, root, dirs, files, ignore_subrepos=True):
         ignore_pattern = self._get_trie_pattern(root)
         if ignore_pattern:
-            return ignore_pattern(root, dirs, files)
-        else:
-            return dirs, files
+            dirs, files = ignore_pattern(root, dirs, files)
+            if not ignore_subrepos:
+                dirs.extend(self._ignored_subrepos.get(root, []))
+        return dirs, files
 
     def _get_trie_pattern(self, dirname):
         ignore_pattern = self.ignores_trie_tree.get(dirname)
@@ -214,7 +284,7 @@ class DvcIgnoreFilter:
 
     def _is_ignored(self, path, is_dir=False):
         if self._outside_repo(path):
-            return True
+            return False
         dirname, basename = os.path.split(os.path.normpath(path))
         ignore_pattern = self._get_trie_pattern(dirname)
         if ignore_pattern:
@@ -222,14 +292,21 @@ class DvcIgnoreFilter:
         else:
             return False
 
-    def is_ignored_dir(self, path):
+    def _is_subrepo(self, path):
+        dirname, basename = os.path.split(os.path.normpath(path))
+        return basename in self._ignored_subrepos.get(dirname, set())
+
+    def is_ignored_dir(self, path, ignore_subrepos=True):
         path = os.path.abspath(path)
+        if not ignore_subrepos:
+            return not self._is_subrepo(path)
         if path == self.root_dir:
             return False
 
         return self._is_ignored(path, True)
 
     def is_ignored_file(self, path):
+        path = os.path.abspath(path)
         return self._is_ignored(path, False)
 
     def _outside_repo(self, path):
@@ -245,3 +322,37 @@ class DvcIgnoreFilter:
         ):
             return True
         return False
+
+    def check_ignore(self, target):
+        full_target = os.path.abspath(target)
+        if not self._outside_repo(full_target):
+            dirname, basename = os.path.split(os.path.normpath(full_target))
+            pattern = self._get_trie_pattern(dirname)
+            if pattern:
+                matches = pattern.match_details(
+                    dirname, basename, os.path.isdir(full_target)
+                )
+
+                if matches:
+                    return CheckIgnoreResult(target, True, matches)
+        return _no_match(target)
+
+    def is_ignored(self, path):
+        # NOTE: can't use self.check_ignore(path).match for now, see
+        # https://github.com/iterative/dvc/issues/4555
+        return self.is_ignored_dir(path) or self.is_ignored_file(path)
+
+
+def init(path):
+    dvcignore = os.path.join(path, DvcIgnore.DVCIGNORE_FILE)
+    if os.path.exists(dvcignore):
+        return dvcignore
+
+    with open(dvcignore, "w") as fobj:
+        fobj.write(
+            "# Add patterns of files dvc should ignore, which could improve\n"
+            "# the performance. Learn more at\n"
+            "# https://dvc.org/doc/user-guide/dvcignore\n"
+        )
+
+    return dvcignore

dvc/info.py

@@ -0,0 +1,136 @@
+import itertools
+import os
+import pathlib
+import platform
+import uuid
+
+from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.repo import Repo
+from dvc.scm.base import SCMError
+from dvc.system import System
+from dvc.tree import TREES
+from dvc.utils import error_link
+from dvc.utils.pkg import PKG
+from dvc.version import __version__
+
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
+if PKG is None:
+    package = ""
+else:
+    package = f"({PKG})"
+
+
+def get_dvc_info():
+    info = [
+        f"DVC version: {__version__} {package}",
+        "---------------------------------",
+        f"Platform: Python {platform.python_version()} on "
+        f"{platform.platform()}",
+        f"Supports: {_get_supported_remotes()}",
+    ]
+
+    try:
+        repo = Repo()
+
+        # cache_dir might not exist yet (e.g. after `dvc init`), and we
+        # can't auto-create it, as it might cause issues if the user
+        # later decides to enable shared cache mode with
+        # `dvc config cache.shared group`.
+        if os.path.exists(repo.cache.local.cache_dir):
+            info.append(
+                "Cache types: {}".format(_get_linktype_support_info(repo))
+            )
+            if psutil:
+                fs_type = get_fs_type(repo.cache.local.cache_dir)
+                info.append(f"Cache directory: {fs_type}")
+        else:
+            info.append("Cache types: " + error_link("no-dvc-cache"))
+
+    except NotDvcRepoError:
+        pass
+    except SCMError:
+        info.append("Repo: dvc, git (broken)")
+    else:
+        root_directory = repo.root_dir
+        if psutil:
+            fs_root = get_fs_type(os.path.abspath(root_directory))
+            info.append(f"Workspace directory: {fs_root}")
+        info.append("Repo: {}".format(_get_dvc_repo_info(repo)))
+    return "\n".join(info)
+
+
+def _get_linktype_support_info(repo):
+
+    links = {
+        "reflink": (System.reflink, None),
+        "hardlink": (System.hardlink, System.is_hardlink),
+        "symlink": (System.symlink, System.is_symlink),
+    }
+
+    fname = "." + str(uuid.uuid4())
+    src = os.path.join(repo.cache.local.cache_dir, fname)
+    open(src, "w").close()
+    dst = os.path.join(repo.root_dir, fname)
+
+    cache = []
+
+    for name, (link, is_link) in links.items():
+        try:
+            link(src, dst)
+            status = "supported"
+            if is_link and not is_link(dst):
+                status = "broken"
+            os.unlink(dst)
+        except DvcException:
+            status = "not supported"
+
+        if status == "supported":
+            cache.append(name)
+    os.remove(src)
+
+    return ", ".join(cache)
+
+
+def _get_supported_remotes():
+
+    supported_remotes = []
+    for tree_cls in TREES:
+        if not tree_cls.get_missing_deps():
+            supported_remotes.append(tree_cls.scheme)
+
+    if len(supported_remotes) == len(TREES):
+        return "All remotes"
+
+    if len(supported_remotes) == 1:
+        return supported_remotes
+
+    return ", ".join(supported_remotes)
+
+
+def get_fs_type(path):
+
+    partition = {
+        pathlib.Path(part.mountpoint): (part.fstype + " on " + part.device)
+        for part in psutil.disk_partitions(all=True)
+    }
+
+    path = pathlib.Path(path)
+
+    for parent in itertools.chain([path], path.parents):
+        if parent in partition:
+            return partition[parent]
+    return ("unknown", "none")
+
+
+def _get_dvc_repo_info(self):
+    if self.config.get("core", {}).get("no_scm", False):
+        return "dvc (no_scm)"
+
+    if self.root_dir != self.scm.root_dir:
+        return "dvc (subdir), git"
+
+    return "dvc, git"

dvc/lock.py

@@ -2,17 +2,17 @@
 
 import hashlib
 import os
-import time
 from datetime import timedelta
 
 import flufl.lock
 import zc.lockfile
+from funcy import retry
 
 from dvc.exceptions import DvcException
 from dvc.progress import Tqdm
 from dvc.utils import format_link
 
-DEFAULT_TIMEOUT = 5
+DEFAULT_TIMEOUT = 3
 
 FAILED_TO_LOCK_MESSAGE = (
     "cannot perform the command because another DVC process seems to be "
@@ -56,13 +56,10 @@ class Lock:
             raise LockError(FAILED_TO_LOCK_MESSAGE)
 
     def lock(self):
-        try:
-            self._do_lock()
-            return
-        except LockError:
-            time.sleep(DEFAULT_TIMEOUT)
-
-        self._do_lock()
+        retries = 6
+        delay = DEFAULT_TIMEOUT / retries
+        lock_retry = retry(retries, LockError, timeout=delay)(self._do_lock)
+        lock_retry()
 
     def unlock(self):
         self._lock.close()

dvc/main.py

@@ -2,6 +2,7 @@
 
 import errno
 import logging
+from contextlib import contextmanager
 
 from dvc import analytics
 from dvc.cli import parse_args
@@ -10,7 +11,7 @@ from dvc.exceptions import DvcException, DvcParserError, NotDvcRepoError
 from dvc.external_repo import clean_repos
 from dvc.logger import FOOTER, disable_other_loggers
 from dvc.tree.pool import close_pools
-from dvc.utils import format_link
+from dvc.utils import error_link
 
 # Workaround for CPython bug. See [1] and [2] for more info.
 # [1] https://github.com/aws/aws-cli/blob/1.16.277/awscli/clidriver.py#L55
@@ -21,7 +22,27 @@ from dvc.utils import format_link
 logger = logging.getLogger("dvc")
 
 
-def main(argv=None):
+@contextmanager
+def profile(enable, dump):
+    if not enable:
+        yield
+        return
+
+    import cProfile
+
+    prof = cProfile.Profile()
+    prof.enable()
+
+    yield
+
+    prof.disable()
+    if not dump:
+        prof.print_stats(sort="cumtime")
+        return
+    prof.dump_stats(dump)
+
+
+def main(argv=None):  # noqa: C901
     """Run dvc CLI command.
 
     Args:
@@ -37,20 +58,22 @@ def main(argv=None):
     try:
         args = parse_args(argv)
 
-        verbosity = args.verbose - args.quiet
-        if verbosity:
-            logger.setLevel(
-                {
-                    -2: logging.CRITICAL,
-                    -1: logging.ERROR,
-                    1: logging.DEBUG,
-                    2: logging.TRACE,
-                }[max(-2, min(verbosity, 2))]
-            )
+        level = None
+        if args.quiet:
+            level = logging.CRITICAL
+        elif args.verbose == 1:
+            level = logging.DEBUG
+        elif args.verbose > 1:
+            level = logging.TRACE
+
+        if level is not None:
+            logger.setLevel(level)
+
         logger.trace(args)
 
-        cmd = args.func(args)
-        ret = cmd.run()
+        with profile(enable=args.cprofile, dump=args.cprofile_dump):
+            cmd = args.func(args)
+            ret = cmd.run()
     except ConfigError:
         logger.exception("configuration error")
         ret = 251
@@ -65,22 +88,29 @@ def main(argv=None):
     except DvcException:
         ret = 255
         logger.exception("")
-    except OSError as exc:
-        if exc.errno == errno.EMFILE:
+    except Exception as exc:  # noqa, pylint: disable=broad-except
+        # pylint: disable=no-member
+        if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
             logger.exception(
                 "too many open files, please visit "
                 "{} to see how to handle this "
-                "problem".format(
-                    format_link("https://error.dvc.org/many-files")
-                ),
+                "problem".format(error_link("many-files")),
                 extra={"tb_only": True},
             )
         else:
+            from dvc.info import get_dvc_info
+
             logger.exception("unexpected error")
+            logger.info(FOOTER)
+            dvc_info = get_dvc_info()
+            logger.info(dvc_info)
         ret = 255
-    except Exception:  # noqa, pylint: disable=broad-except
-        logger.exception("unexpected error")
-        ret = 255
+
+    try:
+        if analytics.is_enabled():
+            analytics.collect_and_send_report(args, ret)
+
+        return ret
     finally:
         logger.setLevel(outerLogLevel)
 
@@ -92,11 +122,3 @@ def main(argv=None):
         # Remove cached repos in the end of the call, these are anonymous
         # so won't be reused by any other subsequent run anyway.
         clean_repos()
-
-    if ret != 0:
-        logger.info(FOOTER)
-
-    if analytics.is_enabled():
-        analytics.collect_and_send_report(args, ret)
-
-    return ret

dvc/output/base.py

@@ -10,8 +10,10 @@ from dvc.cache import NamedCache
 from dvc.exceptions import (
     CollectCacheError,
     DvcException,
+    MergeError,
     RemoteCacheRequiredError,
 )
+from dvc.hash_info import HashInfo
 
 from ..tree.base import BaseTree
 
@@ -45,6 +47,12 @@ class OutputIsStageFileError(DvcException):
         super().__init__(f"Stage file '{path}' cannot be an output.")
 
 
+class OutputIsIgnoredError(DvcException):
+    def __init__(self, match):
+        lines = "\n".join(match.patterns)
+        super().__init__(f"Path '{match.file}' is ignored by\n{lines}")
+
+
 class BaseOutput:
     IS_DEPENDENCY = False
 
@@ -77,6 +85,7 @@ class BaseOutput:
     DoesNotExistError = OutputDoesNotExistError
     IsNotFileOrDirError = OutputIsNotFileOrDirError
     IsStageFileError = OutputIsStageFileError
+    IsIgnoredError = OutputIsIgnoredError
 
     sep = "/"
 
@@ -91,7 +100,7 @@ class BaseOutput:
         plot=False,
         persist=False,
     ):
-        self._validate_output_path(path)
+        self._validate_output_path(path, stage)
         # This output (and dependency) objects have too many paths/urls
         # here is a list and comments:
         #
@@ -105,7 +114,7 @@ class BaseOutput:
         self.stage = stage
         self.repo = stage.repo if stage else None
         self.def_path = path
-        self.info = info
+        self.hash_info = HashInfo.from_dict(info)
         if tree:
             self.tree = tree
         else:
@@ -154,7 +163,7 @@ class BaseOutput:
 
     @property
     def dir_cache(self):
-        return self.cache.get_dir_cache(self.checksum)
+        return self.cache.get_dir_cache(self.hash_info)
 
     @classmethod
     def supported(cls, url):
@@ -162,58 +171,50 @@ class BaseOutput:
 
     @property
     def cache_path(self):
-        return self.cache.tree.hash_to_path_info(self.checksum).url
+        return self.cache.tree.hash_to_path_info(self.hash_info.value).url
 
-    @property
-    def checksum_type(self):
-        return self.tree.PARAM_CHECKSUM
-
-    @property
-    def checksum(self):
-        return self.info.get(self.tree.PARAM_CHECKSUM)
-
-    @checksum.setter
-    def checksum(self, checksum):
-        self.info[self.tree.PARAM_CHECKSUM] = checksum
-
-    def get_checksum(self):
-        return self.tree.get_hash(self.path_info)
+    def get_hash(self):
+        if not self.use_cache:
+            return self.tree.get_hash(self.path_info)
+        return self.cache.get_hash(self.tree, self.path_info)
 
     @property
     def is_dir_checksum(self):
-        return self.tree.is_dir_hash(self.checksum)
+        return self.hash_info.isdir
 
     @property
     def exists(self):
         return self.tree.exists(self.path_info)
 
-    def save_info(self):
-        return self.tree.save_info(self.path_info)
-
     def changed_checksum(self):
-        return self.checksum != self.get_checksum()
+        return self.hash_info != self.get_hash()
 
     def changed_cache(self, filter_info=None):
-        if not self.use_cache or not self.checksum:
+        if not self.use_cache or not self.hash_info:
             return True
 
-        return self.cache.changed_cache(self.checksum, filter_info=filter_info)
-
-    def status(self):
-        if self.checksum and self.use_cache and self.changed_cache():
-            return {str(self): "not in cache"}
+        return self.cache.changed_cache(
+            self.hash_info, filter_info=filter_info
+        )
 
+    def workspace_status(self):
         if not self.exists:
             return {str(self): "deleted"}
 
         if self.changed_checksum():
             return {str(self): "modified"}
 
-        if not self.checksum:
+        if not self.hash_info:
             return {str(self): "new"}
 
         return {}
 
+    def status(self):
+        if self.hash_info and self.use_cache and self.changed_cache():
+            return {str(self): "not in cache"}
+
+        return self.workspace_status()
+
     def changed(self):
         status = self.status()
         logger.debug(str(status))
@@ -264,7 +265,7 @@ class BaseOutput:
             self.verify_metric()
 
         if not self.use_cache:
-            self.info = self.save_info()
+            self.hash_info = self.get_hash()
             if not self.IS_DEPENDENCY:
                 logger.debug(
                     "Output '%s' doesn't use cache. Skipping saving.", self
@@ -277,14 +278,15 @@ class BaseOutput:
             logger.debug("Output '%s' didn't change. Skipping saving.", self)
             return
 
-        self.info = self.save_info()
+        self.hash_info = self.get_hash()
 
     def commit(self):
+        assert self.hash_info
         if self.use_cache:
-            self.cache.save(self.path_info, self.cache.tree, self.info)
+            self.cache.save(self.path_info, self.cache.tree, self.hash_info)
 
     def dumpd(self):
-        ret = copy(self.info)
+        ret = copy(self.hash_info.to_dict())
         ret[self.PARAM_PATH] = self.def_path
 
         if self.IS_DEPENDENCY:
@@ -333,7 +335,7 @@ class BaseOutput:
 
         return self.cache.checkout(
             self.path_info,
-            self.info,
+            self.hash_info,
             force=force,
             progress_callback=progress_callback,
             relink=relink,
@@ -367,7 +369,7 @@ class BaseOutput:
             return 0
 
         return self.cache.get_files_number(
-            self.path_info, self.checksum, filter_info
+            self.path_info, self.hash_info, filter_info
         )
 
     def unprotect(self):
@@ -377,9 +379,9 @@ class BaseOutput:
     def get_dir_cache(self, **kwargs):
         if not self.is_dir_checksum:
             raise DvcException("cannot get dir cache for file checksum")
-        if self.cache.changed_cache_file(self.checksum):
+        if self.cache.changed_cache_file(self.hash_info):
             self.repo.cloud.pull(
-                NamedCache.make("local", self.checksum, str(self)),
+                NamedCache.make("local", self.hash_info.value, str(self)),
                 show_checksums=False,
                 **kwargs,
             )
@@ -414,7 +416,7 @@ class BaseOutput:
         except DvcException:
             logger.debug(f"failed to pull cache for '{self}'")
 
-        if self.cache.changed_cache_file(self.checksum):
+        if self.cache.changed_cache_file(self.hash_info):
             msg = (
                 "Missing cache for directory '{}'. "
                 "Cache for files inside will be lost. "
@@ -464,7 +466,7 @@ class BaseOutput:
             cache.external[dep.repo_pair].add(dep.def_path)
             return cache
 
-        if not self.checksum:
+        if not self.hash_info:
             msg = (
                 "Output '{}'({}) is missing version info. "
                 "Cache for it will not be collected. "
@@ -483,20 +485,61 @@ class BaseOutput:
             logger.warning(msg)
             return NamedCache()
 
-        ret = NamedCache.make(self.scheme, self.checksum, str(self))
+        ret = NamedCache.make(self.scheme, self.hash_info.value, str(self))
 
         if not self.is_dir_checksum:
             return ret
 
         ret.add_child_cache(
-            self.checksum, self.collect_used_dir_cache(**kwargs),
+            self.hash_info.value, self.collect_used_dir_cache(**kwargs),
         )
 
         return ret
 
     @classmethod
-    def _validate_output_path(cls, path):
+    def _validate_output_path(cls, path, stage=None):
         from dvc.dvcfile import is_valid_filename
 
         if is_valid_filename(path):
             raise cls.IsStageFileError(path)
+
+        if stage:
+            check = stage.repo.tree.dvcignore.check_ignore(path)
+            if check.match:
+                raise cls.IsIgnoredError(check)
+
+    def _check_can_merge(self, out):
+        if self.scheme != out.scheme:
+            raise MergeError("unable to auto-merge outputs of different types")
+
+        my = self.dumpd()
+        other = out.dumpd()
+
+        my.pop(self.tree.PARAM_CHECKSUM)
+        other.pop(self.tree.PARAM_CHECKSUM)
+
+        if my != other:
+            raise MergeError(
+                "unable to auto-merge outputs with different options"
+            )
+
+        if not out.is_dir_checksum:
+            raise MergeError(
+                "unable to auto-merge outputs that are not directories"
+            )
+
+    def merge(self, ancestor, other):
+        assert other
+
+        if ancestor:
+            self._check_can_merge(ancestor)
+            ancestor_info = ancestor.hash_info
+        else:
+            ancestor_info = None
+
+        self._check_can_merge(self)
+        self._check_can_merge(other)
+
+        self.hash_info = self.cache.merge(
+            ancestor_info, self.hash_info, other.hash_info
+        )

dvc/output/local.py

@@ -87,8 +87,9 @@ class LocalOutput(BaseOutput):
 
         name = "metrics" if self.metric else "plot"
         if os.path.isdir(path):
-            msg = "directory '{}' cannot be used as {}."
-            raise IsADirectoryError(msg.format(self.path_info, name))
+            msg = "directory '%s' cannot be used as %s."
+            logger.debug(msg, str(self.path_info), name)
+            return
 
         if not istextfile(path):
             msg = "binary file '{}' cannot be used as {}."

dvc/path_info.py

@@ -253,6 +253,15 @@ class HTTPURLInfo(URLInfo):
         self.query = p.query
         self.fragment = p.fragment
 
+    def replace(self, path=None):
+        return self.from_parts(
+            *self._base_parts,
+            params=self.params,
+            query=self.query,
+            fragment=self.fragment,
+            path=path,
+        )
+
     @classmethod
     def from_parts(
         cls,
@@ -315,3 +324,11 @@ class HTTPURLInfo(URLInfo):
             and self._path == other._path
             and self._extra_parts == other._extra_parts
         )
+
+
+class WebDAVURLInfo(URLInfo):
+    @cached_property
+    def url(self):
+        return "{}://{}{}".format(
+            self.scheme.replace("webdav", "http"), self.netloc, self._spath
+        )

dvc/pathspec_math.py

@@ -3,11 +3,14 @@
 # of two path specification patterns with different base
 # All the operations follow the documents of `gitignore`
 import os
+from collections import namedtuple
 
 from pathspec.util import normalize_file
 
 from dvc.utils import relpath
 
+PatternInfo = namedtuple("PatternInfo", ["patterns", "file_info"])
+
 
 def _not_ignore(rule):
     return (True, rule[1:]) if rule.startswith("!") else (False, rule)
@@ -59,7 +62,10 @@ def _change_dirname(dirname, pattern_list, new_dirname):
     if rel.startswith(".."):
         raise ValueError("change dirname can only change to parent path")
 
-    return [change_rule(rule, rel) for rule in pattern_list]
+    return [
+        PatternInfo(change_rule(rule.patterns, rel), rule.file_info)
+        for rule in pattern_list
+    ]
 
 
 def merge_patterns(pattern_a, prefix_a, pattern_b, prefix_b):

dvc/prompt.py

@@ -7,7 +7,7 @@ from getpass import getpass
 logger = logging.getLogger(__name__)
 
 
-def _ask(prompt, limited_to=None):
+def ask(prompt, limited_to=None):
     if not sys.stdout.isatty():
         return None
 
@@ -39,7 +39,7 @@ def confirm(statement):
         bool: whether or not specified statement was confirmed.
     """
     prompt = f"{statement} [y/n]"
-    answer = _ask(prompt, limited_to=["yes", "no", "y", "n"])
+    answer = ask(prompt, limited_to=["yes", "no", "y", "n"])
     return answer and answer.startswith("y")
 
 

dvc/repo/__init__.py

@@ -15,7 +15,7 @@ from dvc.exceptions import (
     OutputNotFoundError,
 )
 from dvc.path_info import PathInfo
-from dvc.repo.tree import RepoTree
+from dvc.tree.repo import RepoTree
 from dvc.utils.fs import path_isin
 
 from ..stage.exceptions import StageFileDoesNotExistError, StageNotFound
@@ -25,16 +25,30 @@ from .graph import check_acyclic, get_pipeline, get_pipelines
 logger = logging.getLogger(__name__)
 
 
+@contextmanager
+def lock_repo(repo):
+    # pylint: disable=protected-access
+    depth = getattr(repo, "_lock_depth", 0)
+    repo._lock_depth = depth + 1
+
+    try:
+        if depth > 0:
+            yield
+        else:
+            with repo.lock, repo.state:
+                repo._reset()
+                yield
+                # Graph cache is no longer valid after we release the repo.lock
+                repo._reset()
+    finally:
+        repo._lock_depth = depth
+
+
 def locked(f):
     @wraps(f)
     def wrapper(repo, *args, **kwargs):
-        with repo.lock, repo.state:
-            # pylint: disable=protected-access
-            repo._reset()
-            ret = f(repo, *args, **kwargs)
-            # Our graph cache is no longer valid after we release the repo.lock
-            repo._reset()
-            return ret
+        with lock_repo(repo):
+            return f(repo, *args, **kwargs)
 
     return wrapper
 
@@ -42,43 +56,43 @@ def locked(f):
 class Repo:
     DVC_DIR = ".dvc"
 
-    from dvc.repo.destroy import destroy
-    from dvc.repo.install import install
     from dvc.repo.add import add
-    from dvc.repo.remove import remove
-    from dvc.repo.ls import ls
-    from dvc.repo.freeze import freeze, unfreeze
-    from dvc.repo.move import move
-    from dvc.repo.run import run
-    from dvc.repo.imp import imp
-    from dvc.repo.imp_url import imp_url
-    from dvc.repo.reproduce import reproduce
-    from dvc.repo.checkout import _checkout
-    from dvc.repo.push import push
-    from dvc.repo.fetch import _fetch
-    from dvc.repo.pull import pull
-    from dvc.repo.status import status
-    from dvc.repo.gc import gc
-    from dvc.repo.commit import commit
-    from dvc.repo.diff import diff
     from dvc.repo.brancher import brancher
+    from dvc.repo.checkout import checkout
+    from dvc.repo.commit import commit
+    from dvc.repo.destroy import destroy
+    from dvc.repo.diff import diff
+    from dvc.repo.fetch import fetch
+    from dvc.repo.freeze import freeze, unfreeze
+    from dvc.repo.gc import gc
     from dvc.repo.get import get
     from dvc.repo.get_url import get_url
+    from dvc.repo.imp import imp
+    from dvc.repo.imp_url import imp_url
+    from dvc.repo.install import install
+    from dvc.repo.ls import ls
+    from dvc.repo.move import move
+    from dvc.repo.pull import pull
+    from dvc.repo.push import push
+    from dvc.repo.remove import remove
+    from dvc.repo.reproduce import reproduce
+    from dvc.repo.run import run
+    from dvc.repo.status import status
     from dvc.repo.update import update
 
-    def __init__(self, root_dir=None, scm=None, rev=None):
-        from dvc.state import State, StateNoop
-        from dvc.lock import make_lock
-        from dvc.scm import SCM
+    def __init__(self, root_dir=None, scm=None, rev=None, subrepos=False):
         from dvc.cache import Cache
         from dvc.data_cloud import DataCloud
+        from dvc.lock import make_lock
         from dvc.repo.experiments import Experiments
         from dvc.repo.metrics import Metrics
-        from dvc.repo.plots import Plots
         from dvc.repo.params import Params
+        from dvc.repo.plots import Plots
+        from dvc.scm import SCM
+        from dvc.stage.cache import StageCache
+        from dvc.state import State, StateNoop
         from dvc.tree.local import LocalTree
         from dvc.utils.fs import makedirs
-        from dvc.stage.cache import StageCache
 
         if scm:
             tree = scm.get_tree(rev)
@@ -116,6 +130,8 @@ class Repo:
             hardlink_lock=hardlink_lock,
             friendly=True,
         )
+        # used by RepoTree to determine if it should traverse subrepos
+        self.subrepos = subrepos
 
         self.cache = Cache(self)
         self.cloud = DataCloud(self)
@@ -441,10 +457,11 @@ class Repo:
         """
         import networkx as nx
         from pygtrie import Trie
+
         from dvc.exceptions import (
             OutputDuplicationError,
-            StagePathAsOutputError,
             OverlappingOutputPathsError,
+            StagePathAsOutputError,
         )
 
         G = nx.DiGraph()
@@ -580,19 +597,20 @@ class Repo:
         path_parts = os.path.normpath(path).split(os.path.sep)
         return self.DVC_DIR in path_parts
 
+    @cached_property
+    def repo_tree(self):
+        return RepoTree(self, subrepos=self.subrepos, fetch=True)
+
     @contextmanager
     def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
         """Opens a specified resource as a file descriptor"""
 
-        tree = RepoTree(self, stream=True)
+        tree = RepoTree(self, stream=True, subrepos=True)
         path = os.path.join(self.root_dir, path)
         try:
             with self.state:
                 with tree.open(
-                    os.path.join(self.root_dir, path),
-                    mode=mode,
-                    encoding=encoding,
-                    remote=remote,
+                    path, mode=mode, encoding=encoding, remote=remote,
                 ) as fobj:
                     yield fobj
         except FileNotFoundError as exc:
@@ -603,16 +621,7 @@ class Repo:
     def close(self):
         self.scm.close()
 
-    @locked
-    def checkout(self, *args, **kwargs):
-        return self._checkout(*args, **kwargs)
-
-    @locked
-    def fetch(self, *args, **kwargs):
-        return self._fetch(*args, **kwargs)
-
     def _reset(self):
         self.__dict__.pop("graph", None)
         self.__dict__.pop("stages", None)
         self.__dict__.pop("pipelines", None)
-        self.tree.__dict__.pop("dvcignore", None)

dvc/repo/checkout.py

@@ -9,6 +9,8 @@ from dvc.exceptions import (
 from dvc.progress import Tqdm
 from dvc.utils import relpath
 
+from . import locked
+
 logger = logging.getLogger(__name__)
 
 
@@ -36,7 +38,8 @@ def get_all_files_numbers(pairs):
     )
 
 
-def _checkout(
+@locked
+def checkout(
     self,
     targets=None,
     with_deps=False,

dvc/repo/commit.py

@@ -45,5 +45,5 @@ def commit(self, target, with_deps=False, recursive=False, force=False):
             stage.save()
         stage.commit()
 
-        Dvcfile(self, stage.path).dump(stage)
+        Dvcfile(self, stage.path).dump(stage, update_pipeline=False)
     return stages

dvc/repo/diff.py

@@ -1,7 +1,11 @@
+import logging
 import os
 
 from dvc.repo import locked
 from dvc.tree.local import LocalTree
+from dvc.tree.repo import RepoTree
+
+logger = logging.getLogger(__name__)
 
 
 @locked
@@ -14,64 +18,32 @@ def diff(self, a_rev="HEAD", b_rev=None):
     `dvc diff` would be the same as `dvc diff HEAD`.
     """
 
-    def _paths_checksums():
-        """
-        A dictionary of checksums addressed by relpaths collected from
-        the current tree outputs.
-
-        To help distinguish between a directory and a file output,
-        the former one will come with a trailing slash in the path:
-
-            directory: "data/"
-            file:      "data"
-        """
-
-        def _to_path(output):
-            return (
-                str(output)
-                if not output.is_dir_checksum
-                else os.path.join(str(output), "")
-            )
-
-        on_working_tree = isinstance(self.tree, LocalTree)
-
-        def _to_checksum(output):
-            if on_working_tree:
-                return self.cache.local.tree.get_hash(output.path_info)
-            return output.checksum
-
-        def _exists(output):
-            if on_working_tree:
-                return output.exists
-            return True
-
-        return {
-            _to_path(output): _to_checksum(output)
-            for stage in self.stages
-            for output in stage.outs
-            if _exists(output)
-        }
-
     if self.scm.no_commits:
         return {}
 
-    working_tree = self.tree
-    a_tree = self.scm.get_tree(a_rev)
-    b_tree = self.scm.get_tree(b_rev) if b_rev else working_tree
+    b_rev = b_rev if b_rev else "workspace"
+    results = {}
+    for rev in self.brancher(revs=[a_rev, b_rev]):
+        if rev == "workspace" and rev != b_rev:
+            # brancher always returns workspace, but we only need to compute
+            # workspace paths/checksums if b_rev was None
+            continue
+        results[rev] = _paths_checksums(self)
 
-    try:
-        self.tree = a_tree
-        old = _paths_checksums()
-
-        self.tree = b_tree
-        new = _paths_checksums()
-    finally:
-        self.tree = working_tree
+    old = results[a_rev]
+    new = results[b_rev]
 
     # Compare paths between the old and new tree.
     # set() efficiently converts dict keys to a set
     added = sorted(set(new) - set(old))
-    deleted = sorted(set(old) - set(new))
+    deleted_or_missing = set(old) - set(new)
+    if b_rev == "workspace":
+        # missing status is only applicable when diffing local workspace
+        # against a commit
+        missing = sorted(_filter_missing(self, deleted_or_missing))
+    else:
+        missing = []
+    deleted = sorted(deleted_or_missing - set(missing))
     modified = sorted(set(old) & set(new))
 
     ret = {
@@ -82,6 +54,73 @@ def diff(self, a_rev="HEAD", b_rev=None):
             for path in modified
             if old[path] != new[path]
         ],
+        "not in cache": [
+            {"path": path, "hash": old[path]} for path in missing
+        ],
     }
 
     return ret if any(ret.values()) else {}
+
+
+def _paths_checksums(repo):
+    """
+    A dictionary of checksums addressed by relpaths collected from
+    the current tree outputs.
+
+    To help distinguish between a directory and a file output,
+    the former one will come with a trailing slash in the path:
+
+        directory: "data/"
+        file:      "data"
+    """
+
+    return dict(_output_paths(repo))
+
+
+def _output_paths(repo):
+    repo_tree = RepoTree(repo, stream=True)
+    on_working_tree = isinstance(repo.tree, LocalTree)
+
+    def _exists(output):
+        if on_working_tree:
+            return output.exists
+        return True
+
+    def _to_path(output):
+        return (
+            str(output)
+            if not output.is_dir_checksum
+            else os.path.join(str(output), "")
+        )
+
+    def _to_checksum(output):
+        if on_working_tree:
+            return repo.cache.local.tree.get_hash(output.path_info).value
+        return output.hash_info.value
+
+    for stage in repo.stages:
+        for output in stage.outs:
+            if _exists(output):
+                yield _to_path(output), _to_checksum(output)
+                if output.is_dir_checksum:
+                    yield from _dir_output_paths(repo_tree, output)
+
+
+def _dir_output_paths(repo_tree, output):
+    from dvc.config import NoRemoteError
+
+    try:
+        for fname in repo_tree.walk_files(output.path_info):
+            yield str(fname), repo_tree.get_hash(fname).value
+    except NoRemoteError:
+        logger.warning("dir cache entry for '%s' is missing", output)
+
+
+def _filter_missing(repo, paths):
+    repo_tree = RepoTree(repo, stream=True)
+    for path in paths:
+        metadata = repo_tree.metadata(path)
+        if metadata.is_dvc:
+            out = metadata.outs[0]
+            if out.status()[str(out)] == "not in cache":
+                yield path

dvc/repo/experiments/__init__.py

@@ -1,21 +1,67 @@
 import logging
 import os
+import re
 import tempfile
+from collections import namedtuple
+from collections.abc import Mapping
+from concurrent.futures import (
+    ProcessPoolExecutor,
+    ThreadPoolExecutor,
+    as_completed,
+)
 from contextlib import contextmanager
+from functools import partial, wraps
+from typing import Iterable, Optional
 
-from funcy import cached_property
+from funcy import cached_property, first
 
-from dvc.exceptions import DvcException
+from dvc.exceptions import DownloadError, DvcException, UploadError
+from dvc.path_info import PathInfo
+from dvc.progress import Tqdm
+from dvc.repo.experiments.executor import ExperimentExecutor, LocalExecutor
 from dvc.scm.git import Git
 from dvc.stage.serialize import to_lockfile
+from dvc.tree.repo import RepoTree
 from dvc.utils import dict_sha256, env2bool, relpath
 from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
 
+def scm_locked(f):
+    # Lock the experiments workspace so that we don't try to perform two
+    # different sequences of git operations at once
+    @wraps(f)
+    def wrapper(exp, *args, **kwargs):
+        with exp.scm_lock:
+            return f(exp, *args, **kwargs)
+
+    return wrapper
+
+
+def hash_exp(stages):
+    exp_data = {}
+    for stage in stages:
+        exp_data.update(to_lockfile(stage))
+    return dict_sha256(exp_data)
+
+
 class UnchangedExperimentError(DvcException):
-    pass
+    def __init__(self, rev):
+        super().__init__(f"Experiment identical to baseline '{rev[:7]}'.")
+        self.rev = rev
+
+
+class BaselineMismatchError(DvcException):
+    def __init__(self, rev, expected):
+        if hasattr(rev, "hexsha"):
+            rev = rev.hexsha
+        rev_str = f"{rev[:7]}" if rev is not None else "dangling commit"
+        super().__init__(
+            f"Experiment derived from '{rev_str}', expected '{expected[:7]}'."
+        )
+        self.rev = rev
+        self.expected_rev = expected
 
 
 class Experiments:
@@ -26,8 +72,21 @@ class Experiments:
     """
 
     EXPERIMENTS_DIR = "experiments"
+    PACKED_ARGS_FILE = "repro.dat"
+    STASH_MSG_PREFIX = "dvc-exp:"
+    STASH_EXPERIMENT_RE = re.compile(
+        r"(?:On \(.*\): )"
+        r"dvc-exp:(?P<baseline_rev>[0-9a-f]+)(:(?P<branch>.+))?$"
+    )
+    BRANCH_RE = re.compile(
+        r"^(?P<baseline_rev>[a-f0-9]{7})-(?P<exp_sha>[a-f0-9]+)$"
+    )
+
+    StashEntry = namedtuple("StashEntry", ["index", "baseline_rev", "branch"])
 
     def __init__(self, repo):
+        from dvc.lock import make_lock
+
         if not (
             env2bool("DVC_TEST")
             or repo.config["core"].get("experiments", False)
@@ -35,6 +94,10 @@ class Experiments:
             raise NotImplementedError
 
         self.repo = repo
+        self.scm_lock = make_lock(
+            os.path.join(self.repo.tmp_dir, "exp_scm_lock"),
+            tmp_dir=self.repo.tmp_dir,
+        )
 
     @cached_property
     def exp_dir(self):
@@ -47,10 +110,13 @@ class Experiments:
             return Git(self.exp_dir)
         return self._init_clone()
 
+    @cached_property
+    def dvc_dir(self):
+        return relpath(self.repo.dvc_dir, self.repo.scm.root_dir)
+
     @cached_property
     def exp_dvc_dir(self):
-        dvc_dir = relpath(self.repo.dvc_dir, self.repo.scm.root_dir)
-        return os.path.join(self.exp_dir, dvc_dir)
+        return os.path.join(self.exp_dir, self.dvc_dir)
 
     @cached_property
     def exp_dvc(self):
@@ -59,20 +125,34 @@ class Experiments:
 
         return Repo(self.exp_dvc_dir)
 
-    @staticmethod
-    def exp_hash(stages):
-        exp_data = {}
-        for stage in stages:
-            exp_data.update(to_lockfile(stage))
-        return dict_sha256(exp_data)
-
     @contextmanager
     def chdir(self):
         cwd = os.getcwd()
         os.chdir(self.exp_dvc.root_dir)
-        yield
+        yield self.exp_dvc.root_dir
         os.chdir(cwd)
 
+    @cached_property
+    def args_file(self):
+        return os.path.join(self.exp_dvc.tmp_dir, self.PACKED_ARGS_FILE)
+
+    @property
+    def stash_reflog(self):
+        if "refs/stash" in self.scm.repo.refs:
+            return self.scm.repo.refs["refs/stash"].log()
+        return []
+
+    @property
+    def stash_revs(self):
+        revs = {}
+        for i, entry in enumerate(self.stash_reflog):
+            m = self.STASH_EXPERIMENT_RE.match(entry.message)
+            if m:
+                revs[entry.newhexsha] = self.StashEntry(
+                    i, m.group("baseline_rev"), m.group("branch")
+                )
+        return revs
+
     def _init_clone(self):
         src_dir = self.repo.scm.root_dir
         logger.debug("Initializing experiments clone")
@@ -90,96 +170,514 @@ class Experiments:
 
     def _scm_checkout(self, rev):
         self.scm.repo.git.reset(hard=True)
+        if self.scm.repo.head.is_detached:
+            self._checkout_default_branch()
         if not Git.is_sha(rev) or not self.scm.has_rev(rev):
-            self.scm.fetch(all=True)
-        logger.debug("Checking out base experiment commit '%s'", rev)
+            self.scm.pull()
+        logger.debug("Checking out experiment commit '%s'", rev)
         self.scm.checkout(rev)
 
-    def _patch_exp(self):
-        """Create a patch based on the current (parent) workspace and apply it
-        to the experiment workspace.
-        """
-        tmp = tempfile.NamedTemporaryFile(delete=False).name
-        try:
-            self.repo.scm.repo.git.diff(patch=True, output=tmp)
-            if os.path.getsize(tmp):
-                logger.debug("Patching experiment workspace")
-                self.scm.repo.git.apply(tmp)
-            else:
-                raise UnchangedExperimentError(
-                    "Experiment identical to baseline commit."
-                )
-        finally:
-            remove(tmp)
+    def _checkout_default_branch(self):
+        from git.refs.symbolic import SymbolicReference
 
-    def _commit(self, stages, check_exists=True, branch=True, rev=None):
-        """Commit stages as an experiment and return the commit SHA."""
-        hash_ = self.exp_hash(stages)
-        exp_name = f"{rev[:7]}-{hash_}"
+        # switch to default branch
+        git_repo = self.scm.repo
+        origin_refs = git_repo.remotes["origin"].refs
+
+        # origin/HEAD will point to tip of the default branch unless we
+        # initially cloned a repo that was in a detached-HEAD state.
+        #
+        # If we are currently detached because we cloned a detached
+        # repo, we can't actually tell what branch should be considered
+        # default, so we just fall back to the first available reference.
+        if "HEAD" in origin_refs:
+            ref = origin_refs["HEAD"].reference
+        else:
+            ref = origin_refs[0]
+            if not isinstance(ref, SymbolicReference):
+                ref = ref.reference
+        branch_name = ref.name.split("/")[-1]
+
+        if branch_name in git_repo.heads:
+            branch = git_repo.heads[branch_name]
+        else:
+            branch = git_repo.create_head(branch_name, ref)
+            branch.set_tracking_branch(ref)
+        branch.checkout()
+
+    def _stash_exp(
+        self,
+        *args,
+        params: Optional[dict] = None,
+        branch: Optional[str] = None,
+        allow_unchanged: Optional[bool] = False,
+        apply_workspace: Optional[bool] = True,
+        **kwargs,
+    ):
+        """Stash changes from the current (parent) workspace as an experiment.
+
+        Args:
+            params: Optional dictionary of parameter values to be used.
+                Values take priority over any parameters specified in the
+                user's workspace.
+            branch: Optional experiment branch name. If specified, the
+                experiment will be added to `branch` instead of creating
+                a new branch.
+            allow_unchanged: Force experiment reproduction even if params are
+                unchanged from the baseline.
+            apply_workspace: Apply changes from the user workspace to the
+                experiment workspace.
+        """
         if branch:
+            rev = self.scm.resolve_rev(branch)
+        else:
+            rev = self.scm.get_rev()
+
+        if apply_workspace:
+            # patch user's workspace into experiments clone
+            #
+            # TODO: patching changes into an extended branch will require
+            # propagating changes down the full branch, since the workspace
+            # patch will be based on the original branch point, not the tip
+            # of the experiment branch
+            tmp = tempfile.NamedTemporaryFile(delete=False).name
+            try:
+                self.repo.scm.repo.git.diff(
+                    patch=True, full_index=True, binary=True, output=tmp
+                )
+                if os.path.getsize(tmp):
+                    logger.debug("Patching experiment workspace")
+                    self.scm.repo.git.apply(tmp)
+                elif not params and not allow_unchanged:
+                    # experiment matches original baseline
+                    raise UnchangedExperimentError(rev)
+            finally:
+                remove(tmp)
+
+        # update experiment params from command line
+        if params:
+            self._update_params(params)
+
+        # save additional repro command line arguments
+        self._pack_args(*args, **kwargs)
+
+        # save experiment as a stash commit w/message containing baseline rev
+        # (stash commits are merge commits and do not contain a parent commit
+        # SHA)
+        msg = self._stash_msg(rev, branch)
+        self.scm.repo.git.stash("push", "-m", msg)
+        return self.scm.resolve_rev("stash@{0}")
+
+    def _stash_msg(self, rev, branch=None):
+        if branch:
+            return f"{self.STASH_MSG_PREFIX}{rev}:{branch}"
+        return f"{self.STASH_MSG_PREFIX}{rev}"
+
+    def _pack_args(self, *args, **kwargs):
+        ExperimentExecutor.pack_repro_args(self.args_file, *args, **kwargs)
+        self.scm.add(self.args_file)
+
+    def _unpack_args(self, tree=None):
+        return ExperimentExecutor.unpack_repro_args(self.args_file, tree=tree)
+
+    def _update_params(self, params: dict):
+        """Update experiment params files with the specified values."""
+        from dvc.utils.serialize import MODIFIERS
+
+        logger.debug("Using experiment params '%s'", params)
+
+        # recursive dict update
+        def _update(dict_, other):
+            for key, value in other.items():
+                if isinstance(value, Mapping):
+                    dict_[key] = _update(dict_.get(key, {}), value)
+                else:
+                    dict_[key] = value
+            return dict_
+
+        for params_fname in params:
+            path = PathInfo(self.exp_dvc.root_dir) / params_fname
+            suffix = path.suffix.lower()
+            modify_data = MODIFIERS[suffix]
+            with modify_data(path, tree=self.exp_dvc.tree) as data:
+                _update(data, params[params_fname])
+
+    def _commit(self, exp_hash, check_exists=True, create_branch=True):
+        """Commit stages as an experiment and return the commit SHA."""
+        if not self.scm.is_dirty():
+            raise UnchangedExperimentError(self.scm.get_rev())
+
+        rev = self.scm.get_rev()
+        exp_name = f"{rev[:7]}-{exp_hash}"
+        if create_branch:
             if check_exists and exp_name in self.scm.list_branches():
                 logger.debug("Using existing experiment branch '%s'", exp_name)
                 return self.scm.resolve_rev(exp_name)
             self.scm.checkout(exp_name, create_new=True)
-        logger.debug("Commit new experiment branch '%s'", exp_name)
+            logger.debug("Commit new experiment branch '%s'", exp_name)
+        else:
+            logger.debug("Commit to current experiment branch")
         self.scm.repo.git.add(A=True)
         self.scm.commit(f"Add experiment {exp_name}")
         return self.scm.get_rev()
 
-    def _reproduce(self, *args, **kwargs):
-        """Run `dvc repro` inside the experiments workspace."""
-        with self.chdir():
-            return self.exp_dvc.reproduce(*args, **kwargs)
+    def reproduce_one(self, queue=False, **kwargs):
+        """Reproduce and checkout a single experiment."""
+        stash_rev = self.new(**kwargs)
+        if queue:
+            logger.info(
+                "Queued experiment '%s' for future execution.", stash_rev[:7]
+            )
+            return [stash_rev]
+        results = self.reproduce([stash_rev], keep_stash=False)
+        exp_rev = first(results)
+        if exp_rev is not None:
+            self.checkout_exp(exp_rev)
+        return results
 
-    def new(self, *args, workspace=True, **kwargs):
+    def reproduce_queued(self, **kwargs):
+        results = self.reproduce(**kwargs)
+        if results:
+            revs = [f"{rev[:7]}" for rev in results]
+            logger.info(
+                "Successfully reproduced experiment(s) '%s'.\n"
+                "Use `dvc exp checkout <exp_rev>` to apply the results of "
+                "a specific experiment to your workspace.",
+                ", ".join(revs),
+            )
+        return results
+
+    @scm_locked
+    def new(self, *args, **kwargs):
         """Create a new experiment.
 
         Experiment will be reproduced and checked out into the user's
         workspace.
         """
-        rev = self.repo.scm.get_rev()
-        self._scm_checkout(rev)
-        if workspace:
-            try:
-                self._patch_exp()
-            except UnchangedExperimentError as exc:
-                logger.info("Reproducing existing experiment '%s'.", rev[:7])
-                raise exc
+        branch = kwargs.get("branch")
+        if branch:
+            rev = self.scm.resolve_rev(branch)
+            logger.debug(
+                "Using '%s' (tip of branch '%s') as baseline", rev, branch
+            )
         else:
-            # configure params via command line here
-            pass
-        self.exp_dvc.checkout()
-        stages = self._reproduce(*args, **kwargs)
-        exp_rev = self._commit(stages, rev=rev)
-        self.checkout_exp(exp_rev, force=True)
-        logger.info("Generated experiment '%s'.", exp_rev[:7])
-        return stages
+            rev = self.repo.scm.get_rev()
+        self._scm_checkout(rev)
+        try:
+            stash_rev = self._stash_exp(*args, **kwargs)
+        except UnchangedExperimentError as exc:
+            logger.info("Reproducing existing experiment '%s'.", rev[:7])
+            raise exc
+        logger.debug(
+            "Stashed experiment '%s' for future execution.", stash_rev[:7]
+        )
+        return stash_rev
 
-    def checkout_exp(self, rev, force=False):
+    @scm_locked
+    def reproduce(
+        self,
+        revs: Optional[Iterable] = None,
+        keep_stash: Optional[bool] = True,
+        **kwargs,
+    ):
+        """Reproduce the specified experiments.
+
+        Args:
+            revs: If revs is not specified, all stashed experiments will be
+                reproduced.
+            keep_stash: If True, stashed experiments will be preserved if they
+                fail to reproduce successfully.
+        """
+        stash_revs = self.stash_revs
+
+        # to_run contains mapping of:
+        #   input_rev: (stash_index, baseline_rev)
+        # where input_rev contains the changes to execute (usually a stash
+        # commit) and baseline_rev is the baseline to compare output against.
+        # The final experiment commit will be branched from baseline_rev.
+        if revs is None:
+            to_run = dict(stash_revs)
+        else:
+            to_run = {
+                rev: stash_revs[rev]
+                if rev in stash_revs
+                else self.StashEntry(None, rev, None)
+                for rev in revs
+            }
+
+        logger.debug(
+            "Reproducing experiment revs '%s'",
+            ", ".join((rev[:7] for rev in to_run)),
+        )
+
+        # setup executors - unstash experiment, generate executor, upload
+        # contents of (unstashed) exp workspace to the executor tree
+        executors = {}
+        for rev, item in to_run.items():
+            self._scm_checkout(item.baseline_rev)
+            self.scm.repo.git.stash("apply", rev)
+            repro_args, repro_kwargs = self._unpack_args()
+            executor = LocalExecutor(
+                item.baseline_rev,
+                branch=item.branch,
+                repro_args=repro_args,
+                repro_kwargs=repro_kwargs,
+                dvc_dir=self.dvc_dir,
+                cache_dir=self.repo.cache.local.cache_dir,
+            )
+            self._collect_input(executor)
+            executors[rev] = executor
+
+        exec_results = self._reproduce(executors, **kwargs)
+
+        if keep_stash:
+            # only drop successfully run stashed experiments
+            to_drop = sorted(
+                (
+                    stash_revs[rev][0]
+                    for rev in exec_results
+                    if rev in stash_revs
+                ),
+                reverse=True,
+            )
+        else:
+            # drop all stashed experiments
+            to_drop = sorted(
+                (stash_revs[rev][0] for rev in to_run if rev in stash_revs),
+                reverse=True,
+            )
+        for index in to_drop:
+            self.scm.repo.git.stash("drop", index)
+
+        result = {}
+        for _, exp_result in exec_results.items():
+            result.update(exp_result)
+        return result
+
+    def _reproduce(self, executors: dict, jobs: Optional[int] = 1) -> dict:
+        """Run dvc repro for the specified ExperimentExecutors in parallel.
+
+        Returns dict containing successfully executed experiments.
+        """
+        result = {}
+
+        with ProcessPoolExecutor(max_workers=jobs) as workers:
+            futures = {}
+            for rev, executor in executors.items():
+                future = workers.submit(
+                    executor.reproduce,
+                    executor.dvc_dir,
+                    cwd=executor.dvc.root_dir,
+                    **executor.repro_kwargs,
+                )
+                futures[future] = (rev, executor)
+            for future in as_completed(futures):
+                rev, executor = futures[future]
+                exc = future.exception()
+                if exc is None:
+                    exp_hash = future.result()
+                    if executor.branch:
+                        self._scm_checkout(executor.branch)
+                    else:
+                        self._scm_checkout(executor.baseline_rev)
+                    try:
+                        self._collect_output(executor)
+                    except DownloadError:
+                        logger.error(
+                            "Failed to collect output for experiment '%s'",
+                            rev,
+                        )
+                        continue
+                    finally:
+                        if os.path.exists(self.args_file):
+                            remove(self.args_file)
+
+                    try:
+                        branch = not executor.branch
+                        exp_rev = self._commit(exp_hash, create_branch=branch)
+                    except UnchangedExperimentError:
+                        logger.debug(
+                            "Experiment '%s' identical to baseline '%s'",
+                            rev,
+                            executor.baseline_rev,
+                        )
+                        exp_rev = executor.baseline_rev
+                    logger.info("Reproduced experiment '%s'.", exp_rev[:7])
+                    result[rev] = {exp_rev: exp_hash}
+                else:
+                    logger.exception(
+                        "Failed to reproduce experiment '%s'", rev
+                    )
+                executor.cleanup()
+
+        return result
+
+    def _collect_input(self, executor: ExperimentExecutor):
+        """Copy (upload) input from the experiments workspace to the executor
+        tree.
+        """
+        logger.debug("Collecting input for '%s'", executor.tmp_dir)
+        repo_tree = RepoTree(self.exp_dvc)
+        self._process(
+            executor.tree,
+            self.exp_dvc.tree,
+            executor.collect_files(self.exp_dvc.tree, repo_tree),
+        )
+
+    def _collect_output(self, executor: ExperimentExecutor):
+        """Copy (download) output from the executor tree into experiments
+        workspace.
+        """
+        logger.debug("Collecting output from '%s'", executor.tmp_dir)
+        self._process(
+            self.exp_dvc.tree,
+            executor.tree,
+            executor.collect_output(),
+            download=True,
+        )
+
+    @staticmethod
+    def _process(dest_tree, src_tree, collected_files, download=False):
+        from dvc.cache.local import _log_exceptions
+
+        from_infos = []
+        to_infos = []
+        names = []
+        for from_info in collected_files:
+            from_infos.append(from_info)
+            fname = from_info.relative_to(src_tree.path_info)
+            names.append(str(fname))
+            to_infos.append(dest_tree.path_info / fname)
+        total = len(from_infos)
+
+        if download:
+            func = partial(
+                _log_exceptions(src_tree.download, "download"),
+                dir_mode=dest_tree.dir_mode,
+                file_mode=dest_tree.file_mode,
+            )
+            desc = "Downloading"
+        else:
+            func = partial(_log_exceptions(dest_tree.upload, "upload"))
+            desc = "Uploading"
+
+        with Tqdm(total=total, unit="file", desc=desc) as pbar:
+            func = pbar.wrap_fn(func)
+            # TODO: parallelize this, currently --jobs for repro applies to
+            # number of repro executors not download threads
+            with ThreadPoolExecutor(max_workers=1) as dl_executor:
+                fails = sum(dl_executor.map(func, from_infos, to_infos, names))
+
+        if fails:
+            if download:
+                raise DownloadError(fails)
+            raise UploadError(fails)
+
+    @scm_locked
+    def checkout_exp(self, rev):
         """Checkout an experiment to the user's workspace."""
         from git.exc import GitCommandError
-        from dvc.repo.checkout import _checkout as dvc_checkout
 
-        if force:
-            self.repo.scm.repo.git.reset(hard=True)
-        logger.debug(f"checkout {rev}")
+        from dvc.repo.checkout import checkout as dvc_checkout
+
+        baseline_rev = self._check_baseline(rev)
         self._scm_checkout(rev)
 
         tmp = tempfile.NamedTemporaryFile(delete=False).name
-        self.scm.repo.head.commit.diff("HEAD~1", patch=True, output=tmp)
+        self.scm.repo.head.commit.diff(
+            baseline_rev, patch=True, full_index=True, binary=True, output=tmp
+        )
+
+        dirty = self.repo.scm.is_dirty()
+        if dirty:
+            logger.debug("Stashing workspace changes.")
+            self.repo.scm.repo.git.stash("push", "--include-untracked")
+
         try:
             if os.path.getsize(tmp):
                 logger.debug("Patching local workspace")
                 self.repo.scm.repo.git.apply(tmp, reverse=True)
-            dvc_checkout(self.repo)
+                need_checkout = True
+            else:
+                need_checkout = False
         except GitCommandError:
-            raise DvcException(
-                "Checkout failed, experiment contains changes which "
-                "conflict with your current workspace. To overwrite "
-                "your workspace, use `dvc experiments checkout --force`."
-            )
+            raise DvcException("failed to apply experiment changes.")
         finally:
             remove(tmp)
+            if dirty:
+                self._unstash_workspace()
+
+        if need_checkout:
+            dvc_checkout(self.repo)
+
+    def _check_baseline(self, exp_rev):
+        baseline_sha = self.repo.scm.get_rev()
+        exp_baseline = self._get_baseline(exp_rev)
+        if exp_baseline is None:
+            # if we can't tell from branch name, fall back to parent commit
+            exp_commit = self.scm.repo.rev_parse(exp_rev)
+            exp_baseline = first(exp_commit.parents).hexsha
+        if exp_baseline == baseline_sha:
+            return exp_baseline
+        raise BaselineMismatchError(exp_baseline, baseline_sha)
+
+    def _unstash_workspace(self):
+        # Essentially we want `git stash pop` with `-X ours` merge strategy
+        # to prefer the applied experiment changes over stashed workspace
+        # changes. git stash doesn't support merge strategy parameters, but we
+        # can do it ourselves with checkout/reset.
+        from git.exc import GitCommandError
+
+        logger.debug("Unstashing workspace changes.")
+        git_repo = self.repo.scm.repo.git
+
+        # stage workspace changes, then apply stashed changes on top
+        git_repo.add(A=True)
+        try:
+            git_repo.stash("apply", "stash@{0}")
+        except GitCommandError:
+            # stash apply will return error code on merge conflicts,
+            # prefer workspace changes over stash changes
+            git_repo.checkout("--ours", "--", ".")
+
+        # unstage changes and drop the stash entry
+        git_repo.reset("HEAD")
+        git_repo.stash("drop", "stash@{0}")
+
+    @scm_locked
+    def get_baseline(self, rev):
+        """Return the baseline rev for an experiment rev."""
+        return self._get_baseline(rev)
+
+    def _get_baseline(self, rev):
+        from git.exc import GitCommandError
+
+        rev = self.scm.resolve_rev(rev)
+        try:
+            name = self.scm.repo.git.name_rev(rev, name_only=True)
+        except GitCommandError:
+            return None
+        if name in ("undefined", "stash"):
+            entry = self.stash_revs.get(rev)
+            if entry:
+                return entry.baseline_rev
+            return None
+        m = self.BRANCH_RE.match(name)
+        if m:
+            return self.scm.resolve_rev(m.group("baseline_rev"))
+        return None
+
+    def _get_branch_containing(self, rev):
+        from git.exc import GitCommandError
+
+        if self.scm.repo.head.is_detached:
+            self._checkout_default_branch()
+        try:
+            name = self.scm.repo.git.branch(contains=rev)
+            return name.rsplit("/")[-1].strip()
+        except GitCommandError:
+            pass
+        return None
 
     def checkout(self, *args, **kwargs):
         from dvc.repo.experiments.checkout import checkout

dvc/repo/experiments/executor.py

@@ -0,0 +1,167 @@
+import logging
+import os
+import pickle
+from tempfile import TemporaryDirectory
+from typing import Iterable, Optional
+
+from funcy import cached_property
+
+from dvc.path_info import PathInfo
+from dvc.stage import PipelineStage
+from dvc.tree.base import BaseTree
+from dvc.tree.local import LocalTree
+from dvc.tree.repo import RepoTree
+
+logger = logging.getLogger(__name__)
+
+
+class ExperimentExecutor:
+    """Base class for executing experiments in parallel.
+
+    Args:
+        baseline_rev: baseline revision that this experiment is derived from.
+
+    Optional keyword args:
+        branch: Existing git branch for this experiment.
+        repro_args: Args to be passed into reproduce.
+        repro_kwargs: Keyword args to be passed into reproduce.
+    """
+
+    def __init__(
+        self, baseline_rev: str, branch: Optional[str] = None, **kwargs,
+    ):
+        self.baseline_rev = baseline_rev
+        self.branch = branch
+        self.repro_args = kwargs.pop("repro_args", [])
+        self.repro_kwargs = kwargs.pop("repro_kwargs", {})
+
+    @property
+    def tree(self) -> BaseTree:
+        raise NotImplementedError
+
+    @staticmethod
+    def reproduce(dvc_dir, cwd=None, **kwargs):
+        raise NotImplementedError
+
+    def collect_output(self) -> Iterable["PathInfo"]:
+        """Iterate over output pathnames for this executor.
+
+        For DVC outs, only the .dvc file path will be yielded. DVC outs
+        themselves should be fetched from remote executor cache in the normal
+        fetch/pull way. For local executors outs will already be available via
+        shared local cache.
+        """
+        raise NotImplementedError
+
+    def cleanup(self):
+        pass
+
+    # TODO: come up with better way to stash repro arguments
+    @staticmethod
+    def pack_repro_args(path, *args, tree=None, **kwargs):
+        open_func = tree.open if tree else open
+        data = {"args": args, "kwargs": kwargs}
+        with open_func(path, "wb") as fobj:
+            pickle.dump(data, fobj)
+
+    @staticmethod
+    def unpack_repro_args(path, tree=None):
+        open_func = tree.open if tree else open
+        with open_func(path, "rb") as fobj:
+            data = pickle.load(fobj)
+        return data["args"], data["kwargs"]
+
+
+class LocalExecutor(ExperimentExecutor):
+    """Local machine experiment executor."""
+
+    def __init__(self, baseline_rev: str, **kwargs):
+        from dvc.repo import Repo
+
+        dvc_dir = kwargs.pop("dvc_dir")
+        cache_dir = kwargs.pop("cache_dir")
+        super().__init__(baseline_rev, **kwargs)
+        self.tmp_dir = TemporaryDirectory()
+
+        # init empty DVC repo (will be overwritten when input is uploaded)
+        Repo.init(root_dir=self.tmp_dir.name, no_scm=True)
+        logger.debug(
+            "Init local executor in dir '%s' with baseline '%s'.",
+            self.tmp_dir,
+            baseline_rev[:7],
+        )
+        self.dvc_dir = os.path.join(self.tmp_dir.name, dvc_dir)
+        self._config(cache_dir)
+        self._tree = LocalTree(self.dvc, {"url": self.dvc.root_dir})
+        # override default CACHE_MODE since files must be writable in order
+        # to run repro
+        self._tree.CACHE_MODE = 0o644
+
+    def _config(self, cache_dir):
+        local_config = os.path.join(self.dvc_dir, "config.local")
+        logger.debug("Writing experiments local config '%s'", local_config)
+        with open(local_config, "w") as fobj:
+            fobj.write("[core]\n    no_scm = true\n")
+            fobj.write(f"[cache]\n    dir = {cache_dir}")
+
+    @cached_property
+    def dvc(self):
+        from dvc.repo import Repo
+
+        return Repo(self.dvc_dir)
+
+    @cached_property
+    def path_info(self):
+        return PathInfo(self.tmp_dir.name)
+
+    @property
+    def tree(self):
+        return self._tree
+
+    @staticmethod
+    def reproduce(dvc_dir, cwd=None, **kwargs):
+        """Run dvc repro and return the result."""
+        from dvc.repo import Repo
+        from dvc.repo.experiments import hash_exp
+
+        unchanged = []
+
+        def filter_pipeline(stage):
+            if isinstance(stage, PipelineStage):
+                unchanged.append(stage)
+
+        if cwd:
+            old_cwd = os.getcwd()
+            os.chdir(cwd)
+        else:
+            old_cwd = None
+            cwd = os.getcwd()
+
+        try:
+            logger.debug("Running repro in '%s'", cwd)
+            dvc = Repo(dvc_dir)
+            dvc.checkout()
+            stages = dvc.reproduce(on_unchanged=filter_pipeline, **kwargs)
+        finally:
+            if old_cwd is not None:
+                os.chdir(old_cwd)
+
+        # ideally we would return stages here like a normal repro() call, but
+        # stages is not currently picklable and cannot be returned across
+        # multiprocessing calls
+        return hash_exp(stages + unchanged)
+
+    def collect_output(self) -> Iterable["PathInfo"]:
+        repo_tree = RepoTree(self.dvc)
+        yield from self.collect_files(self.tree, repo_tree)
+
+    @staticmethod
+    def collect_files(tree: BaseTree, repo_tree: RepoTree):
+        for fname in repo_tree.walk_files(repo_tree.root_dir, dvcfiles=True):
+            if not repo_tree.isdvc(fname):
+                yield tree.path_info / fname.relative_to(repo_tree.root_dir)
+
+    def cleanup(self):
+        logger.debug("Removing tmpdir '%s'", self.tmp_dir)
+        self.tmp_dir.cleanup()
+        super().cleanup()

dvc/repo/experiments/show.py

@@ -1,6 +1,6 @@
 import logging
-import re
 from collections import OrderedDict, defaultdict
+from datetime import datetime
 
 from dvc.repo import locked
 from dvc.repo.metrics.show import _collect_metrics, _read_metrics
@@ -9,36 +9,58 @@ from dvc.repo.params.show import _collect_configs, _read_params
 logger = logging.getLogger(__name__)
 
 
-EXP_RE = re.compile(r"(?P<rev_sha>[a-f0-9]{7})-(?P<exp_sha>[a-f0-9]+)")
+def _collect_experiment(repo, branch, stash=False, sha_only=True):
+    from git.exc import GitCommandError
 
-
-def _collect_experiment(repo, branch):
     res = defaultdict(dict)
     for rev in repo.brancher(revs=[branch]):
+        if rev == "workspace":
+            res["timestamp"] = None
+        else:
+            commit = repo.scm.repo.rev_parse(rev)
+            res["timestamp"] = datetime.fromtimestamp(commit.committed_date)
+
         configs = _collect_configs(repo)
         params = _read_params(repo, configs, rev)
         if params:
             res["params"] = params
 
-        metrics = _collect_metrics(repo, None, False)
-        vals = _read_metrics(repo, metrics, rev)
-        if vals:
+        res["queued"] = stash
+        if not stash:
+            metrics = _collect_metrics(repo, None, False)
+            vals = _read_metrics(repo, metrics, rev)
             res["metrics"] = vals
 
+        if not sha_only and rev != "workspace":
+            try:
+                name = repo.scm.repo.git.describe(
+                    rev, all=True, exact_match=True
+                )
+                name = name.rsplit("/")[-1]
+                res["name"] = name
+            except GitCommandError:
+                pass
+
     return res
 
 
 @locked
 def show(
-    repo, all_branches=False, all_tags=False, revs=None, all_commits=False
+    repo,
+    all_branches=False,
+    all_tags=False,
+    revs=None,
+    all_commits=False,
+    sha_only=False,
 ):
     res = defaultdict(OrderedDict)
 
     if revs is None:
         revs = [repo.scm.get_rev()]
 
-    revs = set(
-        repo.brancher(
+    revs = OrderedDict(
+        (rev, None)
+        for rev in repo.brancher(
             revs=revs,
             all_branches=all_branches,
             all_tags=all_tags,
@@ -47,12 +69,15 @@ def show(
     )
 
     for rev in revs:
-        res[rev]["baseline"] = _collect_experiment(repo, rev)
+        res[rev]["baseline"] = _collect_experiment(
+            repo, rev, sha_only=sha_only
+        )
 
+    # collect reproduced experiments
     for exp_branch in repo.experiments.scm.list_branches():
-        m = re.match(EXP_RE, exp_branch)
+        m = repo.experiments.BRANCH_RE.match(exp_branch)
         if m:
-            rev = repo.scm.resolve_rev(m.group("rev_sha"))
+            rev = repo.scm.resolve_rev(m.group("baseline_rev"))
             if rev in revs:
                 exp_rev = repo.experiments.scm.resolve_rev(exp_branch)
                 with repo.experiments.chdir():
@@ -61,4 +86,13 @@ def show(
                     )
                 res[rev][exp_rev] = experiment
 
+    # collect queued (not yet reproduced) experiments
+    for stash_rev, entry in repo.experiments.stash_revs.items():
+        if entry.baseline_rev in revs:
+            with repo.experiments.chdir():
+                experiment = _collect_experiment(
+                    repo.experiments.exp_dvc, stash_rev, stash=True
+                )
+            res[entry.baseline_rev][stash_rev] = experiment
+
     return res

dvc/repo/fetch.py

@@ -4,10 +4,13 @@ from dvc.config import NoRemoteError
 from dvc.exceptions import DownloadError
 from dvc.scm.base import CloneError
 
+from . import locked
+
 logger = logging.getLogger(__name__)
 
 
-def _fetch(
+@locked
+def fetch(
     self,
     targets=None,
     jobs=None,
@@ -79,12 +82,14 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs):
     from dvc.external_repo import external_repo
 
     failed, downloaded = 0, 0
+    cache = self.cache.local
     try:
-        with external_repo(repo_url, repo_rev) as repo:
-            with repo.use_cache(self.cache.local):
-                d, f, _ = repo.fetch_external(files, jobs=jobs)
-                downloaded += d
-                failed += f
+        with external_repo(
+            repo_url, repo_rev, cache_dir=cache.cache_dir
+        ) as repo:
+            d, f, _ = repo.fetch_external(files, jobs=jobs)
+            downloaded += d
+            failed += f
     except CloneError:
         failed += 1
         logger.exception(

dvc/repo/freeze.py

@@ -8,7 +8,7 @@ def _set(repo, target, frozen):
     path, name = parse_target(target)
     stage = repo.get_stage(path, name)
     stage.frozen = frozen
-    stage.dvcfile.dump(stage, update_pipeline=True)
+    stage.dvcfile.dump(stage, update_lock=False)
 
     return stage
 

dvc/repo/gc.py

@@ -50,6 +50,7 @@ def gc(
     )
 
     from contextlib import ExitStack
+
     from dvc.repo import Repo
 
     if not repos:

dvc/repo/get.py

@@ -20,8 +20,8 @@ class GetDVCFileError(DvcException):
 
 @staticmethod
 def get(url, path, out=None, rev=None):
-    from dvc.external_repo import external_repo
     from dvc.dvcfile import is_valid_filename
+    from dvc.external_repo import external_repo
 
     out = resolve_output(path, out)
 
@@ -35,21 +35,21 @@ def get(url, path, out=None, rev=None):
     # and won't work with reflink/hardlink.
     dpath = os.path.dirname(os.path.abspath(out))
     tmp_dir = os.path.join(dpath, "." + str(shortuuid.uuid()))
-    try:
-        with external_repo(url=url, rev=rev) as repo:
-            if hasattr(repo, "cache"):
-                repo.cache.local.cache_dir = tmp_dir
 
-                # Try any links possible to avoid data duplication.
-                #
-                # Not using symlink, because we need to remove cache after we
-                # are done, and to make that work we would have to copy data
-                # over anyway before removing the cache, so we might just copy
-                # it right away.
-                #
-                # Also, we can't use theoretical "move" link type here, because
-                # the same cache file might be used a few times in a directory.
-                repo.cache.local.cache_types = ["reflink", "hardlink", "copy"]
+    # Try any links possible to avoid data duplication.
+    #
+    # Not using symlink, because we need to remove cache after we
+    # are done, and to make that work we would have to copy data
+    # over anyway before removing the cache, so we might just copy
+    # it right away.
+    #
+    # Also, we can't use theoretical "move" link type here, because
+    # the same cache file might be used a few times in a directory.
+    cache_types = ["reflink", "hardlink", "copy"]
+    try:
+        with external_repo(
+            url=url, rev=rev, cache_dir=tmp_dir, cache_types=cache_types
+        ) as repo:
             repo.get_external(path, out)
     finally:
         remove(tmp_dir)

dvc/repo/graph.py

@@ -1,5 +1,6 @@
 def check_acyclic(graph):
     import networkx as nx
+
     from dvc.exceptions import CyclicGraphError
 
     try:

dvc/repo/init.py

@@ -6,6 +6,7 @@ import colorama
 from dvc import analytics
 from dvc.config import Config
 from dvc.exceptions import InitError, InvalidArgumentError
+from dvc.ignore import init as init_dvcignore
 from dvc.repo import Repo
 from dvc.scm import SCM
 from dvc.scm.base import SCMError
@@ -99,9 +100,13 @@ def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False):
         with config.edit() as conf:
             conf["core"]["no_scm"] = True
 
+    dvcignore = init_dvcignore(root_dir)
+
     proj = Repo(root_dir)
 
-    scm.add([config.files["repo"], proj.plot_templates.templates_dir])
+    scm.add(
+        [config.files["repo"], dvcignore, proj.plot_templates.templates_dir]
+    )
 
     if scm.ignore_file:
         scm.add([os.path.join(dvc_dir, scm.ignore_file)])

dvc/repo/ls.py

@@ -1,3 +1,5 @@
+from itertools import chain
+
 from dvc.exceptions import PathMissingError
 from dvc.path_info import PathInfo
 
@@ -29,7 +31,9 @@ def ls(
     """
     from dvc.external_repo import external_repo
 
-    with external_repo(url, rev) as repo:
+    # use our own RepoTree instance instead of repo.repo_tree since we want to
+    # fetch directory listings, but don't want to fetch file contents.
+    with external_repo(url, rev, fetch=False, stream=True) as repo:
         path_info = PathInfo(repo.root_dir)
         if path:
             path_info /= path
@@ -48,57 +52,37 @@ def ls(
 
 
 def _ls(repo, path_info, recursive=None, dvc_only=False):
-    from dvc.repo.tree import RepoTree
-
     def onerror(exc):
         raise exc
 
-    # use our own RepoTree instance instead of repo.repo_tree since we want to
-    # fetch directory listings, but don't want to fetch file contents.
-    tree = RepoTree(repo, stream=True)
+    tree = repo.repo_tree
 
-    ret = {}
+    infos = []
     try:
         for root, dirs, files in tree.walk(
             path_info.fspath, onerror=onerror, dvcfiles=True
         ):
-            for fname in files:
-                info = PathInfo(root) / fname
-                dvc = tree.isdvc(info)
-                if dvc or not dvc_only:
-                    path = str(info.relative_to(path_info))
-                    ret[path] = {
-                        "isout": dvc,
-                        "isdir": False,
-                        "isexec": False if dvc else tree.isexec(info),
-                    }
-
+            entries = chain(files, dirs) if not recursive else files
+            infos.extend(PathInfo(root) / entry for entry in entries)
             if not recursive:
-                for dname in dirs:
-                    info = PathInfo(root) / dname
-                    if not dvc_only or (
-                        tree.dvctree and tree.dvctree.exists(info)
-                    ):
-                        dvc = tree.isdvc(info)
-                        path = str(info.relative_to(path_info))
-                        ret[path] = {
-                            "isout": dvc,
-                            "isdir": True,
-                            "isexec": False if dvc else tree.isexec(info),
-                        }
                 break
     except NotADirectoryError:
-        dvc = tree.isdvc(path_info)
-        if dvc or not dvc_only:
-            return {
-                path_info.name: {
-                    "isout": dvc,
-                    "isdir": False,
-                    "isexec": False if dvc else tree.isexec(path_info),
-                }
-            }
-        return {}
+        infos.append(path_info)
     except FileNotFoundError:
         return {}
 
+    ret = {}
+    for info in infos:
+        metadata = tree.metadata(info)
+        if metadata.output_exists or not dvc_only:
+            path = (
+                path_info.name
+                if path_info == info
+                else str(info.relative_to(path_info))
+            )
+            ret[path] = {
+                "isout": metadata.is_output,
+                "isdir": metadata.isdir,
+                "isexec": metadata.is_exec,
+            }
     return ret

dvc/repo/metrics/show.py

@@ -1,42 +1,44 @@
 import logging
 import os
 
-import yaml
-
 from dvc.exceptions import NoMetricsError
 from dvc.path_info import PathInfo
 from dvc.repo import locked
-from dvc.repo.tree import RepoTree
+from dvc.tree.repo import RepoTree
+from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml
 
 logger = logging.getLogger(__name__)
 
 
 def _collect_metrics(repo, targets, recursive):
-    metrics = set()
 
+    if targets:
+        target_infos = [
+            PathInfo(os.path.abspath(target)) for target in targets
+        ]
+        tree = RepoTree(repo)
+
+        rec_files = []
+        if recursive:
+            for target_info in target_infos:
+                if tree.isdir(target_info):
+                    rec_files.extend(list(tree.walk_files(target_info)))
+
+        result = [t for t in target_infos if tree.isfile(t)]
+        result.extend(rec_files)
+
+        return result
+
+    metrics = set()
     for stage in repo.stages:
         for out in stage.outs:
             if not out.metric:
                 continue
-
             metrics.add(out.path_info)
-
-    if not targets:
-        return list(metrics)
-
-    target_infos = [PathInfo(os.path.abspath(target)) for target in targets]
-
-    def _filter(path_info):
-        for info in target_infos:
-            func = path_info.isin_or_eq if recursive else path_info.__eq__
-            if func(info):
-                return True
-        return False
-
-    return list(filter(_filter, metrics))
+    return list(metrics)
 
 
-def _extract_metrics(metrics):
+def _extract_metrics(metrics, path, rev):
     if isinstance(metrics, (int, float)):
         return metrics
 
@@ -45,9 +47,18 @@ def _extract_metrics(metrics):
 
     ret = {}
     for key, val in metrics.items():
-        m = _extract_metrics(val)
+        m = _extract_metrics(val, path, rev)
         if m:
             ret[key] = m
+        else:
+            logger.debug(
+                "Could not parse '%s' metric from '%s' at '%s' "
+                "due to its unsupported type: '%s'",
+                key,
+                path,
+                rev,
+                type(val).__name__,
+            )
 
     return ret
 
@@ -61,16 +72,14 @@ def _read_metrics(repo, metrics, rev):
             continue
 
         try:
-            with tree.open(metric, "r") as fobj:
-                # NOTE this also supports JSON
-                val = yaml.safe_load(fobj)
-        except (FileNotFoundError, yaml.YAMLError):
+            val = load_yaml(metric, tree=tree)
+        except (FileNotFoundError, YAMLFileCorruptedError):
             logger.debug(
                 "failed to read '%s' on '%s'", metric, rev, exc_info=True
             )
             continue
 
-        val = _extract_metrics(val)
+        val = _extract_metrics(val, metric, rev)
         if val:
             res[str(metric)] = val
 
@@ -88,6 +97,7 @@ def show(
     all_commits=False,
 ):
     res = {}
+    metrics_found = False
 
     for rev in repo.brancher(
         revs=revs,
@@ -96,13 +106,27 @@ def show(
         all_commits=all_commits,
     ):
         metrics = _collect_metrics(repo, targets, recursive)
+
+        if not metrics_found and metrics:
+            metrics_found = True
+
         vals = _read_metrics(repo, metrics, rev)
 
         if vals:
             res[rev] = vals
 
     if not res:
-        raise NoMetricsError()
+        if metrics_found:
+            msg = (
+                "Could not parse metric files. Use `-v` option to see more "
+                "details."
+            )
+        else:
+            msg = (
+                "no metric files in this repository. Use `-m/-M` options for "
+                "`dvc run` to mark stage outputs as  metrics."
+            )
+        raise NoMetricsError(msg)
 
     # Hide workspace metrics if they are the same as in the active branch
     try:

dvc/repo/move.py

@@ -34,6 +34,7 @@ def move(self, from_path, to_path):
     """
     import dvc.output as Output
     from dvc.stage import Stage
+
     from ..dvcfile import DVC_FILE_SUFFIX, Dvcfile
 
     from_out = Output.loads_from(Stage(self), [from_path])[0]

dvc/repo/params/show.py

@@ -1,12 +1,10 @@
 import logging
 
-import toml
-import yaml
-
 from dvc.dependency.param import ParamsDependency
 from dvc.exceptions import DvcException
 from dvc.path_info import PathInfo
 from dvc.repo import locked
+from dvc.utils.serialize import LOADERS, ParseError
 
 logger = logging.getLogger(__name__)
 
@@ -33,16 +31,15 @@ def _read_params(repo, configs, rev):
         if not repo.tree.exists(config):
             continue
 
-        with repo.tree.open(config, "r") as fobj:
-            try:
-                res[str(config)] = ParamsDependency.PARAMS_FILE_LOADERS[
-                    config.suffix.lower()
-                ](fobj)
-            except (yaml.YAMLError, toml.TomlDecodeError):
-                logger.debug(
-                    "failed to read '%s' on '%s'", config, rev, exc_info=True
-                )
-                continue
+        suffix = config.suffix.lower()
+        loader = LOADERS[suffix]
+        try:
+            res[str(config)] = loader(config, tree=repo.tree)
+        except ParseError:
+            logger.debug(
+                "failed to read '%s' on '%s'", config, rev, exc_info=True
+            )
+            continue
 
     return res
 

dvc/repo/plots/__init__.py

@@ -1,10 +1,12 @@
 import logging
+import os
 
 from funcy import first, project
 
-from dvc.exceptions import DvcException, NoPlotsError, OutputNotFoundError
-from dvc.repo.tree import RepoTree
+from dvc.exceptions import DvcException, NoPlotsError
+from dvc.path_info import PathInfo
 from dvc.schema import PLOT_PROPS
+from dvc.tree.repo import RepoTree
 from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
@@ -75,7 +77,7 @@ class Plots:
             for datafile, desc in plots.items()
         }
 
-    def show(self, targets=None, revs=None, props=None):
+    def show(self, targets=None, revs=None, props=None, templates=None):
         from .data import NoMetricInHistoryError
 
         data = self.collect(targets, revs)
@@ -90,7 +92,9 @@ class Plots:
         if not data:
             raise NoPlotsError()
 
-        return self.render(data, revs, props, self.repo.plot_templates)
+        if templates is None:
+            templates = self.repo.plot_templates
+        return self.render(data, revs, props, templates)
 
     def diff(self, *args, **kwargs):
         from .diff import diff
@@ -136,27 +140,38 @@ class Plots:
         out.verify_metric()
 
         dvcfile = Dvcfile(self.repo, out.stage.path)
-        dvcfile.dump(out.stage, update_pipeline=True, no_lock=True)
+        dvcfile.dump(out.stage, update_lock=False)
 
 
 def _collect_plots(repo, targets=None, rev=None):
-    def _targets_to_outs(targets):
-        for t in targets:
-            try:
-                (out,) = repo.find_outs_by_path(t)
-                yield out
-            except OutputNotFoundError:
-                logger.warning(
-                    "File '{}' was not found at: '{}'. It will not be "
-                    "plotted.".format(t, rev)
-                )
+    plots = {out for stage in repo.stages for out in stage.outs if out.plot}
 
-    if targets:
-        outs = _targets_to_outs(targets)
-    else:
-        outs = (out for stage in repo.stages for out in stage.outs if out.plot)
+    def to_result(plots):
+        return {plot.path_info: _plot_props(plot) for plot in plots}
 
-    return {out.path_info: _plot_props(out) for out in outs}
+    if not targets:
+        return to_result(plots)
+
+    target_infos = {PathInfo(os.path.abspath(target)) for target in targets}
+
+    target_plots = set()
+    for p in plots:
+        if p.path_info in target_infos:
+            target_plots.add(p)
+            target_infos.remove(p.path_info)
+
+    tree = RepoTree(repo)
+    result = to_result(target_plots)
+
+    for t in target_infos:
+        if tree.isfile(t):
+            result[t] = {}
+        else:
+            logger.warning(
+                "'%s' was not found at: '%s'. It will not be plotted.", t, rev,
+            )
+
+    return result
 
 
 def _plot_props(out):
@@ -185,6 +200,11 @@ def _prepare_plots(data, revs, props):
             continue
 
         for datafile, desc in data[rev].items():
+            # We silently skip on an absent data file,
+            # see also try/except/pass in .collect()
+            if "data" not in desc:
+                continue
+
             # props from command line overwrite plot props from out definition
             full_props = {**desc["props"], **props}
 
@@ -210,7 +230,7 @@ def _prepare_plots(data, revs, props):
 
 
 def _render(datafile, datas, props, templates):
-    from .data import plot_data, PlotData
+    from .data import PlotData, plot_data
 
     # Copy it to not modify a passed value
     props = props.copy()

dvc/repo/plots/data.py

@@ -5,11 +5,10 @@ import os
 from collections import OrderedDict
 from copy import copy
 
-import yaml
 from funcy import first
-from yaml import SafeLoader
 
 from dvc.exceptions import DvcException
+from dvc.utils.serialize import loads_yaml
 
 
 class PlotMetricTypeError(DvcException):
@@ -101,7 +100,7 @@ def _apply_path(data, path=None, **kwargs):
 def _lists(dictionary):
     for _, value in dictionary.items():
         if isinstance(value, dict):
-            yield from (_lists(value))
+            yield from _lists(value)
         elif isinstance(value, list):
             yield value
 
@@ -114,10 +113,12 @@ def _find_data(data, fields=None, **kwargs):
         # just look for first list of dicts
         fields = set()
 
-    for l in _lists(data):
-        if all(isinstance(dp, dict) for dp in l):
-            if set(first(l).keys()) & fields == fields:
-                return l
+    for lst in _lists(data):
+        if (
+            all(isinstance(dp, dict) for dp in lst)
+            and set(first(lst).keys()) & fields == fields
+        ):
+            return lst
     raise PlotDataStructureError()
 
 
@@ -206,18 +207,7 @@ class CSVPlotData(PlotData):
 
 class YAMLPlotData(PlotData):
     def raw(self, **kwargs):
-        class OrderedLoader(SafeLoader):
-            pass
-
-        def construct_mapping(loader, node):
-            loader.flatten_mapping(node)
-            return OrderedDict(loader.construct_pairs(node))
-
-        OrderedLoader.add_constructor(
-            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping
-        )
-
-        return yaml.load(self.content, OrderedLoader)
+        return loads_yaml(self.content, typ="rt")
 
     def _processors(self):
         parent_processors = super()._processors()

dvc/repo/plots/diff.py

@@ -1,13 +1,23 @@
-def _revisions(revs, is_dirty):
+def _revisions(repo, revs, experiment):
     revisions = revs or []
+    if experiment and len(revisions) == 1:
+        baseline = repo.experiments.get_baseline(revisions[0])
+        if baseline:
+            revisions.append(baseline[:7])
     if len(revisions) <= 1:
-        if len(revisions) == 0 and is_dirty:
+        if len(revisions) == 0 and repo.scm.is_dirty():
             revisions.append("HEAD")
         revisions.append("workspace")
     return revisions
 
 
-def diff(repo, *args, revs=None, **kwargs):
-    return repo.plots.show(
-        *args, revs=_revisions(revs, repo.scm.is_dirty()), **kwargs
+def diff(repo, *args, revs=None, experiment=False, **kwargs):
+    if experiment:
+        # use experiments repo brancher with templates from the main repo
+        kwargs["templates"] = repo.plot_templates
+        plots_repo = repo.experiments.exp_dvc
+    else:
+        plots_repo = repo
+    return plots_repo.plots.show(
+        *args, revs=_revisions(repo, revs, experiment), **kwargs
     )

dvc/repo/pull.py

@@ -22,7 +22,7 @@ def pull(
     if isinstance(targets, str):
         targets = [targets]
 
-    processed_files_count = self._fetch(  # pylint: disable=protected-access
+    processed_files_count = self.fetch(
         targets,
         jobs,
         remote=remote,
@@ -33,7 +33,7 @@ def pull(
         recursive=recursive,
         run_cache=run_cache,
     )
-    stats = self._checkout(  # pylint: disable=protected-access
+    stats = self.checkout(
         targets=targets, with_deps=with_deps, force=force, recursive=recursive
     )
 

dvc/repo/reproduce.py

@@ -25,7 +25,7 @@ def _reproduce_stage(stage, **kwargs):
         from ..dvcfile import Dvcfile
 
         dvcfile = Dvcfile(stage.repo, stage.path)
-        dvcfile.dump(stage)
+        dvcfile.dump(stage, update_pipeline=False)
 
     return [stage]
 
@@ -60,7 +60,7 @@ def reproduce(
     recursive=False,
     pipeline=False,
     all_pipelines=False,
-    **kwargs
+    **kwargs,
 ):
     from dvc.utils import parse_target
 
@@ -71,13 +71,22 @@ def reproduce(
         )
 
     experiment = kwargs.pop("experiment", False)
-    if experiment and self.experiments:
+    params = _parse_params(kwargs.pop("params", []))
+    queue = kwargs.pop("queue", False)
+    run_all = kwargs.pop("run_all", False)
+    jobs = kwargs.pop("jobs", 1)
+    if (experiment or run_all) and self.experiments:
         try:
-            return self.experiments.new(
+            return _reproduce_experiments(
+                self,
                 target=target,
                 recursive=recursive,
                 all_pipelines=all_pipelines,
-                **kwargs
+                params=params,
+                queue=queue,
+                run_all=run_all,
+                jobs=jobs,
+                **kwargs,
             )
         except UnchangedExperimentError:
             # If experiment contains no changes, just run regular repro
@@ -109,8 +118,41 @@ def reproduce(
     return _reproduce_stages(active_graph, targets, **kwargs)
 
 
+def _parse_params(path_params):
+    from ruamel.yaml import YAMLError
+
+    from dvc.dependency.param import ParamsDependency
+    from dvc.utils.flatten import unflatten
+    from dvc.utils.serialize import loads_yaml
+
+    ret = {}
+    for path_param in path_params:
+        path, _, params_str = path_param.rpartition(":")
+        # remove empty strings from params, on condition such as `-p "file1:"`
+        params = {}
+        for param_str in filter(bool, params_str.split(",")):
+            try:
+                # interpret value strings using YAML rules
+                key, value = param_str.split("=")
+                params[key] = loads_yaml(value)
+            except (ValueError, YAMLError):
+                raise InvalidArgumentError(
+                    f"Invalid param/value pair '{param_str}'"
+                )
+        if not path:
+            path = ParamsDependency.DEFAULT_PARAMS_FILE
+        ret[path] = unflatten(params)
+    return ret
+
+
+def _reproduce_experiments(repo, run_all=False, jobs=1, **kwargs):
+    if run_all:
+        return repo.experiments.reproduce_queued(jobs=jobs)
+    return repo.experiments.reproduce_one(**kwargs)
+
+
 def _reproduce_stages(
-    G, stages, downstream=False, single_item=False, **kwargs
+    G, stages, downstream=False, single_item=False, on_unchanged=None, **kwargs
 ):
     r"""Derive the evaluation of the given node for the given graph.
 
@@ -194,7 +236,10 @@ def _reproduce_stages(
                 # dependencies didn't change.
                 kwargs["force"] = True
 
-            result.extend(ret)
+            if ret:
+                result.extend(ret)
+            elif on_unchanged is not None:
+                on_unchanged(stage)
         except Exception as exc:
             raise ReproductionError(stage.relpath) from exc
 

dvc/repo/run.py

@@ -36,7 +36,7 @@ def parse_params(path_params):
 
 
 def _get_file_path(kwargs):
-    from dvc.dvcfile import DVC_FILE_SUFFIX, DVC_FILE
+    from dvc.dvcfile import DVC_FILE, DVC_FILE_SUFFIX
 
     out = first(
         concat(
@@ -76,8 +76,8 @@ def _check_stage_exists(dvcfile, stage):
 @locked
 @scm_context
 def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
+    from dvc.dvcfile import PIPELINE_FILE, Dvcfile
     from dvc.stage import Stage, create_stage
-    from dvc.dvcfile import Dvcfile, PIPELINE_FILE
 
     if not kwargs.get("cmd"):
         raise InvalidArgumentError("command is not specified")
@@ -134,5 +134,5 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
             run_cache=kwargs.get("run_cache", True),
         )
 
-    dvcfile.dump(stage, update_pipeline=True, no_lock=no_exec)
+    dvcfile.dump(stage, update_lock=not no_exec)
     return stage

dvc/repo/scm_context.py

@@ -3,7 +3,11 @@ def scm_context(method):
         try:
             result = method(repo, *args, **kw)
             repo.scm.reset_ignores()
-            repo.scm.remind_to_track()
+            autostage = repo.config.get("core", {}).get("autostage", False)
+            if autostage:
+                repo.scm.track_changed_files()
+            else:
+                repo.scm.remind_to_track()
             repo.scm.reset_tracked_files()
             return result
         except Exception:

dvc/repo/status.py

@@ -10,31 +10,30 @@ from . import locked
 logger = logging.getLogger(__name__)
 
 
-def _joint_status(stages):
+def _joint_status(pairs):
     status_info = {}
 
-    for stage in stages:
+    for stage, filter_info in pairs:
         if stage.frozen and not stage.is_repo_import:
             logger.warning(
                 "{} is frozen. Its dependencies are"
                 " not going to be shown in the status output.".format(stage)
             )
-
-        status_info.update(stage.status(check_updates=True))
+        status_info.update(
+            stage.status(check_updates=True, filter_info=filter_info)
+        )
 
     return status_info
 
 
 def _local_status(self, targets=None, with_deps=False, recursive=False):
-    if targets:
-        stages = cat(
-            self.collect(t, with_deps=with_deps, recursive=recursive)
-            for t in targets
-        )
-    else:
-        stages = self.collect(None, with_deps=with_deps, recursive=recursive)
+    targets = targets or [None]
+    pairs = cat(
+        self.collect_granular(t, with_deps=with_deps, recursive=recursive)
+        for t in targets
+    )
 
-    return _joint_status(stages)
+    return _joint_status(pairs)
 
 
 def _cloud_status(
@@ -52,6 +51,7 @@ def _cloud_status(
 
     - new: Remote doesn't have the file
     - deleted: File is no longer in the local cache
+    - missing: File doesn't exist neither in the cache, neither in remote
 
     Example:
             Given the following commands:
@@ -89,14 +89,20 @@ def _cloud_status(
     )
 
     ret = {}
-    status_info = self.cloud.status(used, jobs, remote=remote)
+    status_info = self.cloud.status(
+        used, jobs, remote=remote, log_missing=False
+    )
     for info in status_info.values():
         name = info["name"]
         status_ = info["status"]
-        if status_ in [cloud.STATUS_OK, cloud.STATUS_MISSING]:
+        if status_ == cloud.STATUS_OK:
             continue
 
-        prefix_map = {cloud.STATUS_DELETED: "deleted", cloud.STATUS_NEW: "new"}
+        prefix_map = {
+            cloud.STATUS_DELETED: "deleted",
+            cloud.STATUS_NEW: "new",
+            cloud.STATUS_MISSING: "missing",
+        }
 
         ret[name] = prefix_map[status_]
 

dvc/repo/tree.py

@@ -1,444 +0,0 @@
-import logging
-import os
-
-from dvc.dvcfile import is_valid_filename
-from dvc.exceptions import OutputNotFoundError
-from dvc.path_info import PathInfo
-from dvc.tree.base import BaseTree, RemoteActionNotImplemented
-from dvc.utils import file_md5
-from dvc.utils.fs import copy_fobj_to_file, makedirs
-
-logger = logging.getLogger(__name__)
-
-
-class DvcTree(BaseTree):  # pylint:disable=abstract-method
-    """DVC repo tree.
-
-    Args:
-        repo: DVC repo.
-        fetch: if True, uncached DVC outs will be fetched on `open()`.
-        stream: if True, uncached DVC outs will be streamed directly from
-            remote on `open()`.
-
-    `stream` takes precedence over `fetch`. If `stream` is enabled and
-    a remote does not support streaming, uncached DVC outs will be fetched
-    as a fallback.
-    """
-
-    def __init__(self, repo, fetch=False, stream=False):
-        super().__init__(repo, {"url": repo.root_dir})
-        self.fetch = fetch
-        self.stream = stream
-
-    def _find_outs(self, path, *args, **kwargs):
-        outs = self.repo.find_outs_by_path(path, *args, **kwargs)
-
-        def _is_cached(out):
-            return out.use_cache
-
-        outs = list(filter(_is_cached, outs))
-        if not outs:
-            raise OutputNotFoundError(path, self.repo)
-
-        return outs
-
-    def _get_granular_checksum(self, path, out, remote=None):
-        assert isinstance(path, PathInfo)
-        if not self.fetch and not self.stream:
-            raise FileNotFoundError
-        dir_cache = out.get_dir_cache(remote=remote)
-        for entry in dir_cache:
-            entry_relpath = entry[out.tree.PARAM_RELPATH]
-            if os.name == "nt":
-                entry_relpath = entry_relpath.replace("/", os.sep)
-            if path == out.path_info / entry_relpath:
-                return entry[out.tree.PARAM_CHECKSUM]
-        raise FileNotFoundError
-
-    def open(
-        self, path, mode="r", encoding="utf-8", remote=None
-    ):  # pylint: disable=arguments-differ
-        try:
-            outs = self._find_outs(path, strict=False)
-        except OutputNotFoundError as exc:
-            raise FileNotFoundError from exc
-
-        # NOTE: this handles both dirty and checkout-ed out at the same time
-        if self.repo.tree.exists(path):
-            return self.repo.tree.open(path, mode=mode, encoding=encoding)
-
-        if len(outs) != 1 or (
-            outs[0].is_dir_checksum and path == outs[0].path_info
-        ):
-            raise IsADirectoryError
-
-        out = outs[0]
-        if out.changed_cache(filter_info=path):
-            if not self.fetch and not self.stream:
-                raise FileNotFoundError
-
-            remote_obj = self.repo.cloud.get_remote(remote)
-            if self.stream:
-                if out.is_dir_checksum:
-                    checksum = self._get_granular_checksum(path, out)
-                else:
-                    checksum = out.checksum
-                try:
-                    remote_info = remote_obj.tree.hash_to_path_info(checksum)
-                    return remote_obj.tree.open(
-                        remote_info, mode=mode, encoding=encoding
-                    )
-                except RemoteActionNotImplemented:
-                    pass
-            cache_info = out.get_used_cache(filter_info=path, remote=remote)
-            self.repo.cloud.pull(cache_info, remote=remote)
-
-        if out.is_dir_checksum:
-            checksum = self._get_granular_checksum(path, out)
-            cache_path = out.cache.tree.hash_to_path_info(checksum).url
-        else:
-            cache_path = out.cache_path
-        return open(cache_path, mode=mode, encoding=encoding)
-
-    def exists(self, path):  # pylint: disable=arguments-differ
-        try:
-            self._find_outs(path, strict=False, recursive=True)
-            return True
-        except OutputNotFoundError:
-            return False
-
-    def isdir(self, path):  # pylint: disable=arguments-differ
-        if not self.exists(path):
-            return False
-
-        path_info = PathInfo(os.path.abspath(path))
-        outs = self._find_outs(path, strict=False, recursive=True)
-        if len(outs) != 1:
-            return True
-
-        out = outs[0]
-        if not out.is_dir_checksum:
-            if out.path_info != path_info:
-                return True
-            return False
-
-        if out.path_info == path_info:
-            return True
-
-        # for dir checksum, we need to check if this is a file inside the
-        # directory
-        try:
-            self._get_granular_checksum(path_info, out)
-            return False
-        except FileNotFoundError:
-            return True
-
-    def isfile(self, path):  # pylint: disable=arguments-differ
-        if not self.exists(path):
-            return False
-
-        return not self.isdir(path)
-
-    def _add_dir(self, top, trie, out, download_callback=None, **kwargs):
-        if not self.fetch and not self.stream:
-            return
-
-        # pull dir cache if needed
-        dir_cache = out.get_dir_cache(**kwargs)
-
-        # pull dir contents if needed
-        if self.fetch:
-            if out.changed_cache(filter_info=top):
-                used_cache = out.get_used_cache(filter_info=top)
-                downloaded = self.repo.cloud.pull(used_cache, **kwargs)
-                if download_callback:
-                    download_callback(downloaded)
-
-        for entry in dir_cache:
-            entry_relpath = entry[out.tree.PARAM_RELPATH]
-            if os.name == "nt":
-                entry_relpath = entry_relpath.replace("/", os.sep)
-            path_info = out.path_info / entry_relpath
-            trie[path_info.parts] = None
-
-    def _walk(self, root, trie, topdown=True, **kwargs):
-        dirs = set()
-        files = []
-
-        out = trie.get(root.parts)
-        if out and out.is_dir_checksum:
-            self._add_dir(root, trie, out, **kwargs)
-
-        root_len = len(root.parts)
-        for key, out in trie.iteritems(prefix=root.parts):  # noqa: B301
-            if key == root.parts:
-                continue
-
-            name = key[root_len]
-            if len(key) > root_len + 1 or (out and out.is_dir_checksum):
-                dirs.add(name)
-                continue
-
-            files.append(name)
-
-        assert topdown
-        dirs = list(dirs)
-        yield root.fspath, dirs, files
-
-        for dname in dirs:
-            yield from self._walk(root / dname, trie)
-
-    def walk(self, top, topdown=True, onerror=None, **kwargs):
-        from pygtrie import Trie
-
-        assert topdown
-
-        if not self.exists(top):
-            if onerror is not None:
-                onerror(FileNotFoundError(top))
-            return
-
-        if not self.isdir(top):
-            if onerror is not None:
-                onerror(NotADirectoryError(top))
-            return
-
-        root = PathInfo(os.path.abspath(top))
-        outs = self._find_outs(top, recursive=True, strict=False)
-
-        trie = Trie()
-
-        for out in outs:
-            trie[out.path_info.parts] = out
-
-            if out.is_dir_checksum and root.isin_or_eq(out.path_info):
-                self._add_dir(top, trie, out, **kwargs)
-
-        yield from self._walk(root, trie, topdown=topdown, **kwargs)
-
-    def isdvc(self, path, **kwargs):
-        try:
-            return len(self._find_outs(path, **kwargs)) == 1
-        except OutputNotFoundError:
-            pass
-        return False
-
-    def isexec(self, path):  # pylint: disable=unused-argument
-        return False
-
-    def get_file_hash(self, path_info):
-        outs = self._find_outs(path_info, strict=False)
-        if len(outs) != 1:
-            raise OutputNotFoundError
-        out = outs[0]
-        if out.is_dir_checksum:
-            return self._get_granular_checksum(path_info, out)
-        return out.checksum
-
-
-class RepoTree(BaseTree):  # pylint:disable=abstract-method
-    """DVC + git-tracked files tree.
-
-    Args:
-        repo: DVC or git repo.
-
-    Any kwargs will be passed to `DvcTree()`.
-    """
-
-    def __init__(self, repo, **kwargs):
-        super().__init__(repo, {"url": repo.root_dir})
-        if hasattr(repo, "dvc_dir"):
-            self.dvctree = DvcTree(repo, **kwargs)
-        else:
-            # git-only erepo's do not need dvctree
-            self.dvctree = None
-
-    @property
-    def fetch(self):
-        if self.dvctree:
-            return self.dvctree.fetch
-        return False
-
-    @property
-    def stream(self):
-        if self.dvctree:
-            return self.dvctree.stream
-        return False
-
-    def open(
-        self, path, mode="r", encoding="utf-8", **kwargs
-    ):  # pylint: disable=arguments-differ
-        if "b" in mode:
-            encoding = None
-
-        if self.dvctree and self.dvctree.exists(path):
-            return self.dvctree.open(
-                path, mode=mode, encoding=encoding, **kwargs
-            )
-        return self.repo.tree.open(path, mode=mode, encoding=encoding)
-
-    def exists(
-        self, path, use_dvcignore=True
-    ):  # pylint: disable=arguments-differ
-        return self.repo.tree.exists(path) or (
-            self.dvctree and self.dvctree.exists(path)
-        )
-
-    def isdir(self, path):  # pylint: disable=arguments-differ
-        return self.repo.tree.isdir(path) or (
-            self.dvctree and self.dvctree.isdir(path)
-        )
-
-    def isdvc(self, path, **kwargs):
-        return self.dvctree is not None and self.dvctree.isdvc(path, **kwargs)
-
-    def isfile(self, path):  # pylint: disable=arguments-differ
-        return self.repo.tree.isfile(path) or (
-            self.dvctree and self.dvctree.isfile(path)
-        )
-
-    def isexec(self, path):
-        if self.dvctree and self.dvctree.exists(path):
-            return self.dvctree.isexec(path)
-        return self.repo.tree.isexec(path)
-
-    def stat(self, path):
-        return self.repo.tree.stat(path)
-
-    def _walk_one(self, walk):
-        try:
-            root, dirs, files = next(walk)
-        except StopIteration:
-            return
-        yield root, dirs, files
-        for _ in dirs:
-            yield from self._walk_one(walk)
-
-    def _walk(self, dvc_walk, repo_walk, dvcfiles=False):
-        try:
-            _, dvc_dirs, dvc_fnames = next(dvc_walk)
-            repo_root, repo_dirs, repo_fnames = next(repo_walk)
-        except StopIteration:
-            return
-
-        # separate subdirs into shared dirs, dvc-only dirs, repo-only dirs
-        dvc_set = set(dvc_dirs)
-        repo_set = set(repo_dirs)
-        dvc_only = list(dvc_set - repo_set)
-        repo_only = list(repo_set - dvc_set)
-        shared = list(dvc_set & repo_set)
-        dirs = shared + dvc_only + repo_only
-
-        # merge file lists
-        files = {
-            fname
-            for fname in dvc_fnames + repo_fnames
-            if dvcfiles or not is_valid_filename(fname)
-        }
-
-        yield repo_root, dirs, list(files)
-
-        # set dir order for next recursion level - shared dirs first so that
-        # next() for both generators recurses into the same shared directory
-        dvc_dirs[:] = [dirname for dirname in dirs if dirname in dvc_set]
-        repo_dirs[:] = [dirname for dirname in dirs if dirname in repo_set]
-
-        for dirname in dirs:
-            if dirname in shared:
-                yield from self._walk(dvc_walk, repo_walk, dvcfiles=dvcfiles)
-            elif dirname in dvc_set:
-                yield from self._walk_one(dvc_walk)
-            elif dirname in repo_set:
-                yield from self._walk_one(repo_walk)
-
-    def walk(
-        self, top, topdown=True, onerror=None, dvcfiles=False, **kwargs
-    ):  # pylint: disable=arguments-differ
-        """Walk and merge both DVC and repo trees.
-
-        Args:
-            top: path to walk from
-            topdown: if True, tree will be walked from top down.
-            onerror: if set, onerror function will be called if an error
-                occurs (by default errors are ignored).
-            dvcfiles: if True, dvcfiles will be included in the files list
-                for walked directories.
-
-        Any kwargs will be passed into methods used for fetching and/or
-        streaming DVC outs from remotes.
-        """
-        assert topdown
-
-        if not self.exists(top):
-            if onerror is not None:
-                onerror(FileNotFoundError(top))
-            return
-
-        if not self.isdir(top):
-            if onerror is not None:
-                onerror(NotADirectoryError(top))
-            return
-
-        dvc_exists = self.dvctree and self.dvctree.exists(top)
-        repo_exists = self.repo.tree.exists(top)
-        if dvc_exists and not repo_exists:
-            yield from self.dvctree.walk(
-                top, topdown=topdown, onerror=onerror, **kwargs
-            )
-            return
-        if repo_exists and not dvc_exists:
-            yield from self.repo.tree.walk(
-                top, topdown=topdown, onerror=onerror
-            )
-            return
-
-        dvc_walk = self.dvctree.walk(top, topdown=topdown, **kwargs)
-        repo_walk = self.repo.tree.walk(top, topdown=topdown)
-        yield from self._walk(dvc_walk, repo_walk, dvcfiles=dvcfiles)
-
-    def walk_files(self, top, **kwargs):  # pylint: disable=arguments-differ
-        for root, _, files in self.walk(top, **kwargs):
-            for fname in files:
-                yield PathInfo(root) / fname
-
-    def get_file_hash(self, path_info):
-        """Return file checksum for specified path.
-
-        If path_info is a DVC out, the pre-computed checksum for the file
-        will be used. If path_info is a git file, MD5 will be computed for
-        the git object.
-        """
-        if not self.exists(path_info):
-            raise FileNotFoundError
-        if self.dvctree and self.dvctree.exists(path_info):
-            try:
-                return self.dvctree.get_file_hash(path_info)
-            except OutputNotFoundError:
-                pass
-        return file_md5(path_info, self)[0]
-
-    def copytree(self, top, dest):
-        top = PathInfo(top)
-        dest = PathInfo(dest)
-
-        if not self.exists(top):
-            raise FileNotFoundError
-
-        if self.isfile(top):
-            makedirs(dest.parent, exist_ok=True)
-            with self.open(top, mode="rb") as fobj:
-                copy_fobj_to_file(fobj, dest)
-            return
-
-        for root, _, files in self.walk(top):
-            root = PathInfo(root)
-            dest_dir = dest / root.relative_to(top)
-            makedirs(dest_dir, exist_ok=True)
-            for fname in files:
-                src = root / fname
-                with self.open(src, mode="rb") as fobj:
-                    copy_fobj_to_file(fobj, dest_dir / fname)
-
-    @property
-    def hash_jobs(self):  # pylint: disable=invalid-overridden-method
-        return self.repo.tree.hash_jobs

dvc/scheme.py

@@ -9,3 +9,5 @@ class Schemes:
     GDRIVE = "gdrive"
     LOCAL = "local"
     OSS = "oss"
+    WEBDAV = "webdav"
+    WEBDAVS = "webdavs"

dvc/scm/base.py

@@ -154,6 +154,11 @@ class Base:
         Method to remind user to track newly created files handled by scm
         """
 
+    def track_changed_files(self):
+        """
+        Method to stage files that have changed
+        """
+
     def track_file(self, path):
         """
         Method to add file to mechanism that will remind user

dvc/scm/git.py

@@ -5,7 +5,6 @@ import os
 import shlex
 from functools import partial
 
-import yaml
 from funcy import cached_property
 from pathspec.patterns import GitWildMatchPattern
 
@@ -20,6 +19,7 @@ from dvc.scm.base import (
 )
 from dvc.utils import fix_env, is_binary, relpath
 from dvc.utils.fs import path_isin
+from dvc.utils.serialize import modify_yaml
 
 logger = logging.getLogger(__name__)
 
@@ -307,7 +307,21 @@ class Git(Base):
 
         os.chmod(hook, 0o777)
 
+    def _install_merge_driver(self):
+        self.repo.git.config("merge.dvc.name", "DVC merge driver")
+        self.repo.git.config(
+            "merge.dvc.driver",
+            (
+                "dvc git-hook merge-driver "
+                "--ancestor %O "
+                "--our %A "
+                "--their %B "
+            ),
+        )
+
     def install(self, use_pre_commit_tool=False):
+        self._install_merge_driver()
+
         if not use_pre_commit_tool:
             self._verify_dvc_hooks()
             self._install_hook("post-checkout")
@@ -316,41 +330,32 @@ class Git(Base):
             return
 
         config_path = os.path.join(self.root_dir, ".pre-commit-config.yaml")
+        with modify_yaml(config_path) as config:
+            entry = {
+                "repo": "https://github.com/iterative/dvc",
+                "rev": "master",
+                "hooks": [
+                    {
+                        "id": "dvc-pre-commit",
+                        "language_version": "python3",
+                        "stages": ["commit"],
+                    },
+                    {
+                        "id": "dvc-pre-push",
+                        "language_version": "python3",
+                        "stages": ["push"],
+                    },
+                    {
+                        "id": "dvc-post-checkout",
+                        "language_version": "python3",
+                        "stages": ["post-checkout"],
+                        "always_run": True,
+                    },
+                ],
+            }
 
-        config = {}
-        if os.path.exists(config_path):
-            with open(config_path) as fobj:
-                config = yaml.safe_load(fobj)
-
-        entry = {
-            "repo": "https://github.com/iterative/dvc",
-            "rev": "master",
-            "hooks": [
-                {
-                    "id": "dvc-pre-commit",
-                    "language_version": "python3",
-                    "stages": ["commit"],
-                },
-                {
-                    "id": "dvc-pre-push",
-                    "language_version": "python3",
-                    "stages": ["push"],
-                },
-                {
-                    "id": "dvc-post-checkout",
-                    "language_version": "python3",
-                    "stages": ["post-checkout"],
-                    "always_run": True,
-                },
-            ],
-        }
-
-        if entry in config["repos"]:
-            return
-
-        config["repos"].append(entry)
-        with open(config_path, "w+") as fobj:
-            yaml.dump(config, fobj)
+            if entry not in config["repos"]:
+                config["repos"].append(entry)
 
     def cleanup_ignores(self):
         for path in self.ignored_paths:
@@ -376,6 +381,12 @@ class Git(Base):
             "\tgit add {files}".format(files=files)
         )
 
+    def track_changed_files(self):
+        if not self.files_to_track:
+            return
+
+        self.add(self.files_to_track)
+
     def track_file(self, path):
         self.files_to_track.add(path)
 
@@ -393,9 +404,10 @@ class Git(Base):
         return self.repo.rev_parse("HEAD").hexsha
 
     def resolve_rev(self, rev):
-        from git.exc import BadName, GitCommandError
         from contextlib import suppress
 
+        from git.exc import BadName, GitCommandError
+
         def _resolve_rev(name):
             with suppress(BadName, GitCommandError):
                 try:
@@ -404,6 +416,8 @@ class Git(Base):
                 except NotImplementedError:
                     # Fall back to `git rev-parse` for advanced features
                     return self.repo.git.rev_parse(name)
+                except ValueError:
+                    raise RevError(f"unknown Git revision '{name}'")
 
         # Resolve across local names
         sha = _resolve_rev(rev)

dvc/stage/__init__.py

@@ -7,7 +7,7 @@ from funcy import cached_property, project
 
 import dvc.dependency as dependency
 import dvc.prompt as prompt
-from dvc.exceptions import CheckoutError, DvcException
+from dvc.exceptions import CheckoutError, DvcException, MergeError
 from dvc.utils import relpath
 
 from . import params
@@ -306,7 +306,12 @@ class Stage(params.StageParams):
     @rwlocked(read=["deps"], write=["outs"])
     def reproduce(self, interactive=False, **kwargs):
         if not (kwargs.get("force", False) or self.changed()):
-            logger.info("Stage '%s' didn't change, skipping", self.addressing)
+            if not isinstance(self, PipelineStage) and self.is_data_source:
+                logger.info("'%s' didn't change, skipping", self.addressing)
+            else:
+                logger.info(
+                    "Stage '%s' didn't change, skipping", self.addressing
+                )
             return None
 
         msg = (
@@ -396,7 +401,7 @@ class Stage(params.StageParams):
 
     @staticmethod
     def _changed_entries(entries):
-        return [str(entry) for entry in entries if entry.changed_checksum()]
+        return [str(entry) for entry in entries if entry.workspace_status()]
 
     def _changed_stage_entry(self):
         return f"'md5' of {self} changed."
@@ -437,7 +442,7 @@ class Stage(params.StageParams):
             if not no_commit:
                 self.commit()
 
-    def _filter_outs(self, path_info):
+    def filter_outs(self, path_info):
         def _func(o):
             return path_info.isin_or_eq(o.path_info)
 
@@ -446,7 +451,7 @@ class Stage(params.StageParams):
     @rwlocked(write=["outs"])
     def checkout(self, **kwargs):
         stats = defaultdict(list)
-        for out in self._filter_outs(kwargs.get("filter_info")):
+        for out in self.filter_outs(kwargs.get("filter_info")):
             key, outs = self._checkout(out, **kwargs)
             if key:
                 stats[key].extend(outs)
@@ -464,13 +469,13 @@ class Stage(params.StageParams):
             return "failed", exc.target_infos
 
     @rwlocked(read=["deps", "outs"])
-    def status(self, check_updates=False):
+    def status(self, check_updates=False, filter_info=None):
         ret = []
         show_import = self.is_repo_import and check_updates
 
         if not self.frozen or show_import:
             self._status_deps(ret)
-        self._status_outs(ret)
+        self._status_outs(ret, filter_info=filter_info)
         self._status_always_changed(ret)
         self._status_stage(ret)
         return {self.addressing: ret} if ret else {}
@@ -489,8 +494,9 @@ class Stage(params.StageParams):
         if deps_status:
             ret.append({"changed deps": deps_status})
 
-    def _status_outs(self, ret):
-        outs_status = self._status(self.outs)
+    def _status_outs(self, ret, filter_info):
+        filter_outs = self.filter_outs(filter_info)
+        outs_status = self._status(filter_outs)
         if outs_status:
             ret.append({"changed outs": outs_status})
 
@@ -521,18 +527,56 @@ class Stage(params.StageParams):
     def get_all_files_number(self, filter_info=None):
         return sum(
             out.get_files_number(filter_info)
-            for out in self._filter_outs(filter_info)
+            for out in self.filter_outs(filter_info)
         )
 
     def get_used_cache(self, *args, **kwargs):
         from dvc.cache import NamedCache
 
         cache = NamedCache()
-        for out in self._filter_outs(kwargs.get("filter_info")):
+        for out in self.filter_outs(kwargs.get("filter_info")):
             cache.update(out.get_used_cache(*args, **kwargs))
 
         return cache
 
+    @staticmethod
+    def _check_can_merge(stage, ancestor_out=None):
+        if isinstance(stage, PipelineStage):
+            raise MergeError("unable to auto-merge pipeline stages")
+
+        if not stage.is_data_source or stage.deps or len(stage.outs) > 1:
+            raise MergeError(
+                "unable to auto-merge DVC-files that weren't "
+                "created by `dvc add`"
+            )
+
+        if ancestor_out and not stage.outs:
+            raise MergeError(
+                "unable to auto-merge DVC-files with deleted outputs"
+            )
+
+    def merge(self, ancestor, other):
+        assert other
+
+        if not other.outs:
+            return
+
+        if not self.outs:
+            self.outs = other.outs
+            return
+
+        if ancestor:
+            self._check_can_merge(ancestor)
+            outs = ancestor.outs
+            ancestor_out = outs[0] if outs else None
+        else:
+            ancestor_out = None
+
+        self._check_can_merge(self, ancestor_out)
+        self._check_can_merge(other, ancestor_out)
+
+        self.outs[0].merge(ancestor_out, other.outs[0])
+
 
 class PipelineStage(Stage):
     def __init__(self, *args, name=None, **kwargs):
@@ -572,3 +616,6 @@ class PipelineStage(Stage):
 
     def _changed_stage_entry(self):
         return f"'cmd' of {self} has changed."
+
+    def merge(self, ancestor, other):
+        raise NotImplementedError

dvc/stage/cache.py

@@ -1,16 +1,17 @@
 import logging
 import os
+import tempfile
 from contextlib import contextmanager
 
-import yaml
 from funcy import first
 from voluptuous import Invalid
 
 from dvc.cache.local import _log_exceptions
+from dvc.exceptions import DvcException
+from dvc.path_info import PathInfo
 from dvc.schema import COMPILED_LOCK_FILE_STAGE_SCHEMA
 from dvc.utils import dict_sha256, relpath
-from dvc.utils.fs import makedirs
-from dvc.utils.yaml import dump_yaml
+from dvc.utils.serialize import YAMLFileCorruptedError, dump_yaml, load_yaml
 
 from .loader import StageLoader
 from .serialize import to_single_stage_lockfile
@@ -18,6 +19,11 @@ from .serialize import to_single_stage_lockfile
 logger = logging.getLogger(__name__)
 
 
+class RunCacheNotFoundError(DvcException):
+    def __init__(self, stage):
+        super().__init__(f"No run-cache for {stage.addressing}")
+
+
 def _get_cache_hash(cache, key=False):
     if key:
         cache["outs"] = [out["path"] for out in cache.get("outs", [])]
@@ -29,7 +35,7 @@ def _get_stage_hash(stage):
         return None
 
     for dep in stage.deps:
-        if not (dep.scheme == "local" and dep.def_path and dep.get_checksum()):
+        if not (dep.scheme == "local" and dep.def_path and dep.get_hash()):
             return None
 
     for out in stage.outs:
@@ -44,6 +50,10 @@ class StageCache:
         self.repo = repo
         self.cache_dir = os.path.join(repo.cache.local.cache_dir, "runs")
 
+    @property
+    def tree(self):
+        return self.repo.cache.local.tree
+
     def _get_cache_dir(self, key):
         return os.path.join(self.cache_dir, key[:2], key)
 
@@ -54,11 +64,10 @@ class StageCache:
         path = self._get_cache_path(key, value)
 
         try:
-            with open(path) as fobj:
-                return COMPILED_LOCK_FILE_STAGE_SCHEMA(yaml.safe_load(fobj))
+            return COMPILED_LOCK_FILE_STAGE_SCHEMA(load_yaml(path))
         except FileNotFoundError:
             return None
-        except (yaml.error.YAMLError, Invalid):
+        except (YAMLFileCorruptedError, Invalid):
             logger.warning("corrupted cache file '%s'.", relpath(path))
             os.unlink(path)
             return None
@@ -80,7 +89,7 @@ class StageCache:
         return None
 
     def _create_stage(self, cache, wdir=None):
-        from . import create_stage, PipelineStage
+        from . import PipelineStage, create_stage
 
         stage = create_stage(
             PipelineStage,
@@ -139,23 +148,42 @@ class StageCache:
         # sanity check
         COMPILED_LOCK_FILE_STAGE_SCHEMA(cache)
 
-        path = self._get_cache_path(cache_key, cache_value)
-        dpath = os.path.dirname(path)
-        makedirs(dpath, exist_ok=True)
-        dump_yaml(path, cache)
+        path = PathInfo(self._get_cache_path(cache_key, cache_value))
+        self.tree.makedirs(path.parent)
+        tmp = tempfile.NamedTemporaryFile(delete=False, dir=path.parent).name
+        assert os.path.exists(path.parent)
+        assert os.path.isdir(path.parent)
+        dump_yaml(tmp, cache)
+        self.tree.move(PathInfo(tmp), path)
 
-    def is_cached(self, stage):
-        return bool(self._load(stage))
-
-    def restore(self, stage):
+    def _restore(self, stage):
+        stage.save_deps()
         cache = self._load(stage)
         if not cache:
-            return
-        StageLoader.fill_from_lock(stage, cache)
+            raise RunCacheNotFoundError(stage)
 
+        StageLoader.fill_from_lock(stage, cache)
         for out in self._uncached_outs(stage, cache):
             out.checkout()
 
+        if not stage.outs_cached():
+            raise RunCacheNotFoundError(stage)
+
+    def restore(self, stage, run_cache=True):
+        if stage.is_callback or stage.always_changed:
+            raise RunCacheNotFoundError(stage)
+
+        if not stage.already_cached():
+            if not run_cache:  # backward compatibility
+                raise RunCacheNotFoundError(stage)
+            self._restore(stage)
+
+        logger.info(
+            "Stage '%s' is cached - skipping run, checking out outputs",
+            stage.addressing,
+        )
+        stage.checkout()
+
     @staticmethod
     def _transfer(func, from_remote, to_remote):
         ret = []

dvc/stage/decorators.py

@@ -6,8 +6,9 @@ from funcy import decorator
 @decorator
 def rwlocked(call, read=None, write=None):
     import sys
-    from dvc.rwlock import rwlock
+
     from dvc.dependency.repo import RepoDependency
+    from dvc.rwlock import rwlock
 
     if read is None:
         read = []

dvc/stage/loader.py

@@ -6,6 +6,7 @@ from itertools import chain
 from funcy import get_in, lcat, project
 
 from dvc import dependency, output
+from dvc.hash_info import HashInfo
 
 from . import PipelineStage, Stage, loads_from
 from .exceptions import StageNameUnspecified, StageNotFound
@@ -42,7 +43,10 @@ class StageLoader(Mapping):
             if isinstance(item, dependency.ParamsDependency):
                 item.fill_values(get_in(lock_data, [stage.PARAM_PARAMS, path]))
                 continue
-            item.checksum = get_in(checksums, [key, path, item.checksum_type])
+            info = get_in(checksums, [key, path], {})
+            info = info.copy()
+            info.pop("path", None)
+            item.hash_info = HashInfo.from_dict(info)
 
     @classmethod
     def load_stage(cls, dvcfile, name, stage_data, lock_data=None):

dvc/stage/run.py

@@ -81,39 +81,15 @@ def cmd_run(stage, *args, **kwargs):
         raise StageCmdFailedError(stage.cmd, retcode)
 
 
-def _is_cached(stage):
-    cached = (
-        not stage.is_callback
-        and not stage.always_changed
-        and stage.already_cached()
-    )
-    if cached:
-        logger.info("Stage '%s' is cached", stage.addressing)
-    return cached
-
-
-def restored_from_cache(stage):
-    stage.save_deps()
-    stage_cache = stage.repo.stage_cache
-    if not stage_cache.is_cached(stage):
-        return False
-    # restore stage from build cache
-    stage_cache.restore(stage)
-    restored = stage.outs_cached()
-    if restored:
-        logger.info("Restored stage '%s' from run-cache", stage.addressing)
-    return restored
-
-
 def run_stage(stage, dry=False, force=False, run_cache=False):
     if not (dry or force):
-        stage_cached = _is_cached(stage) or (
-            run_cache and restored_from_cache(stage)
-        )
-        if stage_cached:
-            logger.info("Skipping run, checking out outputs")
-            stage.checkout()
+        from .cache import RunCacheNotFoundError
+
+        try:
+            stage.repo.stage_cache.restore(stage, run_cache=run_cache)
             return
+        except RunCacheNotFoundError:
+            pass
 
     callback_str = "callback " if stage.is_callback else ""
     logger.info(

dvc/stage/serialize.py

@@ -8,7 +8,7 @@ from funcy import post_processing
 from dvc.dependency import ParamsDependency
 from dvc.output import BaseOutput
 from dvc.utils.collections import apply_diff
-from dvc.utils.yaml import parse_yaml_for_update
+from dvc.utils.serialize import parse_yaml_for_update
 
 from .params import StageParams
 from .utils import resolve_wdir, split_params_deps
@@ -141,7 +141,7 @@ def to_single_stage_lockfile(stage: "Stage") -> dict:
             OrderedDict(
                 [
                     (PARAM_PATH, item.def_path),
-                    (item.checksum_type, item.checksum),
+                    *item.hash_info.to_dict().items(),
                 ]
             )
             for item in sort_by_path(items)

dvc/stage/utils.py

@@ -110,9 +110,10 @@ def check_circular_dependency(stage):
 
 
 def check_duplicated_arguments(stage):
-    from dvc.exceptions import ArgumentDuplicationError
     from collections import Counter
 
+    from dvc.exceptions import ArgumentDuplicationError
+
     path_counts = Counter(edge.path_info for edge in stage.deps + stage.outs)
 
     for path, occurrence in path_counts.items():

dvc/system.py

@@ -119,13 +119,14 @@ class System:
     @staticmethod
     def _getdirinfo(path):
         from collections import namedtuple
+
         from win32file import (  # pylint: disable=import-error
-            CreateFileW,
-            GetFileInformationByHandle,
             FILE_FLAG_BACKUP_SEMANTICS,
             FILE_FLAG_OPEN_REPARSE_POINT,
             FILE_SHARE_READ,
             OPEN_EXISTING,
+            CreateFileW,
+            GetFileInformationByHandle,
         )
 
         # NOTE: use FILE_FLAG_OPEN_REPARSE_POINT to open symlink itself and not

dvc/tree/__init__.py

@@ -11,6 +11,8 @@ from .local import LocalTree
 from .oss import OSSTree
 from .s3 import S3Tree
 from .ssh import SSHTree
+from .webdav import WebDAVTree
+from .webdavs import WebDAVSTree
 
 TREES = [
     AzureTree,
@@ -22,6 +24,8 @@ TREES = [
     S3Tree,
     SSHTree,
     OSSTree,
+    WebDAVTree,
+    WebDAVSTree,
     # NOTE: LocalTree is the default
 ]
 
@@ -39,10 +43,10 @@ def _get_conf(repo, **kwargs):
         remote_conf = repo.config["remote"][name.lower()]
     else:
         remote_conf = kwargs
-    return _resolve_remote_refs(repo.config, remote_conf)
+    return _resolve_remote_refs(repo, remote_conf)
 
 
-def _resolve_remote_refs(config, remote_conf):
+def _resolve_remote_refs(repo, remote_conf):
     # Support for cross referenced remotes.
     # This will merge the settings, shadowing base ref with remote_conf.
     # For example, having:
@@ -68,11 +72,17 @@ def _resolve_remote_refs(config, remote_conf):
     if parsed.scheme != "remote":
         return remote_conf
 
-    base = config["remote"][parsed.netloc]
+    base = _get_conf(repo, name=parsed.netloc)
     url = posixpath.join(base["url"], parsed.path.lstrip("/"))
     return {**base, **remote_conf, "url": url}
 
 
 def get_cloud_tree(repo, **kwargs):
+    from dvc.config import SCHEMA, ConfigError, Invalid
+
     remote_conf = _get_conf(repo, **kwargs)
+    try:
+        remote_conf = SCHEMA["remote"][str](remote_conf)
+    except Invalid as exc:
+        raise ConfigError(str(exc)) from None
     return _get_tree(remote_conf)(repo, remote_conf)

dvc/tree/_metadata.py

@@ -0,0 +1,72 @@
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, List
+
+from dvc.output import BaseOutput
+from dvc.path_info import PathInfo
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+
+@dataclass
+class Metadata:
+    """
+    Container for storing metadata for a given path, similar in spirit to
+    `os.stat_result`.
+    """
+
+    # required field
+    path_info: PathInfo
+    repo: "Repo"
+
+    # computed fields
+    is_output: bool = field(init=False, default=False)  # is it an output?
+    # is the path part of an output?
+    part_of_output: bool = field(init=False, default=False)
+    # does the path contain outputs?
+    contains_outputs: bool = field(init=False, default=False)
+    # is the path tracked by dvc? equivalent to (is_output or part_of_output)
+    is_dvc: bool = field(init=False, default=False)
+    # does dvc has any output in that path or inside it? equiv. to bool(outs)
+    output_exists: bool = field(init=False)
+
+    # optional fields
+    isdir: bool = False  # is it a directory?
+    is_exec: bool = False  # is it an executable?
+    outs: List[BaseOutput] = field(default_factory=list)  # list of outputs
+
+    def __post_init__(self):
+        self.output_exists = bool(self.outs)
+
+        if not self.output_exists:
+            return
+
+        # it can contain multiple outputs
+        if len(self.outs) > 1:
+            self.contains_outputs = True
+        else:
+            out = self.outs[0]
+            # or, the path itself could be an output
+            self.is_output = self.path_info == out.path_info
+
+            # or, a directory must have an output somewhere deep inside it
+            if not self.is_output:
+                self.contains_outputs = out.path_info.isin(self.path_info)
+
+        # or, the path could be a part of an output, i.e. inside of an output
+        self.part_of_output = not self.is_output and not self.contains_outputs
+
+        self.is_dvc = self.is_output or self.part_of_output
+        # if it contains outputs, it must be a dir
+        self.isdir = self.contains_outputs
+
+    @property
+    def isfile(self):
+        # self.isdir might require additional computation, especially if it's
+        # a dir that's inside a dvc-tracked dir
+        # since, `Metadata()` does not support being used as null (i.e. when
+        # path does not exist), `isfile` becomes `isdir`'s complement.
+        return not self.isdir
+
+    def __str__(self):
+        return f"Metadata: {self.path_info}"

dvc/tree/azure.py

@@ -5,6 +5,7 @@ from datetime import datetime, timedelta
 
 from funcy import cached_property, wrap_prop
 
+from dvc.hash_info import HashInfo
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.scheme import Schemes
@@ -35,16 +36,20 @@ class AzureTree(BaseTree):
             container = self._az_config.get("storage", "container_name", None)
             self.path_info = self.PATH_CLS(f"azure://{container}")
 
-        self._conn_kwargs = {
-            opt: config.get(opt) or self._az_config.get("storage", opt, None)
-            for opt in ["connection_string", "sas_token"]
-        }
-        self._conn_kwargs["account_name"] = self._az_config.get(
-            "storage", "account", None
-        )
-        self._conn_kwargs["account_key"] = self._az_config.get(
-            "storage", "key", None
+        self._conn_str = config.get(
+            "connection_string"
+        ) or self._az_config.get("storage", "connection_string", None)
+
+        self._account_url = None
+        if not self._conn_str:
+            name = self._az_config.get("storage", "account", None)
+            self._account_url = f"https://{name}.blob.core.windows.net"
+
+        self._credential = config.get("sas_token") or self._az_config.get(
+            "storage", "sas_token", None
         )
+        if not self._credential:
+            self._credential = self._az_config.get("storage", "key", None)
 
     @cached_property
     def _az_config(self):
@@ -62,64 +67,78 @@ class AzureTree(BaseTree):
     @cached_property
     def blob_service(self):
         # pylint: disable=no-name-in-module
-        from azure.storage.blob import BlockBlobService
-        from azure.common import AzureMissingResourceHttpError
+        from azure.core.exceptions import (
+            HttpResponseError,
+            ResourceNotFoundError,
+        )
+        from azure.storage.blob import BlobServiceClient
 
         logger.debug(f"URL {self.path_info}")
-        logger.debug(f"Connection options {self._conn_kwargs}")
-        blob_service = BlockBlobService(**self._conn_kwargs)
-        logger.debug(f"Container name {self.path_info.bucket}")
-        try:  # verify that container exists
-            blob_service.list_blobs(
-                self.path_info.bucket, delimiter="/", num_results=1
+
+        if self._conn_str:
+            logger.debug(f"Using connection string '{self._conn_str}'")
+            blob_service = BlobServiceClient.from_connection_string(
+                self._conn_str, credential=self._credential
             )
-        except AzureMissingResourceHttpError:
-            blob_service.create_container(self.path_info.bucket)
+        else:
+            logger.debug(f"Using account url '{self._account_url}'")
+            blob_service = BlobServiceClient(
+                self._account_url, credential=self._credential
+            )
+
+        logger.debug(f"Container name {self.path_info.bucket}")
+        container_client = blob_service.get_container_client(
+            self.path_info.bucket
+        )
+
+        try:  # verify that container exists
+            container_client.get_container_properties()
+        except ResourceNotFoundError:
+            container_client.create_container()
+        except HttpResponseError as exc:
+            # client may not have account-level privileges
+            if exc.status_code != 403:
+                raise
+
         return blob_service
 
     def get_etag(self, path_info):
-        etag = self.blob_service.get_blob_properties(
+        blob_client = self.blob_service.get_blob_client(
             path_info.bucket, path_info.path
-        ).properties.etag
+        )
+        etag = blob_client.get_blob_properties().etag
         return etag.strip('"')
 
     def _generate_download_url(self, path_info, expires=3600):
         from azure.storage.blob import (  # pylint:disable=no-name-in-module
-            BlobPermissions,
+            BlobSasPermissions,
+            generate_blob_sas,
         )
 
         expires_at = datetime.utcnow() + timedelta(seconds=expires)
 
-        sas_token = self.blob_service.generate_blob_shared_access_signature(
-            path_info.bucket,
-            path_info.path,
-            permission=BlobPermissions.READ,
+        blob_client = self.blob_service.get_blob_client(
+            path_info.bucket, path_info.path
+        )
+
+        sas_token = generate_blob_sas(
+            blob_client.account_name,
+            blob_client.container_name,
+            blob_client.blob_name,
+            account_key=blob_client.credential.account_key,
+            permission=BlobSasPermissions(read=True),
             expiry=expires_at,
         )
-        download_url = self.blob_service.make_blob_url(
-            path_info.bucket, path_info.path, sas_token=sas_token
-        )
-        return download_url
+        return blob_client.url + "?" + sas_token
 
     def exists(self, path_info, use_dvcignore=True):
         paths = self._list_paths(path_info.bucket, path_info.path)
         return any(path_info.path == path for path in paths)
 
     def _list_paths(self, bucket, prefix):
-        blob_service = self.blob_service
-        next_marker = None
-        while True:
-            blobs = blob_service.list_blobs(
-                bucket, prefix=prefix, marker=next_marker
-            )
-
-            for blob in blobs:
-                yield blob.name
-
-            if not blobs.next_marker:
-                break
-
-            next_marker = blobs.next_marker
+        container_client = self.blob_service.get_container_client(bucket)
+        for blob in container_client.list_blobs(name_starts_with=prefix):
+            yield blob.name
 
     def walk_files(self, path_info, **kwargs):
         if not kwargs.pop("prefix", False):
@@ -137,29 +156,37 @@ class AzureTree(BaseTree):
             raise NotImplementedError
 
         logger.debug(f"Removing {path_info}")
-        self.blob_service.delete_blob(path_info.bucket, path_info.path)
+        self.blob_service.get_blob_client(
+            path_info.bucket, path_info.path
+        ).delete_blob()
 
     def get_file_hash(self, path_info):
-        return self.get_etag(path_info)
+        return HashInfo(self.PARAM_CHECKSUM, self.get_etag(path_info))
 
     def _upload(
         self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
     ):
-        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
-            self.blob_service.create_blob_from_path(
-                to_info.bucket,
-                to_info.path,
-                from_file,
-                progress_callback=pbar.update_to,
-            )
+
+        blob_client = self.blob_service.get_blob_client(
+            to_info.bucket, to_info.path
+        )
+        total = os.path.getsize(from_file)
+        with open(from_file, "rb") as fobj:
+            with Tqdm.wrapattr(
+                fobj, "read", desc=name, total=total, disable=no_progress_bar
+            ) as wrapped:
+                blob_client.upload_blob(wrapped, overwrite=True)
 
     def _download(
         self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
     ):
-        with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
-            self.blob_service.get_blob_to_path(
-                from_info.bucket,
-                from_info.path,
-                to_file,
-                progress_callback=pbar.update_to,
-            )
+        blob_client = self.blob_service.get_blob_client(
+            from_info.bucket, from_info.path
+        )
+        total = blob_client.get_blob_properties().size
+        stream = blob_client.download_blob()
+        with open(to_file, "wb") as fobj:
+            with Tqdm.wrapattr(
+                fobj, "write", desc=name, total=total, disable=no_progress_bar
+            ) as wrapped:
+                stream.readinto(wrapped)

dvc/tree/base.py

@@ -1,22 +1,16 @@
 import itertools
-import json
 import logging
-import tempfile
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from functools import partial
 from multiprocessing import cpu_count
-from operator import itemgetter
 from urllib.parse import urlparse
 
 from funcy import cached_property
 
-from dvc.exceptions import (
-    DvcException,
-    DvcIgnoreInCollectedDirError,
-    RemoteCacheRequiredError,
-)
+from dvc.exceptions import DvcException, DvcIgnoreInCollectedDirError
+from dvc.hash_info import HashInfo
 from dvc.ignore import DvcIgnore
-from dvc.path_info import PathInfo, URLInfo
+from dvc.path_info import URLInfo
 from dvc.progress import Tqdm
 from dvc.state import StateNoop
 from dvc.utils import tmp_fname
@@ -81,8 +75,8 @@ class BaseTree:
     @cached_property
     def hash_jobs(self):
         return (
-            self.config.get("hash_jobs")
-            or (self.repo and self.repo.config["core"].get("hash_jobs"))
+            self.config.get("checksum_jobs")
+            or (self.repo and self.repo.config["core"].get("checksum_jobs"))
             or self.HASH_JOBS
         )
 
@@ -236,22 +230,17 @@ class BaseTree:
             return False
         return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
 
-    def get_hash(self, path_info, tree=None, **kwargs):
+    def get_hash(self, path_info, **kwargs):
         assert path_info and (
             isinstance(path_info, str) or path_info.scheme == self.scheme
         )
 
-        if not tree:
-            tree = self
-
-        if not tree.exists(path_info):
+        if not self.exists(path_info):
             return None
 
-        if tree == self:
-            # pylint: disable=assignment-from-none
-            hash_ = self.state.get(path_info)
-        else:
-            hash_ = None
+        # pylint: disable=assignment-from-none
+        hash_ = self.state.get(path_info)
+
         # If we have dir hash in state db, but dir cache file is lost,
         # then we need to recollect the dir via .get_dir_hash() call below,
         # see https://github.com/iterative/dvc/issues/2219 for context
@@ -265,28 +254,24 @@ class BaseTree:
             hash_ = None
 
         if hash_:
-            return hash_
+            hash_info = HashInfo(self.PARAM_CHECKSUM, hash_)
+            if hash_info.isdir:
+                hash_info.dir_info = self.cache.get_dir_cache(hash_info)
+            return hash_info
 
-        if tree.isdir(path_info):
-            hash_ = self.get_dir_hash(path_info, tree, **kwargs)
+        if self.isdir(path_info):
+            hash_info = self.get_dir_hash(path_info, **kwargs)
         else:
-            hash_ = tree.get_file_hash(path_info)
+            hash_info = self.get_file_hash(path_info)
 
-        if hash_ and self.exists(path_info):
-            self.state.save(path_info, hash_)
+        if hash_info and self.exists(path_info):
+            self.state.save(path_info, hash_info.value)
 
-        return hash_
+        return hash_info
 
     def get_file_hash(self, path_info):
         raise NotImplementedError
 
-    def get_dir_hash(self, path_info, tree, **kwargs):
-        if not self.cache:
-            raise RemoteCacheRequiredError(path_info)
-
-        dir_info = self._collect_dir(path_info, tree, **kwargs)
-        return self._save_dir_info(dir_info, path_info)
-
     def hash_to_path_info(self, hash_):
         return self.path_info / hash_[0:2] / hash_[2:]
 
@@ -298,29 +283,22 @@ class BaseTree:
 
         return "".join(parts)
 
-    def save_info(self, path_info, tree=None, **kwargs):
-        return {
-            self.PARAM_CHECKSUM: self.get_hash(path_info, tree=tree, **kwargs)
-        }
-
-    @staticmethod
-    def _calculate_hashes(file_infos, tree):
+    def _calculate_hashes(self, file_infos):
         file_infos = list(file_infos)
         with Tqdm(
             total=len(file_infos),
             unit="md5",
             desc="Computing file/dir hashes (only done once)",
         ) as pbar:
-            worker = pbar.wrap_fn(tree.get_file_hash)
-            with ThreadPoolExecutor(max_workers=tree.hash_jobs) as executor:
-                tasks = executor.map(worker, file_infos)
-                hashes = dict(zip(file_infos, tasks))
-        return hashes
+            worker = pbar.wrap_fn(self.get_file_hash)
+            with ThreadPoolExecutor(max_workers=self.hash_jobs) as executor:
+                hashes = (hi.value for hi in executor.map(worker, file_infos))
+                return dict(zip(file_infos, hashes))
 
-    def _collect_dir(self, path_info, tree, **kwargs):
+    def _collect_dir(self, path_info, **kwargs):
         file_infos = set()
 
-        for fname in tree.walk_files(path_info, **kwargs):
+        for fname in self.walk_files(path_info, **kwargs):
             if DvcIgnore.DVCIGNORE_FILE == fname.name:
                 raise DvcIgnoreInCollectedDirError(fname.parent)
 
@@ -329,10 +307,10 @@ class BaseTree:
         hashes = {fi: self.state.get(fi) for fi in file_infos}
         not_in_state = {fi for fi, hash_ in hashes.items() if hash_ is None}
 
-        new_hashes = self._calculate_hashes(not_in_state, tree)
+        new_hashes = self._calculate_hashes(not_in_state)
         hashes.update(new_hashes)
 
-        result = [
+        return [
             {
                 self.PARAM_CHECKSUM: hashes[fi],
                 # NOTE: this is lossy transformation:
@@ -348,36 +326,9 @@ class BaseTree:
             for fi in file_infos
         ]
 
-        # Sorting the list by path to ensure reproducibility
-        return sorted(result, key=itemgetter(self.PARAM_RELPATH))
-
-    def _save_dir_info(self, dir_info, path_info):
-        hash_, tmp_info = self._get_dir_info_hash(dir_info)
-        new_info = self.cache.tree.hash_to_path_info(hash_)
-        if self.cache.changed_cache_file(hash_):
-            self.cache.tree.makedirs(new_info.parent)
-            self.cache.tree.move(
-                tmp_info, new_info, mode=self.cache.CACHE_MODE
-            )
-
-        if self.exists(path_info):
-            self.state.save(path_info, hash_)
-        self.state.save(new_info, hash_)
-
-        return hash_
-
-    def _get_dir_info_hash(self, dir_info):
-        tmp = tempfile.NamedTemporaryFile(delete=False).name
-        with open(tmp, "w+") as fobj:
-            json.dump(dir_info, fobj, sort_keys=True)
-
-        tree = self.cache.tree
-        from_info = PathInfo(tmp)
-        to_info = tree.path_info / tmp_fname("")
-        tree.upload(from_info, to_info, no_progress_bar=True)
-
-        hash_ = tree.get_file_hash(to_info) + self.CHECKSUM_DIR_SUFFIX
-        return hash_, to_info
+    def get_dir_hash(self, path_info, **kwargs):
+        dir_info = self._collect_dir(path_info, **kwargs)
+        return self.repo.cache.local.save_dir_info(dir_info)
 
     def upload(self, from_info, to_info, name=None, no_progress_bar=False):
         if not hasattr(self, "_upload"):

dvc/tree/dvc.py

@@ -0,0 +1,271 @@
+import logging
+import os
+
+from dvc.exceptions import OutputNotFoundError
+from dvc.hash_info import HashInfo
+from dvc.path_info import PathInfo
+
+from ._metadata import Metadata
+from .base import BaseTree, RemoteActionNotImplemented
+
+logger = logging.getLogger(__name__)
+
+
+class DvcTree(BaseTree):  # pylint:disable=abstract-method
+    """DVC repo tree.
+
+    Args:
+        repo: DVC repo.
+        fetch: if True, uncached DVC outs will be fetched on `open()`.
+        stream: if True, uncached DVC outs will be streamed directly from
+            remote on `open()`.
+
+    `stream` takes precedence over `fetch`. If `stream` is enabled and
+    a remote does not support streaming, uncached DVC outs will be fetched
+    as a fallback.
+    """
+
+    scheme = "local"
+    PARAM_CHECKSUM = "md5"
+
+    def __init__(self, repo, fetch=False, stream=False):
+        super().__init__(repo, {"url": repo.root_dir})
+        self.fetch = fetch
+        self.stream = stream
+
+    def _find_outs(self, path, *args, **kwargs):
+        outs = self.repo.find_outs_by_path(path, *args, **kwargs)
+
+        def _is_cached(out):
+            return out.use_cache
+
+        outs = list(filter(_is_cached, outs))
+        if not outs:
+            raise OutputNotFoundError(path, self.repo)
+
+        return outs
+
+    def _get_granular_checksum(self, path, out, remote=None):
+        assert isinstance(path, PathInfo)
+        if not self.fetch and not self.stream:
+            raise FileNotFoundError
+        dir_cache = out.get_dir_cache(remote=remote)
+        for entry in dir_cache:
+            entry_relpath = entry[out.tree.PARAM_RELPATH]
+            if os.name == "nt":
+                entry_relpath = entry_relpath.replace("/", os.sep)
+            if path == out.path_info / entry_relpath:
+                return entry[out.tree.PARAM_CHECKSUM]
+        raise FileNotFoundError
+
+    def open(
+        self, path, mode="r", encoding="utf-8", remote=None
+    ):  # pylint: disable=arguments-differ
+        try:
+            outs = self._find_outs(path, strict=False)
+        except OutputNotFoundError as exc:
+            raise FileNotFoundError from exc
+
+        if len(outs) != 1 or (
+            outs[0].is_dir_checksum and path == outs[0].path_info
+        ):
+            raise IsADirectoryError
+
+        out = outs[0]
+        if out.changed_cache(filter_info=path):
+            if not self.fetch and not self.stream:
+                raise FileNotFoundError
+
+            remote_obj = self.repo.cloud.get_remote(remote)
+            if self.stream:
+                if out.is_dir_checksum:
+                    checksum = self._get_granular_checksum(path, out)
+                else:
+                    checksum = out.hash_info.value
+                try:
+                    remote_info = remote_obj.tree.hash_to_path_info(checksum)
+                    return remote_obj.tree.open(
+                        remote_info, mode=mode, encoding=encoding
+                    )
+                except RemoteActionNotImplemented:
+                    pass
+            cache_info = out.get_used_cache(filter_info=path, remote=remote)
+            self.repo.cloud.pull(cache_info, remote=remote)
+
+        if out.is_dir_checksum:
+            checksum = self._get_granular_checksum(path, out)
+            cache_path = out.cache.tree.hash_to_path_info(checksum).url
+        else:
+            cache_path = out.cache_path
+        return open(cache_path, mode=mode, encoding=encoding)
+
+    def exists(self, path):  # pylint: disable=arguments-differ
+        try:
+            self.metadata(path)
+            return True
+        except FileNotFoundError:
+            return False
+
+    def isdir(self, path):  # pylint: disable=arguments-differ
+        try:
+            meta = self.metadata(path)
+            return meta.isdir
+        except FileNotFoundError:
+            return False
+
+    def check_isdir(self, path_info, outs):
+        if len(outs) != 1:
+            return True
+
+        out = outs[0]
+        if not out.is_dir_checksum:
+            return out.path_info != path_info
+        if out.path_info == path_info:
+            return True
+
+        try:
+            self._get_granular_checksum(path_info, out)
+            return False
+        except FileNotFoundError:
+            return True
+
+    def isfile(self, path):  # pylint: disable=arguments-differ
+        try:
+            meta = self.metadata(path)
+            return meta.isfile
+        except FileNotFoundError:
+            return False
+
+    def _fetch_dir(
+        self, out, filter_info=None, download_callback=None, **kwargs
+    ):
+        # pull dir cache if needed
+        out.get_dir_cache(**kwargs)
+
+        # pull dir contents if needed
+        if self.fetch and out.changed_cache(filter_info=filter_info):
+            used_cache = out.get_used_cache(filter_info=filter_info)
+            downloaded = self.repo.cloud.pull(used_cache, **kwargs)
+            if download_callback:
+                download_callback(downloaded)
+
+    def _add_dir(self, top, trie, out, **kwargs):
+        if not self.fetch and not self.stream:
+            return
+
+        self._fetch_dir(out, filter_info=top, **kwargs)
+
+        for entry in out.dir_cache:
+            entry_relpath = entry[out.tree.PARAM_RELPATH]
+            if os.name == "nt":
+                entry_relpath = entry_relpath.replace("/", os.sep)
+            path_info = out.path_info / entry_relpath
+            trie[path_info.parts] = None
+
+    def _walk(self, root, trie, topdown=True, **kwargs):
+        dirs = set()
+        files = []
+
+        out = trie.get(root.parts)
+        if out and out.is_dir_checksum:
+            self._add_dir(root, trie, out, **kwargs)
+
+        root_len = len(root.parts)
+        for key, out in trie.iteritems(prefix=root.parts):  # noqa: B301
+            if key == root.parts:
+                continue
+
+            name = key[root_len]
+            if len(key) > root_len + 1 or (out and out.is_dir_checksum):
+                dirs.add(name)
+                continue
+
+            files.append(name)
+
+        assert topdown
+        dirs = list(dirs)
+        yield root.fspath, dirs, files
+
+        for dname in dirs:
+            yield from self._walk(root / dname, trie)
+
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
+        from pygtrie import Trie
+
+        assert topdown
+        root = PathInfo(os.path.abspath(top))
+        try:
+            meta = self.metadata(root)
+        except FileNotFoundError:
+            if onerror is not None:
+                onerror(FileNotFoundError(top))
+            return
+
+        if not meta.isdir:
+            if onerror is not None:
+                onerror(NotADirectoryError(top))
+            return
+
+        trie = Trie()
+        for out in meta.outs:
+            trie[out.path_info.parts] = out
+
+            if out.is_dir_checksum and root.isin_or_eq(out.path_info):
+                self._add_dir(top, trie, out, **kwargs)
+
+        yield from self._walk(root, trie, topdown=topdown, **kwargs)
+
+    def walk_files(self, path_info, **kwargs):
+        for root, _, files in self.walk(path_info):
+            for fname in files:
+                # NOTE: os.path.join is ~5.5 times slower
+                yield PathInfo(f"{root}{os.sep}{fname}")
+
+    def isdvc(self, path, recursive=False, strict=True):
+        try:
+            meta = self.metadata(path)
+        except FileNotFoundError:
+            return False
+
+        recurse = recursive or not strict
+        return meta.output_exists if recurse else meta.is_output
+
+    def isexec(self, path):  # pylint: disable=unused-argument
+        return False
+
+    def get_dir_hash(self, path_info, **kwargs):
+        try:
+            outs = self._find_outs(path_info, strict=True)
+            if len(outs) == 1 and outs[0].is_dir_checksum:
+                out = outs[0]
+                # other code expects us to fetch the dir at this point
+                self._fetch_dir(out, **kwargs)
+                return out.hash_info
+        except OutputNotFoundError:
+            pass
+
+        return super().get_dir_hash(path_info, **kwargs)
+
+    def get_file_hash(self, path_info):
+        outs = self._find_outs(path_info, strict=False)
+        if len(outs) != 1:
+            raise OutputNotFoundError
+        out = outs[0]
+        if out.is_dir_checksum:
+            return HashInfo(
+                out.tree.PARAM_CHECKSUM,
+                self._get_granular_checksum(path_info, out),
+            )
+        return out.hash_info
+
+    def metadata(self, path_info):
+        path_info = PathInfo(os.path.abspath(path_info))
+
+        try:
+            outs = self._find_outs(path_info, strict=False, recursive=True)
+        except OutputNotFoundError as exc:
+            raise FileNotFoundError from exc
+
+        meta = Metadata(path_info=path_info, outs=outs, repo=self.repo)
+        meta.isdir = meta.isdir or self.check_isdir(meta.path_info, meta.outs)
+        return meta

dvc/tree/gdrive.py

@@ -366,10 +366,7 @@ class GDriveTree(BaseTree):
                 total=total,
                 disable=no_progress_bar,
             ) as wrapped:
-                # PyDrive doesn't like content property setting for empty files
-                # https://github.com/gsuitedevs/PyDrive/issues/121
-                if total:
-                    item.content = wrapped
+                item.content = wrapped
                 item.Upload()
         return item
 
@@ -381,9 +378,6 @@ class GDriveTree(BaseTree):
         # it does not create a file on the remote
         gdrive_file = self._drive.CreateFile(param)
 
-        gdrive_file.FetchMetadata(fields="fileSize")
-        size = gdrive_file["fileSize"]
-
         with Tqdm(
             desc=progress_desc,
             disable=no_progress_bar,
@@ -391,13 +385,7 @@ class GDriveTree(BaseTree):
             # explicit `bar_format` as `total` will be set by `update_to`
             bar_format=Tqdm.BAR_FMT_DEFAULT,
         ) as pbar:
-            if size:
-                gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
-            else:
-                # PyDrive doesn't like downloading empty files
-                # https://github.com/iterative/dvc/issues/4286
-                with open(to_file, "w"):
-                    pass
+            gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
 
     @contextmanager
     @_gdrive_retry

dvc/tree/git.py

@@ -1,13 +1,13 @@
 import errno
 import io
 import os
-import stat
+from functools import lru_cache
 
 from funcy import cached_property
 
 from dvc.exceptions import DvcException
 from dvc.tree.base import BaseTree
-from dvc.utils import relpath
+from dvc.utils import is_exec, relpath
 
 # see git-fast-import(1)
 GIT_MODE_DIR = 0o40000
@@ -21,6 +21,30 @@ def _item_basename(item):
     return os.path.basename(item.path)
 
 
+def _is_tree_and_contains(obj, path):
+    if obj.mode != GIT_MODE_DIR:
+        return False
+    # see https://github.com/gitpython-developers/GitPython/issues/851
+    # `return (i in tree)` doesn't work so here is a workaround:
+    for item in obj:
+        if _item_basename(item) == path:
+            return True
+    return False
+
+
+# NOTE: small lru cache covers sequential exists() calls
+@lru_cache(maxsize=1)
+def _get_obj(tree, path):
+    if not path or path == ".":
+        return tree
+    for i in path.split(os.sep):
+        if not _is_tree_and_contains(tree, i):
+            # there is no tree for specified path
+            return None
+        tree = tree[i]
+    return tree
+
+
 class GitTree(BaseTree):  # pylint:disable=abstract-method
     """Proxies the repo file access methods to Git objects"""
 
@@ -50,9 +74,10 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
         return cls(self, root)
 
     def open(
-        self, path, mode="r", encoding="utf-8"
+        self, path, mode="r", encoding=None
     ):  # pylint: disable=arguments-differ
         assert mode in {"r", "rb"}
+        encoding = encoding or "utf-8"
 
         relative_path = relpath(path, self.git.working_dir)
 
@@ -84,13 +109,15 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
             path
         ) and not self.dvcignore.is_ignored_dir(path)
 
-    def isdir(self, path):  # pylint: disable=arguments-differ
+    def isdir(
+        self, path, use_dvcignore=True
+    ):  # pylint: disable=arguments-differ
         obj = self._git_object_by_path(path)
         if obj is None:
             return False
         if obj.mode != GIT_MODE_DIR:
             return False
-        return not self.dvcignore.is_ignored_dir(path)
+        return not (use_dvcignore and self.dvcignore.is_ignored_dir(path))
 
     def isfile(self, path):  # pylint: disable=arguments-differ
         obj = self._git_object_by_path(path)
@@ -101,27 +128,12 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
             return False
         return not self.dvcignore.is_ignored_file(path)
 
-    @staticmethod
-    def _is_tree_and_contains(obj, path):
-        if obj.mode != GIT_MODE_DIR:
-            return False
-        # see https://github.com/gitpython-developers/GitPython/issues/851
-        # `return (i in tree)` doesn't work so here is a workaround:
-        for item in obj:
-            if _item_basename(item) == path:
-                return True
-        return False
-
-    def _git_object_by_path(self, path):
+    @cached_property
+    def _tree(self):
         import git
 
-        path = relpath(os.path.realpath(path), self.git.working_dir)
-        if path.split(os.sep, 1)[0] == "..":
-            # path points outside of git repository
-            return None
-
         try:
-            tree = self.git.tree(self.rev)
+            return self.git.tree(self.rev)
         except git.exc.BadName as exc:  # pylint: disable=no-member
             raise DvcException(
                 "revision '{}' not found in Git '{}'".format(
@@ -129,14 +141,13 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
                 )
             ) from exc
 
-        if not path or path == ".":
-            return tree
-        for i in path.split(os.sep):
-            if not self._is_tree_and_contains(tree, i):
-                # there is no tree for specified path
-                return None
-            tree = tree[i]
-        return tree
+    def _git_object_by_path(self, path):
+        path = relpath(os.path.realpath(path), self.git.working_dir)
+        if path.split(os.sep, 1)[0] == "..":
+            # path points outside of git repository
+            return None
+
+        return _get_obj(self._tree, path)
 
     def _walk(self, tree, topdown=True):
         dirs, nondirs = [], []
@@ -156,7 +167,14 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
         if not topdown:
             yield os.path.normpath(tree.abspath), dirs, nondirs
 
-    def walk(self, top, topdown=True, onerror=None, use_dvcignore=True):
+    def walk(
+        self,
+        top,
+        topdown=True,
+        onerror=None,
+        use_dvcignore=True,
+        ignore_subrepos=True,
+    ):
         """Directory tree generator.
 
         See `os.walk` for the docs. Differences:
@@ -176,7 +194,10 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
         for root, dirs, files in self._walk(tree, topdown=topdown):
             if use_dvcignore:
                 dirs[:], files[:] = self.dvcignore(
-                    os.path.abspath(root), dirs, files
+                    os.path.abspath(root),
+                    dirs,
+                    files,
+                    ignore_subrepos=ignore_subrepos,
                 )
             yield root, dirs, files
 
@@ -185,7 +206,7 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
             return False
 
         mode = self.stat(path).st_mode
-        return mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+        return is_exec(mode)
 
     def stat(self, path):
         import git
@@ -194,6 +215,9 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
             sec, nano_sec = git_time
             return sec + nano_sec / 1000000000
 
+        if self.dvcignore.is_ignored(path):
+            raise FileNotFoundError
+
         obj = self._git_object_by_path(path)
         if obj is None:
             raise OSError(errno.ENOENT, "No such file")
@@ -228,3 +252,6 @@ class GitTree(BaseTree):  # pylint:disable=abstract-method
             for file in files:
                 # NOTE: os.path.join is ~5.5 times slower
                 yield f"{root}{os.sep}{file}"
+
+    def _reset(self):
+        return self.__dict__.pop("dvcignore", None)

dvc/tree/gs.py

@@ -7,6 +7,7 @@ from functools import wraps
 from funcy import cached_property, wrap_prop
 
 from dvc.exceptions import DvcException
+from dvc.hash_info import HashInfo
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.scheme import Schemes
@@ -28,7 +29,6 @@ def dynamic_chunk_size(func):
         # Note: must be multiple of 256K.
         #
         # [#2572]: https://github.com/iterative/dvc/issues/2572
-
         # skipcq: PYL-W0212
         multiplier = 40
         while True:
@@ -190,11 +190,14 @@ class GSTree(BaseTree):
         path = path_info.path
         blob = self.gs.bucket(bucket).get_blob(path)
         if not blob:
-            return None
+            return HashInfo(self.PARAM_CHECKSUM, None)
 
         b64_md5 = blob.md5_hash
         md5 = base64.b64decode(b64_md5)
-        return codecs.getencoder("hex")(md5)[0].decode("utf-8")
+        return HashInfo(
+            self.PARAM_CHECKSUM,
+            codecs.getencoder("hex")(md5)[0].decode("utf-8"),
+        )
 
     def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
         bucket = self.gs.bucket(to_info.bucket)

dvc/tree/hdfs.py

@@ -7,6 +7,7 @@ from collections import deque
 from contextlib import closing, contextmanager
 from urllib.parse import urlparse
 
+from dvc.hash_info import HashInfo
 from dvc.scheme import Schemes
 from dvc.utils import fix_env, tmp_fname
 
@@ -130,6 +131,14 @@ class HDFSTree(BaseTree):
                     self.remove(tmp_info)
                     raise
 
+    def isfile(self, path_info):
+        with self.hdfs(path_info) as hdfs:
+            return hdfs.isfile(path_info.path)
+
+    def isdir(self, path_info):
+        with self.hdfs(path_info) as hdfs:
+            return hdfs.isdir(path_info.path)
+
     def hadoop_fs(self, cmd, user=None):
         cmd = "hadoop fs -" + cmd
         if user:
@@ -167,7 +176,9 @@ class HDFSTree(BaseTree):
         stdout = self.hadoop_fs(
             f"checksum {path_info.url}", user=path_info.user
         )
-        return self._group(regex, stdout, "checksum")
+        return HashInfo(
+            self.PARAM_CHECKSUM, self._group(regex, stdout, "checksum")
+        )
 
     def _upload(self, from_file, to_info, **_kwargs):
         with self.hdfs(to_info) as hdfs:

dvc/tree/http.py

@@ -6,6 +6,7 @@ from funcy import cached_property, memoize, wrap_prop, wrap_with
 
 import dvc.prompt as prompt
 from dvc.exceptions import DvcException, HTTPError
+from dvc.hash_info import HashInfo
 from dvc.path_info import HTTPURLInfo
 from dvc.progress import Tqdm
 from dvc.scheme import Schemes
@@ -52,6 +53,8 @@ class HTTPTree(BaseTree):  # pylint:disable=abstract-method
         self.password = config.get("password", None)
         self.ask_password = config.get("ask_password", False)
         self.headers = {}
+        self.ssl_verify = config.get("ssl_verify", True)
+        self.method = config.get("method", "POST")
 
     def _auth_method(self, path_info=None):
         from requests.auth import HTTPBasicAuth, HTTPDigestAuth
@@ -80,6 +83,8 @@ class HTTPTree(BaseTree):  # pylint:disable=abstract-method
 
         session = requests.Session()
 
+        session.verify = self.ssl_verify
+
         retries = Retry(
             total=self.SESSION_RETRIES,
             backoff_factor=self.SESSION_BACKOFF_FACTOR,
@@ -122,12 +127,27 @@ class HTTPTree(BaseTree):  # pylint:disable=abstract-method
         except requests.exceptions.RequestException:
             raise DvcException(f"could not perform a {method} request")
 
+    def _head(self, url):
+        response = self.request("HEAD", url)
+        if response.ok:
+            return response
+
+        # Sometimes servers are configured to forbid HEAD requests
+        # Context: https://github.com/iterative/dvc/issues/4131
+        with self.request("GET", url, stream=True) as r:
+            if r.ok:
+                return r
+
+        return response
+
     def exists(self, path_info, use_dvcignore=True):
-        return bool(self.request("HEAD", path_info.url))
+        return bool(self._head(path_info.url))
 
     def get_file_hash(self, path_info):
         url = path_info.url
-        headers = self.request("HEAD", url).headers
+
+        headers = self._head(url).headers
+
         etag = headers.get("ETag") or headers.get("Content-MD5")
 
         if not etag:
@@ -136,13 +156,7 @@ class HTTPTree(BaseTree):  # pylint:disable=abstract-method
                 "Content-MD5 header for '{url}'".format(url=url)
             )
 
-        if etag.startswith("W/"):
-            raise DvcException(
-                "Weak ETags are not supported."
-                " (Etag: '{etag}', URL: '{url}')".format(etag=etag, url=url)
-            )
-
-        return etag
+        return HashInfo(self.PARAM_CHECKSUM, etag)
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
         response = self.request("GET", from_info.url, stream=True)
@@ -181,7 +195,7 @@ class HTTPTree(BaseTree):  # pylint:disable=abstract-method
                             break
                         yield chunk
 
-        response = self.request("POST", to_info.url, data=chunks())
+        response = self.request(self.method, to_info.url, data=chunks())
         if response.status_code not in (200, 201):
             raise HTTPError(response.status_code, response.reason)
 

dvc/tree/local.py

@@ -7,10 +7,11 @@ from funcy import cached_property
 from shortuuid import uuid
 
 from dvc.exceptions import DvcException
+from dvc.hash_info import HashInfo
 from dvc.path_info import PathInfo
 from dvc.scheme import Schemes
 from dvc.system import System
-from dvc.utils import file_md5, relpath, tmp_fname
+from dvc.utils import file_md5, is_exec, relpath, tmp_fname
 from dvc.utils.fs import (
     copy_fobj_to_file,
     copyfile,
@@ -86,18 +87,26 @@ class LocalTree(BaseTree):
 
         return not self.dvcignore.is_ignored_file(path_info)
 
-    def isdir(self, path_info):
+    def isdir(
+        self, path_info, use_dvcignore=True
+    ):  # pylint: disable=arguments-differ
         if not os.path.isdir(path_info):
             return False
-
-        return not self.dvcignore.is_ignored_dir(path_info)
+        return not (use_dvcignore and self.dvcignore.is_ignored_dir(path_info))
 
     def iscopy(self, path_info):
         return not (
             System.is_symlink(path_info) or System.is_hardlink(path_info)
         )
 
-    def walk(self, top, topdown=True, onerror=None, use_dvcignore=True):
+    def walk(
+        self,
+        top,
+        topdown=True,
+        onerror=None,
+        use_dvcignore=True,
+        ignore_subrepos=True,
+    ):
         """Directory tree generator.
 
         See `os.walk` for the docs. Differences:
@@ -108,7 +117,10 @@ class LocalTree(BaseTree):
         ):
             if use_dvcignore:
                 dirs[:], files[:] = self.dvcignore(
-                    os.path.abspath(root), dirs, files
+                    os.path.abspath(root),
+                    dirs,
+                    files,
+                    ignore_subrepos=ignore_subrepos,
                 )
 
             yield os.path.normpath(root), dirs, files
@@ -146,10 +158,12 @@ class LocalTree(BaseTree):
 
     def isexec(self, path):
         mode = os.stat(path).st_mode
-        return mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+        return is_exec(mode)
+
+    def stat(self, path):
+        if self.dvcignore.is_ignored(path):
+            raise FileNotFoundError
 
-    @staticmethod
-    def stat(path):
         return os.stat(path)
 
     def move(self, from_info, to_info, mode=None):
@@ -298,7 +312,7 @@ class LocalTree(BaseTree):
         return stat.S_IMODE(mode) == self.CACHE_MODE
 
     def get_file_hash(self, path_info):
-        return file_md5(path_info)[0]
+        return HashInfo(self.PARAM_CHECKSUM, file_md5(path_info)[0])
 
     @staticmethod
     def getsize(path_info):
@@ -315,7 +329,7 @@ class LocalTree(BaseTree):
         )
 
         self.protect(tmp_file)
-        os.rename(tmp_file, to_info)
+        os.replace(tmp_file, to_info)
 
     @staticmethod
     def _download(
@@ -346,3 +360,6 @@ class LocalTree(BaseTree):
         info = self.hash_to_path_info(hash_)
         path_info = info.with_name(info.name + self.UNPACKED_DIR_SUFFIX)
         self.remove(path_info)
+
+    def _reset(self):
+        return self.__dict__.pop("dvcignore", None)

dvc/tree/repo.py

@@ -0,0 +1,451 @@
+import logging
+import os
+import stat
+import threading
+from contextlib import suppress
+from itertools import takewhile
+from typing import TYPE_CHECKING, Callable, Optional, Tuple, Union
+
+from funcy import lfilter, wrap_with
+from pygtrie import StringTrie
+
+from dvc.dvcfile import is_valid_filename
+from dvc.hash_info import HashInfo
+from dvc.path_info import PathInfo
+from dvc.utils import file_md5, is_exec
+from dvc.utils.fs import copy_fobj_to_file, makedirs
+
+from ._metadata import Metadata
+from .base import BaseTree
+from .dvc import DvcTree
+
+if TYPE_CHECKING:
+    from dvc.repo import Repo
+
+    from .git import GitTree
+    from .local import LocalTree
+
+
+logger = logging.getLogger(__name__)
+
+
+class RepoTree(BaseTree):  # pylint:disable=abstract-method
+    """DVC + git-tracked files tree.
+
+    Args:
+        repo: DVC or git repo.
+        subrepos: traverse to subrepos (by default, it ignores subrepos)
+        repo_factory: A function to initialize subrepo with, default is Repo.
+        kwargs: Additional keyword arguments passed to the `DvcTree()`.
+    """
+
+    scheme = "local"
+    PARAM_CHECKSUM = "md5"
+
+    def __init__(
+        self,
+        repo,
+        subrepos=False,
+        repo_factory: Callable[[str], "Repo"] = None,
+        **kwargs
+    ):
+        super().__init__(repo, {"url": repo.root_dir})
+
+        if not repo_factory:
+            from dvc.repo import Repo
+
+            self.repo_factory = Repo
+        else:
+            self.repo_factory = repo_factory
+
+        self._main_repo = repo
+        self.root_dir = repo.root_dir
+        self._traverse_subrepos = subrepos
+
+        self._subrepos_trie = StringTrie(separator=os.sep)
+        """Keeps track of each and every path with the corresponding repo."""
+
+        self._subrepos_trie[self.root_dir] = repo
+
+        self._dvctrees = {}
+        """Keep a dvctree instance of each repo."""
+
+        self._dvctree_configs = kwargs
+
+        if hasattr(repo, "dvc_dir"):
+            self._dvctrees[repo.root_dir] = DvcTree(repo, **kwargs)
+
+    def _get_repo(self, path) -> Optional["Repo"]:
+        """Returns repo that the path falls in, using prefix.
+
+        If the path is already tracked/collected, it just returns the repo.
+
+        Otherwise, it collects the repos that might be in the path's parents
+        and then returns the appropriate one.
+        """
+        repo = self._subrepos_trie.get(path)
+        if repo:
+            return repo
+
+        prefix, repo = self._subrepos_trie.longest_prefix(path)
+        if not prefix:
+            return None
+
+        parents = (parent.fspath for parent in PathInfo(path).parents)
+        dirs = [path] + list(takewhile(lambda p: p != prefix, parents))
+        dirs.reverse()
+        self._update(dirs, starting_repo=repo)
+        return self._subrepos_trie.get(path)
+
+    @wrap_with(threading.Lock())
+    def _update(self, dirs, starting_repo):
+        """Checks for subrepo in directories and updates them."""
+        repo = starting_repo
+        for d in dirs:
+            if self._is_dvc_repo(d):
+                repo = self.repo_factory(d)
+                self._dvctrees[repo.root_dir] = DvcTree(
+                    repo, **self._dvctree_configs
+                )
+            self._subrepos_trie[d] = repo
+
+    def _is_dvc_repo(self, dir_path):
+        """Check if the directory is a dvc repo."""
+        if not self._traverse_subrepos:
+            return False
+
+        from dvc.repo import Repo
+
+        repo_path = os.path.join(dir_path, Repo.DVC_DIR)
+        # dvcignore will ignore subrepos, therefore using `use_dvcignore=False`
+        return self._main_repo.tree.isdir(repo_path, use_dvcignore=False)
+
+    def _get_tree_pair(
+        self, path
+    ) -> Tuple[Union["GitTree", "LocalTree"], DvcTree]:
+        """
+        Returns a pair of trees based on repo the path falls in, using prefix.
+        """
+        path = os.path.abspath(path)
+
+        # fallback to the top-level repo if repo was not found
+        # this can happen if the path is outside of the repo
+        repo = self._get_repo(path) or self._main_repo
+
+        dvc_tree = self._dvctrees.get(repo.root_dir)
+        return repo.tree, dvc_tree
+
+    @property
+    def fetch(self):
+        return "fetch" in self._dvctree_configs
+
+    @property
+    def stream(self):
+        return "stream" in self._dvctree_configs
+
+    def open(
+        self, path, mode="r", encoding="utf-8", **kwargs
+    ):  # pylint: disable=arguments-differ
+        if "b" in mode:
+            encoding = None
+
+        tree, dvc_tree = self._get_tree_pair(path)
+        try:
+            return tree.open(path, mode=mode, encoding=encoding)
+        except FileNotFoundError:
+            if not dvc_tree:
+                raise
+
+        return dvc_tree.open(path, mode=mode, encoding=encoding, **kwargs)
+
+    def exists(
+        self, path, use_dvcignore=True
+    ):  # pylint: disable=arguments-differ
+        tree, dvc_tree = self._get_tree_pair(path)
+
+        if not dvc_tree:
+            return tree.exists(path)
+
+        if tree.exists(path):
+            return True
+
+        try:
+            meta = dvc_tree.metadata(path)
+        except FileNotFoundError:
+            return False
+
+        (out,) = meta.outs
+        assert len(meta.outs) == 1
+        if tree.exists(out.path_info):
+            return False
+        return True
+
+    def isdir(self, path):  # pylint: disable=arguments-differ
+        tree, dvc_tree = self._get_tree_pair(path)
+
+        try:
+            st = tree.stat(path)
+            return stat.S_ISDIR(st.st_mode)
+        except (OSError, ValueError):
+            # from CPython's os.path.isdir()
+            pass
+
+        if not dvc_tree:
+            return False
+
+        try:
+            meta = dvc_tree.metadata(path)
+        except FileNotFoundError:
+            return False
+
+        (out,) = meta.outs
+        assert len(meta.outs) == 1
+        if tree.exists(out.path_info):
+            return False
+        return meta.isdir
+
+    def isdvc(self, path, **kwargs):
+        _, dvc_tree = self._get_tree_pair(path)
+        return dvc_tree is not None and dvc_tree.isdvc(path, **kwargs)
+
+    def isfile(self, path):  # pylint: disable=arguments-differ
+        tree, dvc_tree = self._get_tree_pair(path)
+
+        try:
+            st = tree.stat(path)
+            return stat.S_ISREG(st.st_mode)
+        except (OSError, ValueError):
+            # from CPython's os.path.isfile()
+            pass
+
+        if not dvc_tree:
+            return False
+
+        try:
+            meta = dvc_tree.metadata(path)
+        except FileNotFoundError:
+            return False
+
+        (out,) = meta.outs
+        assert len(meta.outs) == 1
+        if tree.exists(out.path_info):
+            return False
+        return meta.isfile
+
+    def isexec(self, path):
+        tree, dvc_tree = self._get_tree_pair(path)
+        if dvc_tree and dvc_tree.exists(path):
+            return dvc_tree.isexec(path)
+        return tree.isexec(path)
+
+    def stat(self, path):
+        tree, _ = self._get_tree_pair(path)
+        return tree.stat(path)
+
+    def _dvc_walk(self, walk):
+        try:
+            root, dirs, files = next(walk)
+        except StopIteration:
+            return
+        yield root, dirs, files
+        for _ in dirs:
+            yield from self._dvc_walk(walk)
+
+    def _subrepo_walk(self, dir_path, **kwargs):
+        """Walk into a new repo.
+
+         NOTE: subrepo will only be discovered when walking if
+         ignore_subrepos is set to False.
+        """
+        tree, dvc_tree = self._get_tree_pair(dir_path)
+        tree_walk = tree.walk(
+            dir_path, topdown=True, ignore_subrepos=not self._traverse_subrepos
+        )
+        if dvc_tree:
+            dvc_walk = dvc_tree.walk(dir_path, topdown=True, **kwargs)
+        else:
+            dvc_walk = None
+        yield from self._walk(tree_walk, dvc_walk, **kwargs)
+
+    def _walk(self, repo_walk, dvc_walk=None, dvcfiles=False):
+        assert repo_walk
+        try:
+            _, dvc_dirs, dvc_fnames = (
+                next(dvc_walk) if dvc_walk else (None, [], [])
+            )
+            repo_root, repo_dirs, repo_fnames = next(repo_walk)
+        except StopIteration:
+            return
+
+        # separate subdirs into shared dirs, dvc-only dirs, repo-only dirs
+        dvc_set = set(dvc_dirs)
+        repo_set = set(repo_dirs)
+        dvc_only = list(dvc_set - repo_set)
+        repo_only = list(repo_set - dvc_set)
+        shared = list(dvc_set & repo_set)
+        dirs = shared + dvc_only + repo_only
+
+        # merge file lists
+        files = {
+            fname
+            for fname in dvc_fnames + repo_fnames
+            if dvcfiles or not is_valid_filename(fname)
+        }
+
+        yield repo_root, dirs, list(files)
+
+        def is_dvc_repo(d):
+            return self._is_dvc_repo(os.path.join(repo_root, d))
+
+        # remove subrepos to prevent it from being traversed
+        subrepos = set(filter(is_dvc_repo, repo_only))
+        # set dir order for next recursion level - shared dirs first so that
+        # next() for both generators recurses into the same shared directory
+        dvc_dirs[:] = [dirname for dirname in dirs if dirname in dvc_set]
+        repo_dirs[:] = lfilter(lambda d: d in (repo_set - subrepos), dirs)
+
+        for dirname in dirs:
+            if dirname in subrepos:
+                dir_path = os.path.join(repo_root, dirname)
+                yield from self._subrepo_walk(dir_path, dvcfiles=dvcfiles)
+            elif dirname in shared:
+                yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles)
+            elif dirname in dvc_set:
+                yield from self._dvc_walk(dvc_walk)
+            elif dirname in repo_set:
+                yield from self._walk(repo_walk, None, dvcfiles=dvcfiles)
+
+    def walk(
+        self, top, topdown=True, onerror=None, dvcfiles=False, **kwargs
+    ):  # pylint: disable=arguments-differ
+        """Walk and merge both DVC and repo trees.
+
+        Args:
+            top: path to walk from
+            topdown: if True, tree will be walked from top down.
+            onerror: if set, onerror function will be called if an error
+                occurs (by default errors are ignored).
+            dvcfiles: if True, dvcfiles will be included in the files list
+                for walked directories.
+
+        Any kwargs will be passed into methods used for fetching and/or
+        streaming DVC outs from remotes.
+        """
+        assert topdown
+
+        if not self.exists(top):
+            if onerror is not None:
+                onerror(FileNotFoundError(top))
+            return
+
+        if not self.isdir(top):
+            if onerror is not None:
+                onerror(NotADirectoryError(top))
+            return
+
+        tree, dvc_tree = self._get_tree_pair(top)
+        repo_exists = tree.exists(top)
+        repo_walk = tree.walk(
+            top,
+            topdown=topdown,
+            onerror=onerror,
+            ignore_subrepos=not self._traverse_subrepos,
+        )
+
+        if not dvc_tree or (repo_exists and dvc_tree.isdvc(top)):
+            yield from self._walk(repo_walk, None, dvcfiles=dvcfiles)
+            return
+
+        if not repo_exists:
+            yield from dvc_tree.walk(top, topdown=topdown, **kwargs)
+
+        dvc_walk = None
+        if dvc_tree.exists(top):
+            dvc_walk = dvc_tree.walk(top, topdown=topdown, **kwargs)
+
+        yield from self._walk(repo_walk, dvc_walk, dvcfiles=dvcfiles)
+
+    def walk_files(self, top, **kwargs):  # pylint: disable=arguments-differ
+        for root, _, files in self.walk(top, **kwargs):
+            for fname in files:
+                yield PathInfo(root) / fname
+
+    def get_dir_hash(self, path_info, **kwargs):
+        tree, dvc_tree = self._get_tree_pair(path_info)
+        if tree.exists(path_info):
+            return super().get_dir_hash(path_info, **kwargs)
+        if not dvc_tree:
+            raise FileNotFoundError
+        return dvc_tree.get_dir_hash(path_info, **kwargs)
+
+    def get_file_hash(self, path_info):
+        """Return file checksum for specified path.
+
+        If path_info is a DVC out, the pre-computed checksum for the file
+        will be used. If path_info is a git file, MD5 will be computed for
+        the git object.
+        """
+        if not self.exists(path_info):
+            raise FileNotFoundError
+        _, dvc_tree = self._get_tree_pair(path_info)
+        if dvc_tree and dvc_tree.exists(path_info):
+            try:
+                return dvc_tree.get_file_hash(path_info)
+            except FileNotFoundError:
+                pass
+        return HashInfo(self.PARAM_CHECKSUM, file_md5(path_info, self)[0])
+
+    def copytree(self, top, dest):
+        top = PathInfo(top)
+        dest = PathInfo(dest)
+
+        if not self.exists(top):
+            raise FileNotFoundError
+
+        if self.isfile(top):
+            makedirs(dest.parent, exist_ok=True)
+            with self.open(top, mode="rb") as fobj:
+                copy_fobj_to_file(fobj, dest)
+            return
+
+        for root, _, files in self.walk(top):
+            root = PathInfo(root)
+            dest_dir = dest / root.relative_to(top)
+            makedirs(dest_dir, exist_ok=True)
+            for fname in files:
+                src = root / fname
+                with self.open(src, mode="rb") as fobj:
+                    copy_fobj_to_file(fobj, dest_dir / fname)
+
+    @property
+    def hash_jobs(self):  # pylint: disable=invalid-overridden-method
+        return self._main_repo.tree.hash_jobs
+
+    def metadata(self, path):
+        abspath = os.path.abspath(path)
+        path_info = PathInfo(abspath)
+        tree, dvc_tree = self._get_tree_pair(path_info)
+
+        dvc_meta = None
+        if dvc_tree:
+            with suppress(FileNotFoundError):
+                dvc_meta = dvc_tree.metadata(path_info)
+
+        stat_result = None
+        with suppress(FileNotFoundError):
+            stat_result = tree.stat(path_info)
+
+        if not stat_result and not dvc_meta:
+            raise FileNotFoundError
+
+        meta = dvc_meta or Metadata(
+            path_info=path_info,
+            repo=self._get_repo(abspath) or self._main_repo,
+        )
+
+        isdir = bool(stat_result) and stat.S_ISDIR(stat_result.st_mode)
+        meta.isdir = meta.isdir or isdir
+
+        if not dvc_meta:
+            meta.is_exec = bool(stat_result) and is_exec(stat_result.st_mode)
+        return meta

dvc/tree/s3.py

@@ -1,14 +1,17 @@
 import logging
 import os
 import threading
+from contextlib import contextmanager
 
 from funcy import cached_property, wrap_prop
 
 from dvc.config import ConfigError
 from dvc.exceptions import DvcException, ETagMismatchError
+from dvc.hash_info import HashInfo
 from dvc.path_info import CloudURLInfo
 from dvc.progress import Tqdm
 from dvc.scheme import Schemes
+from dvc.utils import error_link
 
 from .base import BaseTree
 
@@ -31,11 +34,6 @@ class S3Tree(BaseTree):
         self.profile = config.get("profile")
         self.endpoint_url = config.get("endpointurl")
 
-        if config.get("listobjects"):
-            self.list_objects_api = "list_objects"
-        else:
-            self.list_objects_api = "list_objects_v2"
-
         self.use_ssl = config.get("use_ssl", True)
 
         self.extra_args = {}
@@ -75,24 +73,49 @@ class S3Tree(BaseTree):
 
         session = boto3.session.Session(**session_opts)
 
-        return session.client(
+        return session.resource(
             "s3", endpoint_url=self.endpoint_url, use_ssl=self.use_ssl
         )
 
-    @classmethod
-    def get_etag(cls, s3, bucket, path):
-        obj = cls.get_head_object(s3, bucket, path)
-
-        return obj["ETag"].strip('"')
-
-    @staticmethod
-    def get_head_object(s3, bucket, path, *args, **kwargs):
+    @contextmanager
+    def _get_s3(self):
+        from botocore.exceptions import (
+            EndpointConnectionError,
+            NoCredentialsError,
+        )
 
         try:
-            obj = s3.head_object(Bucket=bucket, Key=path, *args, **kwargs)
-        except Exception as exc:
-            raise DvcException(f"s3://{bucket}/{path} does not exist") from exc
-        return obj
+            yield self.s3
+        except NoCredentialsError as exc:
+            link = error_link("no-credentials")
+            raise DvcException(
+                f"Unable to find AWS credentials. {link}"
+            ) from exc
+        except EndpointConnectionError as exc:
+            link = error_link("connection-error")
+            name = self.endpoint_url or "AWS S3"
+            raise DvcException(
+                f"Unable to connect to '{name}'. {link}"
+            ) from exc
+
+    @contextmanager
+    def _get_bucket(self, bucket):
+        with self._get_s3() as s3:
+            try:
+                yield s3.Bucket(bucket)
+            except s3.meta.client.exceptions.NoSuchBucket as exc:
+                link = error_link("no-bucket")
+                raise DvcException(
+                    f"Bucket '{bucket}' does not exist. {link}"
+                ) from exc
+
+    @contextmanager
+    def _get_obj(self, path_info):
+        with self._get_bucket(path_info.bucket) as bucket:
+            try:
+                yield bucket.Object(path_info.path)
+            except bucket.meta.client.exceptions.NoSuchKey as exc:
+                raise DvcException(f"{path_info.url} does not exist") from exc
 
     def _append_aws_grants_to_extra_args(self, config):
         # Keys for extra_args can be one of the following list:
@@ -127,9 +150,12 @@ class S3Tree(BaseTree):
 
     def _generate_download_url(self, path_info, expires=3600):
         params = {"Bucket": path_info.bucket, "Key": path_info.path}
-        return self.s3.generate_presigned_url(
-            ClientMethod="get_object", Params=params, ExpiresIn=int(expires)
-        )
+        with self._get_s3() as s3:
+            return s3.meta.client.generate_presigned_url(
+                ClientMethod="get_object",
+                Params=params,
+                ExpiresIn=int(expires),
+            )
 
     def exists(self, path_info, use_dvcignore=True):
         """Check if the blob exists. If it does not exist,
@@ -164,36 +190,19 @@ class S3Tree(BaseTree):
         return bool(list(self._list_paths(dir_path, max_items=1)))
 
     def isfile(self, path_info):
-        from botocore.exceptions import ClientError
-
         if path_info.path.endswith("/"):
             return False
 
-        try:
-            self.s3.head_object(Bucket=path_info.bucket, Key=path_info.path)
-        except ClientError as exc:
-            if exc.response["Error"]["Code"] != "404":
-                raise
-            return False
-
-        return True
-
-    def _list_objects(self, path_info, max_items=None):
-        """ Read config for list object api, paginate through list objects."""
-        kwargs = {
-            "Bucket": path_info.bucket,
-            "Prefix": path_info.path,
-            "PaginationConfig": {"MaxItems": max_items},
-        }
-        paginator = self.s3.get_paginator(self.list_objects_api)
-        for page in paginator.paginate(**kwargs):
-            contents = page.get("Contents", ())
-            yield from contents
+        return path_info.path in self._list_paths(path_info)
 
     def _list_paths(self, path_info, max_items=None):
-        return (
-            item["Key"] for item in self._list_objects(path_info, max_items)
-        )
+        kwargs = {"Prefix": path_info.path}
+        if max_items is not None:
+            kwargs["MaxKeys"] = max_items
+
+        with self._get_bucket(path_info.bucket) as bucket:
+            for obj_summary in bucket.objects.filter(**kwargs):
+                yield obj_summary.key
 
     def walk_files(self, path_info, **kwargs):
         if not kwargs.pop("prefix", False):
@@ -209,7 +218,8 @@ class S3Tree(BaseTree):
             raise NotImplementedError
 
         logger.debug(f"Removing {path_info}")
-        self.s3.delete_object(Bucket=path_info.bucket, Key=path_info.path)
+        with self._get_obj(path_info) as obj:
+            obj.delete()
 
     def makedirs(self, path_info):
         # We need to support creating empty directories, which means
@@ -221,10 +231,12 @@ class S3Tree(BaseTree):
             return
 
         dir_path = path_info / ""
-        self.s3.put_object(Bucket=path_info.bucket, Key=dir_path.path, Body="")
+        with self._get_obj(dir_path) as obj:
+            obj.put(Body="")
 
     def copy(self, from_info, to_info):
-        self._copy(self.s3, from_info, to_info, self.extra_args)
+        with self._get_s3() as s3:
+            self._copy(s3.meta.client, from_info, to_info, self.extra_args)
 
     @classmethod
     def _copy_multipart(
@@ -238,8 +250,8 @@ class S3Tree(BaseTree):
         parts = []
         byte_position = 0
         for i in range(1, n_parts + 1):
-            obj = S3Tree.get_head_object(
-                s3, from_info.bucket, from_info.path, PartNumber=i
+            obj = s3.head_object(
+                Bucket=from_info.bucket, Key=from_info.path, PartNumber=i
             )
             part_size = obj["ContentLength"]
             lastbyte = byte_position + part_size - 1
@@ -293,7 +305,7 @@ class S3Tree(BaseTree):
         # object is transfered in the same chunks as it was originally.
         from boto3.s3.transfer import TransferConfig
 
-        obj = S3Tree.get_head_object(s3, from_info.bucket, from_info.path)
+        obj = s3.head_object(Bucket=from_info.bucket, Key=from_info.path)
         etag = obj["ETag"].strip('"')
         size = obj["ContentLength"]
 
@@ -313,36 +325,32 @@ class S3Tree(BaseTree):
                 Config=TransferConfig(multipart_threshold=size + 1),
             )
 
-        cached_etag = S3Tree.get_etag(s3, to_info.bucket, to_info.path)
+        cached_etag = s3.head_object(Bucket=to_info.bucket, Key=to_info.path)[
+            "ETag"
+        ].strip('"')
         if etag != cached_etag:
             raise ETagMismatchError(etag, cached_etag)
 
     def get_file_hash(self, path_info):
-        return self.get_etag(self.s3, path_info.bucket, path_info.path)
+        with self._get_obj(path_info) as obj:
+            return HashInfo(self.PARAM_CHECKSUM, obj.e_tag.strip('"'))
 
     def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
-        total = os.path.getsize(from_file)
-        with Tqdm(
-            disable=no_progress_bar, total=total, bytes=True, desc=name
-        ) as pbar:
-            self.s3.upload_file(
-                from_file,
-                to_info.bucket,
-                to_info.path,
-                Callback=pbar.update,
-                ExtraArgs=self.extra_args,
-            )
+        with self._get_obj(to_info) as obj:
+            total = os.path.getsize(from_file)
+            with Tqdm(
+                disable=no_progress_bar, total=total, bytes=True, desc=name
+            ) as pbar:
+                obj.upload_file(
+                    from_file, Callback=pbar.update, ExtraArgs=self.extra_args,
+                )
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        if no_progress_bar:
-            total = None
-        else:
-            total = self.s3.head_object(
-                Bucket=from_info.bucket, Key=from_info.path
-            )["ContentLength"]
-        with Tqdm(
-            disable=no_progress_bar, total=total, bytes=True, desc=name
-        ) as pbar:
-            self.s3.download_file(
-                from_info.bucket, from_info.path, to_file, Callback=pbar.update
-            )
+        with self._get_obj(from_info) as obj:
+            with Tqdm(
+                disable=no_progress_bar,
+                total=obj.content_length,
+                bytes=True,
+                desc=name,
+            ) as pbar:
+                obj.download_file(to_file, Callback=pbar.update)

dvc/tree/ssh/__init__.py

@@ -10,6 +10,7 @@ from urllib.parse import urlparse
 from funcy import first, memoize, silent, wrap_with
 
 import dvc.prompt as prompt
+from dvc.hash_info import HashInfo
 from dvc.scheme import Schemes
 
 from ..base import BaseTree
@@ -238,7 +239,7 @@ class SSHTree(BaseTree):
             raise NotImplementedError
 
         with self.ssh(path_info) as ssh:
-            return ssh.md5(path_info.path)
+            return HashInfo(self.PARAM_CHECKSUM, ssh.md5(path_info.path))
 
     def getsize(self, path_info):
         with self.ssh(path_info) as ssh: