dvc: bump to 2.1.0

I found that it's harder to style a header with the list using index,
they are to be applied by header name. Otherwise, we might end up using
a different header for coloring, or worse throw some error because the
header name and style don't match.
Row does have it's property of index, so it makes sense there.

.coveragerc

@@ -13,3 +13,4 @@ exclude_lines =
     raise NotImplementedError
     # pragma: no cover
     #  pragma: no cover
+    \.\.\.$

.github/ISSUE_TEMPLATE/bug_report.md

@@ -48,14 +48,17 @@ A clear and concise description of what you expect to happen.
 This is required to ensure that we can reproduce the bug.
 -->
 
-**Output of `dvc version`:**
+**Output of `dvc doctor`:**
 
 ```console
-$ dvc version
+$ dvc doctor
 ```
 
 **Additional Information (if any):**
 
 <!--
+Please check https://github.com/iterative/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
+
 If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
--->
+If the issue is regarding the performance, please attach the profiling information and the benchmark comparisons.
+-->

.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+
+updates:
+  - directory: "/"
+    package-ecosystem: "pip"
+    schedule:
+      interval: "daily"
+    labels:
+      - "maintenance"
+
+  - directory: "/"
+    package-ecosystem: "github-actions"
+    schedule:
+      interval: "daily"
+    labels:
+      - "maintenance"

.github/workflows/packages.yaml

@@ -6,91 +6,130 @@ on:
   release:
     types: [released, prereleased]
 jobs:
-  linux:
+  deb:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+    - uses: actions/[email protected].4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2.2.2
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Set up Ruby 2.4
       uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.4
-    - name: Build
-      run: ./scripts/build_posix.sh
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build deb
+      run: python scripts/build.py deb
     - name: Publish deb
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_path: scripts/fpm/dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_content_type: binary/octet-stream
-    - name: Publish rpm
-      if: github.event_name == 'release'
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
-        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
-        asset_content_type: binary/octet-stream
-  mac:
-    runs-on: macos-10.15
+  rpm:
+    runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+    - uses: actions/[email protected].4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2.2.2
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Set up Ruby 2.4
       uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.4
-    - name: Build
-      run: ./scripts/build_posix.sh
-    - name: Publish
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build rpm
+      run: python scripts/build.py rpm
+    - name: Publish rpm
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_content_type: binary/octet-stream
+  osxpkg:
+    runs-on: macos-10.15
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Install deps
+      run: |
+        # available lxml (webdav dependency) wheels are built with an old
+        # SDK, which breaks osxpkg notarization. Building from scratch
+        # instead.
+        pip install .[all] --no-binary lxml
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build
+      run: python scripts/build.py osxpkg
+    - name: Publish
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}.pkg
         asset_name: dvc-${{ github.event.release.tag_name }}.pkg
         asset_content_type: binary/octet-stream
-  windows:
+  exe:
     runs-on: windows-2019
     steps:
-    - uses: actions/[email protected].2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+    - uses: actions/[email protected].4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2.2.2
       with:
-        python-version: 3.7
+        python-version: 3.8
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+    - name: Pull images
+      run: |
+        dvc pull
     - name: Build
-      run: scripts\build_windows.cmd
+      run: python scripts\build.py exe
       shell: cmd
     - name: Publish
       if: github.event_name == 'release'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}.exe
+        asset_path: scripts/innosetup/dvc-${{ github.event.release.tag_name }}.exe
         asset_name: dvc-${{ github.event.release.tag_name }}.exe
         asset_content_type: binary/octet-stream
-  python:
+  pip:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.7
     - name: Install
@@ -103,29 +142,3 @@ jobs:
       with:
         user: __token__
         password: ${{ secrets.PYPI_TOKEN }}
-  snap:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/[email protected]
-    - name: Build the package
-      uses: snapcore/action-build@v1
-      id: snapcraft
-    - name: Test the package
-      run: |
-        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
-        dvc version
-        sudo snap remove dvc
-    - name: Publish to the edge channel
-      uses: snapcore/action-publish@v1
-      if: github.event_name == 'push'
-      with:
-        store_login: ${{ secrets.STORE_LOGIN }}
-        snap: ${{ steps.snapcraft.outputs.snap }}
-        release: edge,v2/edge
-    - name: Publish to the stable channel
-      uses: snapcore/action-publish@v1
-      if: ${{ github.event_name == 'release' && github.event.release.released }}
-      with:
-        store_login: ${{ secrets.STORE_LOGIN }}
-        snap: ${{ steps.snapcraft.outputs.snap }}
-        release: stable,v2/stable

.github/workflows/tests.yaml

@@ -12,29 +12,27 @@ env:
   DVC_TEST: "true"
   HOMEBREW_NO_AUTO_UPDATE: 1
   SHELL: /bin/bash
-  GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.8
     - name: Install requirements
-      run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
+      run: |
+        pip install wheel
+        pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
     - name: install temporary dependencies
       run: |
         pip install git+https://github.com/isidentical/gdrivefs@service-account
         pip install git+https://github.com/isidentical/pydata-google-auth@service-account
     - name: Check README
       run: python setup.py checkdocs
-    - uses: pre-commit/[email protected].0
+    - uses: pre-commit/[email protected].3
   tests:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -51,9 +49,9 @@ jobs:
             pyv: "3.9"
 
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: ${{ matrix.pyv }}
     - name: get pip cache dir
@@ -62,7 +60,7 @@ jobs:
         echo "::set-output name=dir::$(pip cache dir)"
     - name: set pip cache
       id: pip-cache
-      uses: actions/cache@v2
+      uses: actions/cache@v2.1.5
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
@@ -83,25 +81,20 @@ jobs:
         git config --global user.name "DVC Tester"
     - name: setup gs creds
       shell: bash
+      env:
+        GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
       if: env.GS_CREDS_JSON != ''
       run: |
         mkdir -p scripts/ci
         echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
     - name: run tests
-      run: python -m tests --all --cov-report=xml --cov-report=term --tap-combined -n=4
+      env:
+        GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      run: python -m tests --all --cov-report=xml --cov-report=term -n=4
     - name: upload coverage report
-      uses: codecov/codecov-action@v1.0.13
+      uses: codecov/codecov-action@v1.4.1
       with:
         file: ./coverage.xml
         fail_ci_if_error: false
-    - name: Post TAP
-      # Running custom fork with an option, that does not fail CI on error
-      uses: skshetry/flaky-service/packages/action@master
-      if: always() && github.ref == 'refs/heads/master'
-      with:
-        file-path: ${{github.workspace}}/testresults.tap
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        os: ${{runner.os}}
-        matrix: ${{toJson(matrix)}}
-        repo-description: ${{github.event.repository.description}}
-        fail_ci_if_error: false

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 5.6.4
+    rev: 5.8.0
   - hooks:
       - id: flake8
         language_version: python3
@@ -18,13 +18,13 @@ repos:
           - flake8-debugger
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 3.9.0
   - hooks:
       -  id: mypy
          language_version: python3
          files: ^dvc/
     repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.790'
+    rev: 'v0.812'
   - repo: local
     hooks:
     - id: pylint

.pylintrc

@@ -15,10 +15,13 @@ ignore-patterns=
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-init-hook="import sys; \
-           from pathlib import Path; \
-           from pylint.config import find_pylintrc; \
-           sys.path.append(Path(find_pylintrc()).parent / 'tests');"
+init-hook="import sys;
+           from pathlib import Path;
+           from pylint.config import find_pylintrc;
+
+           tests_folder = Path(find_pylintrc()).parent / 'tests'
+           sys.path.append(str(tests_folder));
+           "
 
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
@@ -178,7 +181,10 @@ disable=format,
         # we use it to speedup/optimize
         import-outside-toplevel,
         fixme,
-
+        # we use cause of the exception for error message output
+        # so some of the times we don't use `raise from` to not show the cause
+        # of the error
+        raise-missing-from,
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

MANIFEST.in

@@ -1,2 +1,3 @@
 include fastentrypoints.py
 include LICENSE
+include test_requirements.txt

README.rst

@@ -129,7 +129,11 @@ Conda (Anaconda)
 
 .. code-block:: bash
 
-   conda install -c conda-forge dvc
+   conda install -c conda-forge mamba # installs much faster than conda
+   mamba install -c conda-forge dvc
+
+Depending on the remote storage type you plan to use to keep and share your data, you might need to
+install optional dependencies: `dvc-s3`, `dvc-azure`, `dvc-gdrive`, `dvc-gs`, `dvc-oss`, `dvc-ssh`.
 
 pip (PyPI)
 ----------
@@ -222,6 +226,9 @@ Citation
 Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
 `DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
 
+Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and Source Code - Empirical Study of DVC Projects <https://mcis.cs.queensu.ca/publications/2021/saner.pdf>`_ , in Proceedings of the 28th IEEE International Conference on Software Analysis, Evolution, and Reengineering, SANER 2021. Hawaii, USA.
+
+
 .. |Banner| image:: https://dvc.org/img/logo-github-readme.png
    :target: https://dvc.org
    :alt: DVC logo

debian/changelog

@@ -1,5 +0,0 @@
-dvc (0.75.b1) experimental; urgency=low
-
-  * First native Debian package version.
-
- -- Dmitry Petrov <[email protected]>  Sat, 07 Dec 2019 09:49:28 +0300

debian/compat

@@ -1 +0,0 @@
-10

debian/control

@@ -1,33 +0,0 @@
-Source: dvc
-Maintainer: Dmitry Petrov <[email protected]>
-Section: python
-Priority: optional
-Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)
-Standards-Version: 4.0.0
-Homepage: http://dvc.org
-Vcs-Git: https://github.com/iterative/dvc.git
-Vcs-browser: https://github.com/iterative/dvc
-
-Package: dvc
-Architecture: all
-Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2)
-Recommends: python3-arrow (=0.14.0),
-            python3-paramiko (>=2.5.0),
-            python3-gssapi (>= 2.5.0),
-            python3-boto3 (>=1.9.86),
-            python3-pydrive (=1.3.1),
-            python3-azure-storage-blob (=12.1.0),
-            python3-oss2 (=2.6.1),
-            python3-google-cloud-storage (=1.19.0)
-Description: Git for data scientists - manage your code and data together
- Data Version Control or DVC is an open-source tool for data science and machine learning projects.
- .
- Key features:
- .
- 1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services.
- .
- 2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
- .
- 3. Makes projects reproducible and shareable; helping to answer questions about how a model was built.
- .
- 4. Helps manage experiments with Git tags/branches and metrics tracking.

debian/copyright

@@ -1,11 +0,0 @@
-Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: DVC
-Upstream-Contact: Dmitry Petrov <[email protected]>
-Source: https://github.com/iterative/dvc
-
-Files: *
-Copyright: Dmitry Petrov <[email protected]>
-License: Apache 2.0
- On Debian systems, the full text of the Apache 2.0
- License can be found in the file
- `/usr/share/common-licenses/Apache-2.0`.

debian/rules

@@ -1,6 +0,0 @@
-#!/usr/bin/make -f
-
-export PYBUILD_NAME=dvc
-%:
-	dh $@ --with python3 --buildsystem=pybuild
-

debian/source/format

@@ -1 +0,0 @@
-3.0 (native)

debian/watch

@@ -1,4 +0,0 @@
-# please also check http://pypi.debian.net/dvc/watch
-version=3
-opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
-http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))

dvc/api.py

@@ -4,7 +4,6 @@ from contextlib import _GeneratorContextManager as GCM
 from funcy import reraise
 
 from dvc.exceptions import OutputNotFoundError, PathMissingError
-from dvc.objects.stage import get_hash
 from dvc.path_info import PathInfo
 from dvc.repo import Repo
 
@@ -29,8 +28,8 @@ def get_url(path, repo=None, rev=None, remote=None):
             raise OutputNotFoundError(path, repo)
 
         cloud = metadata.repo.cloud
-        hash_info = get_hash(path_info, _repo.repo_fs, "md5", _repo.odb)
-        return cloud.get_url_for(remote, checksum=hash_info.value)
+        md5 = metadata.repo.dvcfs.info(path_info)["md5"]
+        return cloud.get_url_for(remote, checksum=md5)
 
 
 def open(  # noqa, pylint: disable=redefined-builtin

dvc/checkout.py

@@ -9,8 +9,9 @@ from dvc.exceptions import (
     ConfirmRemoveError,
     DvcException,
 )
-from dvc.objects import ObjectFormatError, check, load
-from dvc.objects.stage import get_hash
+from dvc.objects import check, load
+from dvc.objects.errors import ObjectFormatError
+from dvc.objects.stage import stage
 from dvc.remote.slow_link_detection import (  # type: ignore[attr-defined]
     slow_link_guard,
 )
@@ -30,7 +31,7 @@ def _changed(path_info, fs, obj, cache):
         return True
 
     try:
-        actual = get_hash(path_info, fs, obj.hash_info.name, cache)
+        actual = stage(cache, path_info, fs, obj.hash_info.name).hash_info
     except FileNotFoundError:
         logger.debug("'%s' doesn't exist.", path_info)
         return True
@@ -56,7 +57,7 @@ def _remove(path_info, fs, cache, force=False):
         fs.remove(path_info)
         return
 
-    current = get_hash(path_info, fs, fs.PARAM_CHECKSUM, cache)
+    current = stage(cache, path_info, fs, fs.PARAM_CHECKSUM).hash_info
     try:
         obj = load(cache, current)
         check(cache, obj)
@@ -174,10 +175,10 @@ def _checkout_file(
     return modified
 
 
-def _remove_redundant_files(path_info, fs, dir_info, cache, force):
+def _remove_redundant_files(path_info, fs, obj, cache, force):
     existing_files = set(fs.walk_files(path_info))
 
-    needed_files = {path_info.joinpath(*key) for key, _ in dir_info.items()}
+    needed_files = {path_info.joinpath(*key) for key, _ in obj}
     redundant_files = existing_files - needed_files
     for path in redundant_files:
         _remove(path, fs, cache, force)
@@ -197,11 +198,11 @@ def _checkout_dir(
 
     logger.debug("Linking directory '%s'.", path_info)
 
-    for entry_key, entry_hash_info in obj.hash_info.dir_info.items():
+    for entry_key, entry_obj in obj:
         entry_modified = _checkout_file(
             path_info.joinpath(*entry_key),
             fs,
-            cache.get(entry_hash_info),
+            entry_obj,
             cache,
             force,
             progress_callback,
@@ -211,10 +212,7 @@ def _checkout_dir(
             modified = True
 
     modified = (
-        _remove_redundant_files(
-            path_info, fs, obj.hash_info.dir_info, cache, force
-        )
-        or modified
+        _remove_redundant_files(path_info, fs, obj, cache, force) or modified
     )
 
     fs.repo.state.save(path_info, fs, obj.hash_info)

dvc/command/data_sync.py

@@ -156,6 +156,7 @@ def add_parser(subparsers, _parent_parser):
         help="Fetch cache for all tags.",
     )
     pull_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -224,6 +225,7 @@ def add_parser(subparsers, _parent_parser):
         help="Push cache for all tags.",
     )
     push_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -290,6 +292,7 @@ def add_parser(subparsers, _parent_parser):
         help="Fetch cache for all tags.",
     )
     fetch_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -370,6 +373,7 @@ def add_parser(subparsers, _parent_parser):
         "for all tags.",
     )
     status_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,

dvc/command/diff.py

@@ -18,9 +18,9 @@ def _digest(checksum):
 
 
 def _show_md(diff, show_hash=False, hide_missing=False):
-    from dvc.utils.diff import table
+    from dvc.ui import ui
 
-    header = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
+    headers = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
     rows = []
     statuses = ["added", "deleted", "renamed", "modified"]
     if not hide_missing:
@@ -39,7 +39,7 @@ def _show_md(diff, show_hash=False, hide_missing=False):
             else:
                 rows.append([status, path])
 
-    return table(header, rows, True)
+    ui.table(rows, headers=headers, markdown=True)
 
 
 class CmdDiff(CmdBase):

dvc/command/experiments.py

@@ -1,10 +1,11 @@
 import argparse
 import logging
 from collections import Counter, OrderedDict, defaultdict
-from collections.abc import Mapping
 from datetime import date, datetime
-from itertools import groupby
-from typing import Dict, Iterable, Optional
+from fnmatch import fnmatch
+from typing import TYPE_CHECKING, Dict, Iterable, Optional
+
+from funcy import constantly, iffy, lmap
 
 import dvc.prompt as prompt
 from dvc.command import completion
@@ -15,10 +16,15 @@ from dvc.command.repro import add_arguments as add_repro_arguments
 from dvc.exceptions import DvcException, InvalidArgumentError
 from dvc.utils.flatten import flatten
 
+if TYPE_CHECKING:
+    from dvc.compare import TabularData
+
+
 logger = logging.getLogger(__name__)
 
 
 SHOW_MAX_WIDTH = 1024
+FILL_VALUE = "-"
 
 
 def _filter_name(names, label, filter_strs):
@@ -27,32 +33,23 @@ def _filter_name(names, label, filter_strs):
 
     for filter_s in filter_strs:
         path, _, name = filter_s.rpartition(":")
-        path_filters[path].append(tuple(name.split(".")))
+        path_filters[path].append(name)
 
     for path, filters in path_filters.items():
         if path:
             match_paths = [path]
         else:
             match_paths = names.keys()
-        for length, groups in groupby(filters, len):
-            for group in groups:
-                for match_path in match_paths:
-                    possible_names = [
-                        tuple(name.split(".")) for name in names[match_path]
-                    ]
-                    matches = [
-                        name
-                        for name in possible_names
-                        if name[:length] == group
-                    ]
-                    if not matches:
-                        name = ".".join(group)
-                        raise InvalidArgumentError(
-                            f"'{name}' does not match any known {label}"
-                        )
-                    ret[match_path].update(
-                        {".".join(match): None for match in matches}
+        for match_path in match_paths:
+            for f in filters:
+                matches = [
+                    name for name in names[match_path] if fnmatch(name, f)
+                ]
+                if not matches:
+                    raise InvalidArgumentError(
+                        f"'{f}' does not match any known {label}"
                     )
+                ret[match_path].update({match: None for match in matches})
 
     return ret
 
@@ -103,7 +100,6 @@ def _collect_names(all_experiments, **kwargs):
         for exp in experiments.values():
             _update_names(metric_names, exp.get("metrics", {}).items())
             _update_names(param_names, exp.get("params", {}).items())
-
     metric_names = _filter_names(
         metric_names,
         "metrics",
@@ -111,7 +107,7 @@ def _collect_names(all_experiments, **kwargs):
         kwargs.get("exclude_metrics"),
     )
     param_names = _filter_names(
-        (param_names),
+        param_names,
         "params",
         kwargs.get("include_params"),
         kwargs.get("exclude_params"),
@@ -126,7 +122,6 @@ def _collect_rows(
     metric_names,
     param_names,
     precision=DEFAULT_PRECISION,
-    no_timestamp=False,
     sort_by=None,
     sort_order=None,
 ):
@@ -143,56 +138,59 @@ def _collect_rows(
 
     new_checkpoint = True
     for i, (rev, exp) in enumerate(experiments.items()):
-        row = []
-        style = None
-        queued = "*" if exp.get("queued", False) else ""
+        queued = str(exp.get("queued") or "")
+        is_baseline = rev == "baseline"
+
+        if is_baseline:
+            name_rev = base_rev[:7] if Git.is_sha(base_rev) else base_rev
+        else:
+            name_rev = rev[:7]
+
+        exp_name = exp.get("name", "")
+        if is_baseline:
+            name = exp_name or name_rev
+        elif exp_name:
+            name = f"{rev[:7]} [[bold]{exp['name']}[/]]"
+        else:
+            name = name_rev
 
         tip = exp.get("checkpoint_tip")
+
+        parent_rev = exp.get("checkpoint_parent", "")
+        parent_exp = experiments.get(parent_rev, {})
+        parent_tip = parent_exp.get("checkpoint_tip")
+
         parent = ""
-        if rev == "baseline":
-            if Git.is_sha(base_rev):
-                name_rev = base_rev[:7]
+        if is_baseline:
+            tree = ""
+        elif tip:
+            if tip == parent_tip:
+                tree = "│ ╓" if new_checkpoint else "│ ╟"
+            elif parent_rev == base_rev:
+                tree = "├─╨"
             else:
-                name_rev = base_rev
-            name = exp.get("name", name_rev)
-            row.append(f"{name}")
-            style = "bold"
+                tree = "│ ╟"
+                parent = parent_rev[:7]
         else:
-            if tip:
-                parent_rev = exp.get("checkpoint_parent", "")
-                parent_exp = experiments.get(parent_rev, {})
-                parent_tip = parent_exp.get("checkpoint_tip")
-                if tip == parent_tip:
-                    if new_checkpoint:
-                        tree = "│ ╓"
-                    else:
-                        tree = "│ ╟"
-                    new_checkpoint = False
-                else:
-                    if parent_rev == base_rev:
-                        tree = "├─╨"
-                    else:
-                        tree = "│ ╟"
-                        parent = f" ({parent_rev[:7]})"
-                    new_checkpoint = True
-            else:
-                if i < len(experiments) - 1:
-                    tree = "├──"
-                else:
-                    tree = "└──"
-                new_checkpoint = True
-            name = exp.get("name", rev[:7])
-            row.append(f"{tree} {queued}{name}{parent}")
+            tree = "├──" if i < len(experiments) - 1 else "└──"
 
-        if not no_timestamp:
-            row.append(_format_time(exp.get("timestamp")))
+        if not is_baseline:
+            new_checkpoint = not (tip and tip == parent_tip)
 
+        row = [
+            name,
+            queued,
+            tree,
+            _format_time(exp.get("timestamp")),
+            rev == "baseline",
+            parent,
+        ]
         _extend_row(
             row, metric_names, exp.get("metrics", {}).items(), precision
         )
         _extend_row(row, param_names, exp.get("params", {}).items(), precision)
 
-        yield row, style
+        yield row
 
 
 def _sort_column(sort_by, metric_names, param_names):
@@ -233,7 +231,7 @@ def _sort_exp(experiments, sort_path, sort_name, typ, reverse):
             return _sort((tip, experiments[tip]))
         data = exp.get(typ, {}).get(sort_path, {})
         val = flatten(data).get(sort_name)
-        return (val is None, val)
+        return val is None, val
 
     ret = OrderedDict()
     if "baseline" in experiments:
@@ -245,7 +243,7 @@ def _sort_exp(experiments, sort_path, sort_name, typ, reverse):
 
 def _format_time(timestamp):
     if timestamp is None:
-        return "-"
+        return FILL_VALUE
     if timestamp.date() == date.today():
         fmt = "%I:%M %p"
     else:
@@ -253,42 +251,23 @@ def _format_time(timestamp):
     return timestamp.strftime(fmt)
 
 
-def _format_field(val, precision=DEFAULT_PRECISION):
-    if isinstance(val, float):
-        fmt = f"{{:.{precision}g}}"
-        return fmt.format(val)
-    if isinstance(val, Mapping):
-        return {k: _format_field(v) for k, v in val.items()}
-    if isinstance(val, list):
-        return [_format_field(x) for x in val]
-    return str(val)
-
-
 def _extend_row(row, names, items, precision):
     from rich.text import Text
 
+    from dvc.compare import _format_field, with_value
+
     if not items:
-        row.extend(["-"] * len(names))
+        row.extend(FILL_VALUE for keys in names.values() for _ in keys)
         return
 
     for fname, item in items:
-        if isinstance(item, dict):
-            item = flatten(item)
-        else:
-            item = {fname: item}
+        item = flatten(item) if isinstance(item, dict) else {fname: item}
         for name in names[fname]:
-            if name in item:
-                value = item[name]
-                if value is None:
-                    text = "-"
-                else:
-                    # wrap field data in rich.Text, otherwise rich may
-                    # interpret unescaped braces from list/dict types as rich
-                    # markup tags
-                    text = Text(str(_format_field(value, precision)))
-                row.append(text)
-            else:
-                row.append("-")
+            value = with_value(item.get(name), FILL_VALUE)
+            # wrap field data in rich.Text, otherwise rich may
+            # interpret unescaped braces from list/dict types as rich
+            # markup tags
+            row.append(Text(str(_format_field(value, precision))))
 
 
 def _parse_filter_list(param_list):
@@ -302,9 +281,62 @@ def _parse_filter_list(param_list):
     return ret
 
 
-def _experiments_table(all_experiments, **kwargs):
-    from dvc.utils.table import Table
+def experiments_table(
+    all_experiments,
+    metric_headers,
+    metric_names,
+    param_headers,
+    param_names,
+    sort_by=None,
+    sort_order=None,
+    precision=DEFAULT_PRECISION,
+) -> "TabularData":
+    from funcy import lconcat
 
+    from dvc.compare import TabularData
+
+    headers = [
+        "Experiment",
+        "queued",
+        "ident_guide",
+        "Created",
+        "is_baseline",
+        "parent",
+    ]
+    td = TabularData(
+        lconcat(headers, metric_headers, param_headers), fill_value=FILL_VALUE
+    )
+    for base_rev, experiments in all_experiments.items():
+        rows = _collect_rows(
+            base_rev,
+            experiments,
+            metric_names,
+            param_names,
+            sort_by=sort_by,
+            sort_order=sort_order,
+            precision=precision,
+        )
+        td.extend(rows)
+
+    return td
+
+
+def prepare_exp_id(kwargs):
+    exp_name = kwargs["Experiment"]
+    ident = kwargs.get("ident_guide")
+    queued = kwargs.get("queued")
+    parent = kwargs.get("parent")
+    return "{}{}{}{}".format(
+        f"{ident} " if ident else "",
+        "*" if queued else "",
+        exp_name,
+        f" ({parent})" if parent else "",
+    )
+
+
+def show_experiments(
+    all_experiments, pager=True, no_timestamp=False, **kwargs
+):
     include_metrics = _parse_filter_list(kwargs.pop("include_metrics", []))
     exclude_metrics = _parse_filter_list(kwargs.pop("exclude_metrics", []))
     include_params = _parse_filter_list(kwargs.pop("include_params", []))
@@ -317,44 +349,68 @@ def _experiments_table(all_experiments, **kwargs):
         include_params=include_params,
         exclude_params=exclude_params,
     )
+    metric_headers = _normalize_headers(metric_names)
+    param_headers = _normalize_headers(param_names)
 
-    table = Table()
-    table.add_column(
-        "Experiment", no_wrap=True, header_style="black on grey93"
-    )
-    if not kwargs.get("no_timestamp", False):
-        table.add_column("Created", header_style="black on grey93")
-    _add_data_columns(
-        table,
+    td = experiments_table(
+        all_experiments,
+        metric_headers,
         metric_names,
-        justify="right",
-        no_wrap=True,
-        header_style="black on cornsilk1",
-    )
-    _add_data_columns(
-        table, param_names, justify="left", header_style="black on light_cyan1"
+        param_headers,
+        param_names,
+        kwargs.get("sort_by"),
+        kwargs.get("sort_order"),
+        kwargs.get("precision"),
     )
 
-    for base_rev, experiments in all_experiments.items():
-        for row, _, in _collect_rows(
-            base_rev, experiments, metric_names, param_names, **kwargs,
-        ):
-            table.add_row(*row)
+    if no_timestamp:
+        td.drop("Created")
 
-    return table
+    baseline_styler = iffy(constantly({"style": "bold"}), default={})
+    row_styles = lmap(baseline_styler, td.column("is_baseline"))
+    td.drop("is_baseline")
+
+    merge_headers = ["Experiment", "queued", "ident_guide", "parent"]
+    td.column("Experiment")[:] = map(prepare_exp_id, td.as_dict(merge_headers))
+    td.drop(*merge_headers[1:])
+
+    headers = {"metrics": metric_headers, "params": param_headers}
+    styles = {
+        "Experiment": {"no_wrap": True, "header_style": "black on grey93"},
+        "Created": {"header_style": "black on grey93"},
+    }
+    header_bg_colors = {"metrics": "cornsilk1", "params": "light_cyan1"}
+    styles.update(
+        {
+            header: {
+                "justify": "left" if typ == "metrics" else "params",
+                "header_style": f"black on {header_bg_colors[typ]}",
+                "collapse": idx != 0,
+                "no_wrap": typ == "metrics",
+            }
+            for typ, hs in headers.items()
+            for idx, header in enumerate(hs)
+        }
+    )
+
+    td.render(
+        pager=pager,
+        borders=True,
+        rich_table=True,
+        header_styles=styles,
+        row_styles=row_styles,
+    )
 
 
-def _add_data_columns(table, names, **kwargs):
+def _normalize_headers(names):
     count = Counter(
         name for path in names for name in names[path] for path in names
     )
-    first = True
-    for path in names:
-        for name in names[path]:
-            col_name = name if count[name] == 1 else f"{path}:{name}"
-            kwargs["collapse"] = False if first else True
-            table.add_column(col_name, **kwargs)
-            first = False
+    return [
+        name if count[name] == 1 else f"{path}:{name}"
+        for path in names
+        for name in names[path]
+    ]
 
 
 def _format_json(item):
@@ -365,8 +421,6 @@ def _format_json(item):
 
 class CmdExperimentsShow(CmdBase):
     def run(self):
-        from rich.console import Console
-
         try:
             all_experiments = self.repo.experiments.show(
                 all_branches=self.args.all_branches,
@@ -374,20 +428,18 @@ class CmdExperimentsShow(CmdBase):
                 all_commits=self.args.all_commits,
                 sha_only=self.args.sha,
                 num=self.args.num,
+                param_deps=self.args.param_deps,
             )
+        except DvcException:
+            logger.exception("failed to show experiments")
+            return 1
 
-            if self.args.show_json:
-                import json
+        if self.args.show_json:
+            import json
 
-                logger.info(json.dumps(all_experiments, default=_format_json))
-                return 0
-
-            if self.args.precision is None:
-                precision = DEFAULT_PRECISION
-            else:
-                precision = self.args.precision
-
-            table = _experiments_table(
+            logger.info(json.dumps(all_experiments, default=_format_json))
+        else:
+            show_experiments(
                 all_experiments,
                 include_metrics=self.args.include_metrics,
                 exclude_metrics=self.args.exclude_metrics,
@@ -396,29 +448,9 @@ class CmdExperimentsShow(CmdBase):
                 no_timestamp=self.args.no_timestamp,
                 sort_by=self.args.sort_by,
                 sort_order=self.args.sort_order,
-                precision=precision,
+                precision=self.args.precision or DEFAULT_PRECISION,
+                pager=not self.args.no_pager,
             )
-
-            console = Console()
-            if self.args.no_pager:
-                console.print(table)
-            else:
-                from dvc.utils.pager import DvcPager
-
-                # NOTE: rich does not have native support for unlimited width
-                # via pager. we override rich table compression by setting
-                # console width to the full width of the table
-                measurement = table.__rich_measure__(console, SHOW_MAX_WIDTH)
-                console._width = (  # pylint: disable=protected-access
-                    measurement.maximum
-                )
-                with console.pager(pager=DvcPager(), styles=True):
-                    console.print(table)
-
-        except DvcException:
-            logger.exception("failed to show experiments")
-            return 1
-
         return 0
 
 
@@ -432,43 +464,6 @@ class CmdExperimentsApply(CmdBase):
         return 0
 
 
-def _show_diff(
-    diff,
-    title="",
-    markdown=False,
-    no_path=False,
-    old=False,
-    precision=DEFAULT_PRECISION,
-):
-    from dvc.utils.diff import table
-
-    rows = []
-    for fname, diff_ in diff.items():
-        sorted_diff = OrderedDict(sorted(diff_.items()))
-        for item, change in sorted_diff.items():
-            row = [] if no_path else [fname]
-            row.append(item)
-            if old:
-                row.append(_format_field(change.get("old"), precision))
-            row.append(_format_field(change["new"], precision))
-            row.append(
-                _format_field(
-                    change.get("diff", "diff not supported"), precision
-                )
-            )
-            rows.append(row)
-
-    header = [] if no_path else ["Path"]
-    header.append(title)
-    if old:
-        header.extend(["Old", "New"])
-    else:
-        header.append("Value")
-    header.append("Change")
-
-    return table(header, rows, markdown)
-
-
 class CmdExperimentsDiff(CmdBase):
     def run(self):
 
@@ -477,42 +472,56 @@ class CmdExperimentsDiff(CmdBase):
                 a_rev=self.args.a_rev,
                 b_rev=self.args.b_rev,
                 all=self.args.all,
+                param_deps=self.args.param_deps,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(diff))
-            else:
-                if self.args.precision is None:
-                    precision = DEFAULT_PRECISION
-                else:
-                    precision = self.args.precision
-
-                diffs = [("metrics", "Metric"), ("params", "Param")]
-                for key, title in diffs:
-                    table = _show_diff(
-                        diff[key],
-                        title=title,
-                        markdown=self.args.show_md,
-                        no_path=self.args.no_path,
-                        old=self.args.old,
-                        precision=precision,
-                    )
-                    if table:
-                        logger.info(table)
-                        logger.info("")
-
         except DvcException:
             logger.exception("failed to show experiments diff")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            logger.info(json.dumps(diff))
+        else:
+            from dvc.compare import show_diff
+
+            precision = self.args.precision or DEFAULT_PRECISION
+            diffs = [("metrics", "Metric"), ("params", "Param")]
+            for idx, (key, title) in enumerate(diffs):
+                if idx:
+                    from dvc.ui import ui
+
+                    # we are printing tables even in `--quiet` mode
+                    # so we should also be printing the "table" separator
+                    ui.write(force=True)
+
+                show_diff(
+                    diff[key],
+                    title=title,
+                    markdown=self.args.show_md,
+                    no_path=self.args.no_path,
+                    old=self.args.old,
+                    on_empty_diff="diff not supported",
+                    precision=precision if key == "metrics" else None,
+                )
+
         return 0
 
 
 class CmdExperimentsRun(CmdRepro):
     def run(self):
-        from dvc.command.metrics import _show_metrics
+        from dvc.compare import show_metrics
+
+        if self.args.checkpoint_resume:
+            if self.args.reset:
+                raise InvalidArgumentError(
+                    "--reset and --rev are mutually exclusive."
+                )
+            if not (self.args.queue or self.args.tmp_dir):
+                raise InvalidArgumentError(
+                    "--rev can only be used in conjunction with "
+                    "--queue or --temp."
+                )
 
         if self.args.reset:
             logger.info("Any existing checkpoints will be reset and re-run.")
@@ -532,7 +541,7 @@ class CmdExperimentsRun(CmdRepro):
         if self.args.metrics and results:
             metrics = self.repo.metrics.show(revs=list(results))
             metrics.pop("workspace", None)
-            logger.info(_show_metrics(metrics))
+            logger.info(show_metrics(metrics))
 
         return 0
 
@@ -783,6 +792,12 @@ def add_parser(subparsers, parent_parser):
         help="Exclude the specified params from output table.",
         metavar="<params_list>",
     )
+    experiments_show_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     experiments_show_parser.add_argument(
         "--sort-by",
         help="Sort related experiments by the specified metric or param.",
@@ -868,6 +883,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show unchanged metrics/params as well.",
     )
+    experiments_diff_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     experiments_diff_parser.add_argument(
         "--show-json",
         action="store_true",
@@ -920,9 +941,8 @@ def add_parser(subparsers, parent_parser):
         type=str,
         dest="checkpoint_resume",
         help=(
-            "Continue the specified checkpoint experiment. "
-            "(Only required for explicitly resuming checkpoints in queued "
-            "or temp dir runs.)"
+            "Continue the specified checkpoint experiment. Can only be used "
+            "in conjunction with --queue or --temp."
         ),
         metavar="<experiment_rev>",
     ).complete = completion.EXPERIMENT
@@ -967,6 +987,7 @@ def add_parser(subparsers, parent_parser):
         help="Keep experiments derived from all Git tags.",
     )
     experiments_gc_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -1215,6 +1236,7 @@ def _add_run_common(parser):
         "-j",
         "--jobs",
         type=int,
+        default=1,
         help="Run the specified number of experiments at a time in parallel.",
         metavar="<number>",
     )

dvc/command/gc.py

@@ -97,6 +97,7 @@ def add_parser(subparsers, parent_parser):
         help="Keep data files for all Git tags.",
     )
     gc_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,

dvc/command/live.py

@@ -4,7 +4,6 @@ import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, fix_subparsers
-from dvc.utils.html import write
 
 logger = logging.getLogger(__name__)
 
@@ -16,7 +15,7 @@ class CmdLive(CmdBase):
         metrics, plots = self.repo.live.show(target=target, revs=revs)
 
         html_path = self.args.target + ".html"
-        write(html_path, plots, metrics)
+        self.repo.plots.write_html(html_path, plots, metrics)
 
         logger.info(f"\nfile://{os.path.abspath(html_path)}")
 

dvc/command/ls/__init__.py

@@ -34,7 +34,11 @@ class CmdList(CmdBaseNoRepo):
                 recursive=self.args.recursive,
                 dvc_only=self.args.dvc_only,
             )
-            if entries:
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(entries))
+            elif entries:
                 entries = _prettify(entries, sys.stdout.isatty())
                 logger.info("\n".join(entries))
             return 0
@@ -65,6 +69,9 @@ def add_parser(subparsers, parent_parser):
     list_parser.add_argument(
         "--dvc-only", action="store_true", help="Show only DVC outputs."
     )
+    list_parser.add_argument(
+        "--show-json", action="store_true", help="Show output in JSON format."
+    )
     list_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/metrics.py

@@ -11,64 +11,6 @@ logger = logging.getLogger(__name__)
 DEFAULT_PRECISION = 5
 
 
-def _show_metrics(
-    metrics,
-    markdown=False,
-    all_branches=False,
-    all_tags=False,
-    all_commits=False,
-    precision=None,
-):
-    from dvc.utils.diff import format_dict, table
-    from dvc.utils.flatten import flatten
-
-    # When `metrics` contains a `None` key, it means that some files
-    # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
-    missing = metrics.pop(None, None)
-    with_rev = any([all_branches, all_tags, all_commits])
-    header_set = set()
-    rows = []
-
-    if precision is None:
-        precision = DEFAULT_PRECISION
-
-    def _round(val):
-        if isinstance(val, float):
-            return round(val, precision)
-        return val
-
-    for _branch, val in metrics.items():
-        for _fname, metric in val.items():
-            if not isinstance(metric, dict):
-                header_set.add("")
-                continue
-            for key, _val in flatten(format_dict(metric)).items():
-                header_set.add(key)
-    header = sorted(header_set)
-    for branch, val in metrics.items():
-        for fname, metric in val.items():
-            row = []
-            if with_rev:
-                row.append(branch)
-            row.append(fname)
-            if not isinstance(metric, dict):
-                row.append(str(metric))
-                rows.append(row)
-                continue
-            flattened_val = flatten(format_dict(metric))
-
-            for i in header:
-                row.append(_round(flattened_val.get(i)))
-            rows.append(row)
-    header.insert(0, "Path")
-    if with_rev:
-        header.insert(0, "Revision")
-
-    if missing:
-        raise BadMetricError(missing)
-    return table(header, rows, markdown)
-
-
 class CmdMetricsBase(CmdBase):
     UNINITIALIZED = True
 
@@ -83,61 +25,37 @@ class CmdMetricsShow(CmdMetricsBase):
                 all_commits=self.args.all_commits,
                 recursive=self.args.recursive,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(metrics))
-            else:
-                table = _show_metrics(
-                    metrics,
-                    self.args.show_md,
-                    self.args.all_branches,
-                    self.args.all_tags,
-                    self.args.all_commits,
-                )
-                if table:
-                    logger.info(table)
         except DvcException:
             logger.exception("")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            logger.info(json.dumps(metrics))
+        else:
+            from dvc.compare import show_metrics
+
+            # When `metrics` contains a `None` key, it means that some files
+            # specified as `targets` in `repo.metrics.show` didn't contain any
+            # metrics.
+            missing = metrics.pop(None, None)
+            if missing:
+                raise BadMetricError(missing)
+
+            show_metrics(
+                metrics,
+                markdown=self.args.show_md,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
+                precision=self.args.precision or DEFAULT_PRECISION,
+                round_digits=True,
+            )
+
         return 0
 
 
-def _show_diff(diff, markdown=False, no_path=False, precision=None):
-    from collections import OrderedDict
-
-    from dvc.utils.diff import table
-
-    if precision is None:
-        precision = DEFAULT_PRECISION
-
-    def _round(val):
-        if isinstance(val, float):
-            return round(val, precision)
-
-        return val
-
-    rows = []
-    for fname, mdiff in diff.items():
-        sorted_mdiff = OrderedDict(sorted(mdiff.items()))
-        for metric, change in sorted_mdiff.items():
-            row = [] if no_path else [fname]
-            row.append(metric)
-            row.append(_round(change.get("old")))
-            row.append(_round(change["new"]))
-            row.append(_round(change.get("diff")))
-            rows.append(row)
-
-    header = [] if no_path else ["Path"]
-    header.append("Metric")
-    header.extend(["Old", "New"])
-    header.append("Change")
-
-    return table(header, rows, markdown)
-
-
 class CmdMetricsDiff(CmdMetricsBase):
     def run(self):
         try:
@@ -148,25 +66,26 @@ class CmdMetricsDiff(CmdMetricsBase):
                 recursive=self.args.recursive,
                 all=self.args.all,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(diff))
-            else:
-                table = _show_diff(
-                    diff,
-                    self.args.show_md,
-                    self.args.no_path,
-                    precision=self.args.precision,
-                )
-                if table:
-                    logger.info(table)
-
         except DvcException:
             logger.exception("failed to show metrics diff")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            logger.info(json.dumps(diff))
+        else:
+            from dvc.compare import show_diff
+
+            show_diff(
+                diff,
+                title="Metric",
+                markdown=self.args.show_md,
+                no_path=self.args.no_path,
+                precision=self.args.precision or DEFAULT_PRECISION,
+                round_digits=True,
+            )
+
         return 0
 
 
@@ -219,6 +138,7 @@ def add_parser(subparsers, parent_parser):
         help="Show metrics for all tags.",
     )
     metrics_show_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,

dvc/command/params.py

@@ -1,6 +1,5 @@
 import argparse
 import logging
-from collections import OrderedDict
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
@@ -9,27 +8,6 @@ from dvc.exceptions import DvcException
 logger = logging.getLogger(__name__)
 
 
-def _show_diff(diff, markdown=False, no_path=False):
-    from dvc.utils.diff import table
-
-    rows = []
-    for fname, pdiff in diff.items():
-        sorted_pdiff = OrderedDict(sorted(pdiff.items()))
-        for param, change in sorted_pdiff.items():
-            row = [] if no_path else [fname]
-            row.append(param)
-            row.append(change["old"])
-            row.append(change["new"])
-            rows.append(row)
-
-    header = [] if no_path else ["Path"]
-    header.append("Param")
-    header.append("Old")
-    header.append("New")
-
-    return table(header, rows, markdown)
-
-
 class CmdParamsDiff(CmdBase):
     UNINITIALIZED = True
 
@@ -40,21 +18,27 @@ class CmdParamsDiff(CmdBase):
                 b_rev=self.args.b_rev,
                 targets=self.args.targets,
                 all=self.args.all,
+                deps=self.args.deps,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(diff))
-            else:
-                table = _show_diff(diff, self.args.show_md, self.args.no_path)
-                if table:
-                    logger.info(table)
-
         except DvcException:
             logger.exception("failed to show params diff")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            logger.info(json.dumps(diff))
+        else:
+            from dvc.compare import show_diff
+
+            show_diff(
+                diff,
+                title="Param",
+                markdown=self.args.show_md,
+                no_path=self.args.no_path,
+                show_changes=False,
+            )
+
         return 0
 
 
@@ -111,6 +95,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show unchanged params as well.",
     )
+    params_diff_parser.add_argument(
+        "--deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     params_diff_parser.add_argument(
         "--show-json",
         action="store_true",

dvc/command/plots.py

@@ -1,12 +1,10 @@
 import argparse
 import logging
-import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
 from dvc.utils import format_link
-from dvc.utils.html import write
 
 logger = logging.getLogger(__name__)
 
@@ -23,6 +21,8 @@ class CmdPlots(CmdBase):
         return {k: v for k, v in props.items() if v is not None}
 
     def run(self):
+        from pathlib import Path
+
         if self.args.show_vega:
             if not self.args.targets:
                 logger.error("please specify a target for `--show-vega`")
@@ -41,17 +41,27 @@ class CmdPlots(CmdBase):
                 logger.info(plots[target])
                 return 0
 
-            path = self.args.out or "plots.html"
-            path = os.path.join(os.getcwd(), path)
-
-            write(path, plots)
-
-            logger.info(f"file://{path}")
+            rel: str = self.args.out or "plots.html"
+            path = (Path.cwd() / rel).resolve()
+            self.repo.plots.write_html(
+                path, plots=plots, html_template_path=self.args.html_template
+            )
 
         except DvcException:
             logger.exception("")
             return 1
 
+        assert path.is_absolute()  # as_uri throws ValueError if not absolute
+        url = path.as_uri()
+        logger.info(url)
+        if self.args.open:
+            import webbrowser
+
+            opened = webbrowser.open(rel)
+            if not opened:
+                logger.error("Failed to open. Please try opening it manually.")
+                return 1
+
         return 0
 
 
@@ -85,7 +95,7 @@ class CmdPlotsModify(CmdPlots):
 def add_parser(subparsers, parent_parser):
     PLOTS_HELP = (
         "Commands to visualize and compare plot metrics in structured files "
-        "(JSON, YAML, CSV, TSV)"
+        "(JSON, YAML, CSV, TSV)."
     )
 
     plots_parser = subparsers.add_parser(
@@ -229,3 +239,15 @@ def _add_output_arguments(parser):
         default=False,
         help="Show output in Vega format.",
     )
+    parser.add_argument(
+        "--open",
+        action="store_true",
+        default=False,
+        help="Open plot file directly in the browser.",
+    )
+    parser.add_argument(
+        "--html-template",
+        default=None,
+        help="Custom HTML template for VEGA visualization.",
+        metavar="<path>",
+    )

dvc/command/repro.py

@@ -3,7 +3,6 @@ import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.metrics import _show_metrics
 from dvc.command.status import CmdDataStatus
 
 logger = logging.getLogger(__name__)
@@ -20,8 +19,10 @@ class CmdRepro(CmdBase):
             )
 
         if self.args.metrics:
+            from dvc.compare import show_metrics
+
             metrics = self.repo.metrics.show()
-            logger.info(_show_metrics(metrics))
+            show_metrics(metrics)
 
         return 0
 

dvc/command/stage.py

@@ -84,7 +84,7 @@ class CmdStageList(CmdBase):
         return dict.fromkeys(collected).keys()
 
     def run(self):
-        from dvc.utils.diff import table
+        from dvc.ui import ui
 
         def log_error(relpath: str, exc: Exception):
             if self.args.fail:
@@ -98,8 +98,7 @@ class CmdStageList(CmdBase):
         names_only = self.args.names_only
 
         data = prepare_stages_data(stages, description=not names_only)
-        if data:
-            print(table(header=(), rows=data.items()))
+        ui.table(data.items())
 
         return 0
 
@@ -255,7 +254,8 @@ def _add_common_args(parser):
         "--checkpoints",
         action="append",
         default=[],
-        help=argparse.SUPPRESS,
+        help="Declare checkpoint output file or directory for 'dvc exp run'. "
+        "Not compatible with 'dvc repro'.",
         metavar="<filename>",
     ).complete = completion.FILE
     parser.add_argument(

dvc/command/status.py

@@ -44,19 +44,27 @@ class CmdDataStatus(CmdDataBase):
                 self._show(value, indent + 1)
 
     def run(self):
+        from dvc.repo import lock_repo
+        from dvc.ui import ui
+
         indent = 1 if self.args.cloud else 0
-        try:
-            st = self.repo.status(
-                targets=self.args.targets,
-                jobs=self.args.jobs,
-                cloud=self.args.cloud,
-                remote=self.args.remote,
-                all_branches=self.args.all_branches,
-                all_tags=self.args.all_tags,
-                all_commits=self.args.all_commits,
-                with_deps=self.args.with_deps,
-                recursive=self.args.recursive,
-            )
+
+        with lock_repo(self.repo):
+            try:
+                st = self.repo.status(
+                    targets=self.args.targets,
+                    jobs=self.args.jobs,
+                    cloud=self.args.cloud,
+                    remote=self.args.remote,
+                    all_branches=self.args.all_branches,
+                    all_tags=self.args.all_tags,
+                    all_commits=self.args.all_commits,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("")
+                return 1
 
             if self.args.quiet:
                 return bool(st)
@@ -65,19 +73,21 @@ class CmdDataStatus(CmdDataBase):
                 import json
 
                 logger.info(json.dumps(st))
-            elif st:
+                return 0
+
+            if st:
                 self._show(st, indent)
-            elif not self.repo.stages:
-                logger.info(self.EMPTY_PROJECT_MSG)
+                return 0
+
+            # additional hints for the user
+            if not self.repo.stages:
+                ui.write(self.EMPTY_PROJECT_MSG)
             elif self.args.cloud or self.args.remote:
                 remote = self.args.remote or self.repo.config["core"].get(
                     "remote"
                 )
-                logger.info(self.IN_SYNC_MSG.format(remote=remote))
+                ui.write(self.IN_SYNC_MSG.format(remote=remote))
             else:
-                logger.info(self.UP_TO_DATE_MSG)
+                ui.write(self.UP_TO_DATE_MSG)
 
-        except DvcException:
-            logger.exception("")
-            return 1
         return 0

dvc/command/unprotect.py

@@ -23,7 +23,7 @@ class CmdUnprotect(CmdBase):
 def add_parser(subparsers, parent_parser):
     UNPROTECT_HELP = (
         "Unprotect tracked files or directories (when hardlinks or symlinks "
-        "have been enabled with `dvc config cache.type`)"
+        "have been enabled with `dvc config cache.type`)."
     )
     unprotect_parser = subparsers.add_parser(
         "unprotect",

dvc/compare.py

@@ -0,0 +1,321 @@
+from collections.abc import Mapping
+from itertools import chain, repeat, zip_longest
+from operator import itemgetter
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    ItemsView,
+    Iterable,
+    Iterator,
+    List,
+    MutableSequence,
+    Sequence,
+    Tuple,
+    Union,
+    overload,
+)
+
+from funcy import reraise
+
+if TYPE_CHECKING:
+    from dvc.ui.table import CellT
+
+
+class Column(List["CellT"]):
+    pass
+
+
+def with_value(value, default):
+    return default if value is None else value
+
+
+class TabularData(MutableSequence[Sequence["CellT"]]):
+    def __init__(self, columns: Sequence[str], fill_value: str = ""):
+        self._columns: Dict[str, Column] = {name: Column() for name in columns}
+        self._keys: List[str] = list(columns)
+        self._fill_value = fill_value
+
+    @property
+    def columns(self) -> List[Column]:
+        return list(map(self.column, self.keys()))
+
+    def column(self, name: str) -> Column:
+        return self._columns[name]
+
+    def items(self) -> ItemsView[str, Column]:
+        projection = {k: self.column(k) for k in self.keys()}
+        return projection.items()
+
+    def keys(self) -> List[str]:
+        return self._keys
+
+    def _iter_col_row(
+        self, row: Sequence["CellT"]
+    ) -> Iterator[Tuple["CellT", Column]]:
+        for val, col in zip_longest(row, self.columns):
+            if col is None:
+                break
+            yield with_value(val, self._fill_value), col
+
+    def append(self, value: Sequence["CellT"]) -> None:
+        for val, col in self._iter_col_row(value):
+            col.append(val)
+
+    def extend(self, values: Iterable[Sequence["CellT"]]) -> None:
+        for row in values:
+            self.append(row)
+
+    def insert(self, index: int, value: Sequence["CellT"]) -> None:
+        for val, col in self._iter_col_row(value):
+            col.insert(index, val)
+
+    def __iter__(self) -> Iterator[List["CellT"]]:
+        return map(list, zip(*self.columns))
+
+    def __getattr__(self, item: str) -> Column:
+        with reraise(KeyError, AttributeError):
+            return self.column(item)
+
+    def __getitem__(self, item: Union[int, slice]):
+        func = itemgetter(item)
+        it = map(func, self.columns)
+        if isinstance(item, slice):
+            it = map(list, zip(*it))
+        return list(it)
+
+    @overload
+    def __setitem__(self, item: int, value: Sequence["CellT"]) -> None:
+        ...
+
+    @overload
+    def __setitem__(
+        self, item: slice, value: Iterable[Sequence["CellT"]]
+    ) -> None:
+        ...
+
+    def __setitem__(self, item, value) -> None:
+        it = value
+        if isinstance(item, slice):
+            n = len(self.columns)
+            normalized_rows = (
+                chain(val, repeat(self._fill_value, n - len(val)))
+                for val in value
+            )
+            # we need to transpose those rows into columnar format
+            # as we work in terms of column-based arrays
+            it = zip(*normalized_rows)
+
+        for i, col in self._iter_col_row(it):
+            col[item] = i
+
+    def __delitem__(self, item: Union[int, slice]) -> None:
+        for col in self.columns:
+            del col[item]
+
+    def __len__(self) -> int:
+        return len(self.columns[0])
+
+    @property
+    def shape(self) -> Tuple[int, int]:
+        return len(self.columns), len(self)
+
+    def drop(self, *col_names: str) -> None:
+        for col_name in col_names:
+            self._keys.remove(col_name)
+            self._columns.pop(col_name)
+
+    def rename(self, from_col_name: str, to_col_name: str) -> None:
+        self._columns[to_col_name] = self._columns.pop(from_col_name)
+        self._keys[self._keys.index(from_col_name)] = to_col_name
+
+    def project(self, *col_names: str) -> None:
+        self.drop(*(set(self._keys) - set(col_names)))
+        self._keys = list(col_names)
+
+    def to_csv(self) -> str:
+        import csv
+        from io import StringIO
+
+        buff = StringIO()
+        writer = csv.writer(buff)
+        writer.writerow(self.keys())
+
+        for row in self:
+            writer.writerow(row)
+        return buff.getvalue()
+
+    def add_column(self, name: str) -> None:
+        self._columns[name] = Column([self._fill_value] * len(self))
+        self._keys.append(name)
+
+    def row_from_dict(self, d: Dict[str, "CellT"]) -> None:
+        keys = self.keys()
+        for key in d:
+            if key not in keys:
+                self.add_column(key)
+
+        row: List["CellT"] = [
+            with_value(d.get(key), self._fill_value) for key in self.keys()
+        ]
+        self.append(row)
+
+    def render(self, **kwargs: Any):
+        from dvc.ui import ui
+
+        ui.table(self, headers=self.keys(), **kwargs)
+
+    def as_dict(self, cols: Iterable[str] = None) -> Iterable[Dict[str, str]]:
+        keys = self.keys() if cols is None else set(cols)
+        return [
+            {k: self._columns[k][i] for k in keys} for i in range(len(self))
+        ]
+
+
+def _normalize_float(val: float, precision: int):
+    return f"{val:.{precision}g}"
+
+
+def _format_field(
+    val: Any, precision: int = None, round_digits: bool = False
+) -> str:
+    def _format(_val):
+        if isinstance(_val, float) and precision:
+            func = round if round_digits else _normalize_float
+            return func(_val, precision)
+        if isinstance(_val, Mapping):
+            return {k: _format(v) for k, v in _val.items()}
+        if isinstance(_val, list):
+            return [_format(x) for x in _val]
+        return _val
+
+    return str(_format(val))
+
+
+def diff_table(
+    diff,
+    title: str,
+    old: bool = True,
+    no_path: bool = False,
+    show_changes: bool = True,
+    precision: int = None,
+    round_digits: bool = False,
+    on_empty_diff: str = None,
+) -> TabularData:
+    headers: List[str] = ["Path", title, "Old", "New", "Change"]
+    fill_value = "-"
+    td = TabularData(headers, fill_value=fill_value)
+
+    for fname, diff_in_file in diff.items():
+        for item, change in sorted(diff_in_file.items()):
+            old_value = with_value(change.get("old"), fill_value)
+            new_value = with_value(change.get("new"), fill_value)
+            diff_value = with_value(
+                change.get("diff", on_empty_diff), fill_value
+            )
+            td.append(
+                [
+                    fname,
+                    str(item),
+                    _format_field(old_value, precision, round_digits),
+                    _format_field(new_value, precision, round_digits),
+                    _format_field(diff_value, precision, round_digits),
+                ]
+            )
+
+    if no_path:
+        td.drop("Path")
+
+    if not show_changes:
+        td.drop("Change")
+
+    if not old:
+        td.drop("Old")
+        td.rename("New", "Value")
+
+    return td
+
+
+def show_diff(
+    diff,
+    title: str,
+    old: bool = True,
+    no_path: bool = False,
+    show_changes: bool = True,
+    precision: int = None,
+    round_digits: bool = False,
+    on_empty_diff: str = None,
+    markdown: bool = False,
+) -> None:
+    td = diff_table(
+        diff,
+        title=title,
+        old=old,
+        no_path=no_path,
+        show_changes=show_changes,
+        precision=precision,
+        round_digits=round_digits,
+        on_empty_diff=on_empty_diff,
+    )
+    td.render(markdown=markdown)
+
+
+def metrics_table(
+    metrics,
+    all_branches: bool = False,
+    all_tags: bool = False,
+    all_commits: bool = False,
+    precision: int = None,
+    round_digits: bool = False,
+):
+    from dvc.utils.diff import format_dict
+    from dvc.utils.flatten import flatten
+
+    td = TabularData(["Revision", "Path"], fill_value="-")
+
+    for branch, val in metrics.items():
+        for fname, metric in val.items():
+            row_data: Dict[str, str] = {
+                "Revision": branch,
+                "Path": fname,
+            }
+            flattened = (
+                flatten(format_dict(metric))
+                if isinstance(metric, dict)
+                else {"": metric}
+            )
+            row_data.update(
+                {
+                    k: _format_field(v, precision, round_digits)
+                    for k, v in flattened.items()
+                }
+            )
+            td.row_from_dict(row_data)
+
+    rev, path, *metrics_headers = td.keys()
+    td.project(rev, path, *sorted(metrics_headers))
+
+    if not any([all_branches, all_tags, all_commits]):
+        td.drop("Revision")
+
+    return td
+
+
+def show_metrics(
+    metrics,
+    markdown: bool = False,
+    all_branches: bool = False,
+    all_tags: bool = False,
+    all_commits: bool = False,
+    precision: int = None,
+    round_digits: bool = False,
+) -> None:
+    td = metrics_table(
+        metrics,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        all_commits=all_commits,
+        precision=precision,
+        round_digits=round_digits,
+    )
+    td.render(markdown=markdown)

dvc/config_schema.py

@@ -145,6 +145,7 @@ SCHEMA = {
                     "session_token": str,
                     Optional("listobjects", default=False): Bool,  # obsoleted
                     Optional("use_ssl", default=True): Bool,
+                    Optional("ssl_verify", default=True): Bool,
                     "sse": str,
                     "sse_kms_key_id": str,
                     "acl": str,
@@ -171,7 +172,7 @@ SCHEMA = {
                     "allow_agent": Bool,
                     **REMOTE_COMMON,
                 },
-                "hdfs": {"user": str, **REMOTE_COMMON},
+                "hdfs": {"user": str, "kerb_ticket": str, **REMOTE_COMMON},
                 "webhdfs": {
                     "hdfscli_config": str,
                     "webhdfs_token": str,
@@ -222,4 +223,5 @@ SCHEMA = {
         # enabled by default. It's of no use, kept for backward compatibility.
         Optional("parametrization", default=True): Bool
     },
+    "plots": {"html_template": str},
 }

dvc/dagascii.py

@@ -3,52 +3,14 @@
 import logging
 import math
 import os
-import pydoc
-import sys
 
 from grandalf.graphs import Edge, Graph, Vertex
 from grandalf.layouts import SugiyamaLayout
 from grandalf.routing import EdgeViewer, route_with_lines
 
-from dvc.env import DVC_PAGER
-from dvc.utils import format_link
-
 logger = logging.getLogger(__name__)
 
 
-DEFAULT_PAGER = "less"
-DEFAULT_PAGER_FORMATTED = "{} --chop-long-lines --clear-screen".format(
-    DEFAULT_PAGER
-)
-
-
-def make_pager(cmd):
-    def pager(text):
-        return pydoc.tempfilepager(pydoc.plain(text), cmd)
-
-    return pager
-
-
-def find_pager():
-    if not sys.stdout.isatty():
-        return pydoc.plainpager
-
-    env_pager = os.getenv(DVC_PAGER)
-    if env_pager:
-        return make_pager(env_pager)
-
-    if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
-        return make_pager(DEFAULT_PAGER_FORMATTED)
-
-    logger.warning(
-        "Unable to find `less` in the PATH. Check out "
-        "{} for more info.".format(
-            format_link("https://man.dvc.org/pipeline/show")
-        )
-    )
-    return pydoc.plainpager
-
-
 class VertexViewer:
     """Class to define vertex box boundaries that will be accounted for during
     graph building by grandalf.

dvc/data_cloud.py

@@ -92,24 +92,6 @@ class DataCloud:
             show_checksums=show_checksums,
         )
 
-    def transfer(self, source, jobs=None, remote=None, command=None):
-        """Transfer data items in a cloud-agnostic way.
-
-        Args:
-            source (str): url for the source location.
-            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.BaseRemote): optional remote to compare
-                cache to. By default remote from core.remote config option
-                is used.
-            command (str): the command which is benefitting from this function
-                (to be used for reporting better error messages).
-        """
-        from dvc.fs import get_cloud_fs
-
-        from_fs = get_cloud_fs(self.repo, url=source)
-        remote = self.get_remote(remote, command)
-        return remote.transfer(from_fs, from_fs.path_info, jobs=jobs)
-
     def status(
         self,
         cache,

dvc/dependency/param.py

@@ -39,6 +39,7 @@ class ParamsDependency(LocalDependency):
             path
             or os.path.join(stage.repo.root_dir, self.DEFAULT_PARAMS_FILE),
             info=info,
+            fs=stage.repo.fs,
         )
 
     def dumpd(self):
@@ -84,19 +85,34 @@ class ParamsDependency(LocalDependency):
     def status(self):
         return self.workspace_status()
 
-    def read_params(self):
+    def _read(self):
         if not self.exists:
             return {}
 
         suffix = self.path_info.suffix.lower()
         loader = LOADERS[suffix]
         try:
-            config = loader(self.path_info, fs=self.repo.fs)
+            return loader(self.path_info, fs=self.repo.fs)
         except ParseError as exc:
             raise BadParamFileError(
                 f"Unable to read parameters from '{self}'"
             ) from exc
 
+    def read_params_d(self):
+        config = self._read()
+
+        ret = {}
+        for param in self.params:
+            dpath.util.merge(
+                ret,
+                dpath.util.search(config, param, separator="."),
+                separator=".",
+            )
+        return ret
+
+    def read_params(self):
+        config = self._read()
+
         ret = {}
         for param in self.params:
             try:

dvc/dependency/repo.py

@@ -55,6 +55,7 @@ class RepoDependency(LocalDependency):
                 self.repo.odb.local,
                 path_info,
                 repo.repo_fs,
+                self.repo.odb.local.fs.PARAM_CHECKSUM,
                 follow_subrepos=False,
             ).hash_info
 
@@ -78,6 +79,8 @@ class RepoDependency(LocalDependency):
 
     def download(self, to, jobs=None):
         from dvc.checkout import checkout
+        from dvc.config import NoRemoteError
+        from dvc.exceptions import NoOutputOrStageError
         from dvc.objects import save
         from dvc.objects.stage import stage
 
@@ -87,8 +90,17 @@ class RepoDependency(LocalDependency):
             if self.def_repo.get(self.PARAM_REV_LOCK) is None:
                 self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
             path_info = PathInfo(repo.root_dir) / self.def_path
+            try:
+                repo.fetch([path_info.fspath], jobs=jobs, recursive=True)
+            except (NoOutputOrStageError, NoRemoteError):
+                pass
             obj = stage(
-                odb, path_info, repo.repo_fs, jobs=jobs, follow_subrepos=False,
+                odb,
+                path_info,
+                repo.repo_fs,
+                odb.fs.PARAM_CHECKSUM,
+                jobs=jobs,
+                follow_subrepos=False,
             )
             save(odb, obj, jobs=jobs)
 

dvc/dir_info.py

@@ -1,117 +0,0 @@
-import posixpath
-from operator import itemgetter
-
-from funcy import cached_property
-from pygtrie import Trie
-
-from .hash_info import HashInfo
-
-
-def _diff(ancestor, other, allow_removed=False):
-    from dictdiffer import diff
-
-    from dvc.exceptions import MergeError
-
-    allowed = ["add"]
-    if allow_removed:
-        allowed.append("remove")
-
-    result = list(diff(ancestor, other))
-    for typ, _, _ in result:
-        if typ not in allowed:
-            raise MergeError(
-                "unable to auto-merge directories with diff that contains "
-                f"'{typ}'ed files"
-            )
-    return result
-
-
-def _merge(ancestor, our, their):
-    import copy
-
-    from dictdiffer import patch
-
-    our_diff = _diff(ancestor, our)
-    if not our_diff:
-        return copy.deepcopy(their)
-
-    their_diff = _diff(ancestor, their)
-    if not their_diff:
-        return copy.deepcopy(our)
-
-    # make sure there are no conflicting files
-    _diff(our, their, allow_removed=True)
-
-    return patch(our_diff + their_diff, ancestor)
-
-
-class DirInfo:
-    PARAM_RELPATH = "relpath"
-
-    def __init__(self):
-        self._dict = {}
-
-    def as_dict(self):
-        return self._dict.copy()
-
-    def add(self, key, hash_info):
-        self.__dict__.pop("trie", None)
-        self._dict[key] = hash_info
-
-    def get(self, key, default=None):
-        return self._dict.get(key, default)
-
-    @cached_property
-    def trie(self):
-        return Trie(self._dict)
-
-    @property
-    def size(self):
-        try:
-            return sum(hash_info.size for _, hash_info in self._dict.items())
-        except TypeError:
-            return None
-
-    @property
-    def nfiles(self):
-        return len(self._dict)
-
-    def items(self):
-        for key, hash_info in self._dict.items():
-            yield key, hash_info
-
-    @classmethod
-    def from_list(cls, lst):
-        ret = DirInfo()
-        for _entry in lst:
-            entry = _entry.copy()
-            relpath = entry.pop(cls.PARAM_RELPATH)
-            parts = tuple(relpath.split(posixpath.sep))
-            hash_info = HashInfo.from_dict(entry)
-            ret.add(parts, hash_info)
-        return ret
-
-    def to_list(self):
-        # Sorting the list by path to ensure reproducibility
-        return sorted(
-            (
-                {
-                    # NOTE: not using hash_info.to_dict() because we don't want
-                    # size/nfiles fields at this point.
-                    hash_info.name: hash_info.value,
-                    self.PARAM_RELPATH: posixpath.sep.join(parts),
-                }
-                for parts, hash_info in self._dict.items()  # noqa: B301
-            ),
-            key=itemgetter(self.PARAM_RELPATH),
-        )
-
-    def merge(self, ancestor, their):
-        merged_dict = _merge(
-            ancestor.as_dict(), self.as_dict(), their.as_dict(),
-        )
-
-        merged = DirInfo()
-        for key, hi in merged_dict.items():
-            merged.add(key, hi)
-        return merged

dvc/external_repo.py

@@ -144,7 +144,7 @@ def _cached_clone(url, rev, for_write=False):
     revision checked out. If for_write is set prevents reusing this dir via
     cache.
     """
-    from distutils.dir_util import copy_tree
+    from shutil import copytree
 
     # even if we have already cloned this repo, we may need to
     # fetch/fast-forward to get specified rev
@@ -156,7 +156,7 @@ def _cached_clone(url, rev, for_write=False):
     # Copy to a new dir to keep the clone clean
     repo_path = tempfile.mkdtemp("dvc-erepo")
     logger.debug("erepo: making a copy of %s clone", url)
-    copy_tree(clone_path, repo_path)
+    copytree(clone_path, repo_path)
 
     # Check out the specified revision
     if for_write:

dvc/fs/azure.py

@@ -48,11 +48,12 @@ class AzureFileSystem(FSSpecWrapper):
         self.path_info = self.PATH_CLS(url)
         self.bucket = self.path_info.bucket
 
-        self.login_method, self.login_info = self._prepare_credentials(config)
-
     def _prepare_credentials(self, config):
         from azure.identity.aio import DefaultAzureCredential
 
+        # Disable spam from failed cred types for DefaultAzureCredential
+        logging.getLogger("azure.identity.aio").setLevel(logging.ERROR)
+
         login_info = {}
         login_info["connection_string"] = config.get(
             "connection_string",
@@ -71,8 +72,12 @@ class AzureFileSystem(FSSpecWrapper):
         login_info["client_id"] = config.get("client_id")
         login_info["client_secret"] = config.get("client_secret")
 
-        if not any(login_info.values()):
-            login_info["credentials"] = DefaultAzureCredential()
+        if not any(
+            value for key, value in login_info.items() if key != "account_name"
+        ):
+            login_info["credential"] = DefaultAzureCredential(
+                exclude_interactive_browser_credential=False
+            )
 
         for login_method, required_keys in [  # noqa
             ("connection string", ["connection_string"]),
@@ -82,15 +87,19 @@ class AzureFileSystem(FSSpecWrapper):
             ),
             ("account key", ["account_name", "account_key"]),
             ("SAS token", ["account_name", "sas_token"]),
+            (
+                f"default credentials ({_DEFAULT_CREDS_STEPS})",
+                ["account_name", "credential"],
+            ),
             ("anonymous login", ["account_name"]),
-            (f"default credentials ({_DEFAULT_CREDS_STEPS})", ["credentials"]),
         ]:
             if all(login_info.get(key) is not None for key in required_keys):
                 break
         else:
             login_method = None
 
-        return login_method, login_info
+        self.login_method = login_method
+        return login_info
 
     @cached_property
     def _az_config(self):
@@ -111,7 +120,7 @@ class AzureFileSystem(FSSpecWrapper):
         from azure.core.exceptions import AzureError
 
         try:
-            file_system = AzureBlobFileSystem(**self.login_info)
+            file_system = AzureBlobFileSystem(**self.fs_args)
             if self.bucket not in [
                 container.rstrip("/") for container in file_system.ls("/")
             ]:

dvc/fs/base.py

@@ -1,6 +1,6 @@
 import logging
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from functools import partial
+from functools import partial, partialmethod
 from multiprocessing import cpu_count
 from typing import Any, ClassVar, Dict, FrozenSet, Optional
 
@@ -38,7 +38,7 @@ class BaseFileSystem:
     scheme = "base"
     REQUIRES: ClassVar[Dict[str, str]] = {}
     PATH_CLS = URLInfo  # type: Any
-    JOBS = 4 * cpu_count()
+    _JOBS = 4 * cpu_count()
 
     CHECKSUM_DIR_SUFFIX = ".dir"
     HASH_JOBS = max(1, min(4, cpu_count() // 2))
@@ -67,7 +67,7 @@ class BaseFileSystem:
         return (
             self.config.get("jobs")
             or (self.repo and self.repo.config["core"].get("jobs"))
-            or self.JOBS
+            or self._JOBS
         )
 
     @cached_property
@@ -175,9 +175,12 @@ class BaseFileSystem:
         """
         raise NotImplementedError
 
-    def ls(self, path_info, detail=False, **kwargs):
+    def ls(self, path_info, detail=False):
         raise RemoteActionNotImplemented("ls", self.scheme)
 
+    def find(self, path_info, detail=False):
+        raise RemoteActionNotImplemented("find", self.scheme)
+
     def is_empty(self, path_info):
         return False
 
@@ -258,6 +261,7 @@ class BaseFileSystem:
         name=None,
         no_progress_bar=False,
         jobs=None,
+        _only_file=False,
         **kwargs,
     ):
         if not hasattr(self, "_download"):
@@ -273,16 +277,21 @@ class BaseFileSystem:
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        if self.isdir(from_info):
+        if not _only_file and self.isdir(from_info):
             return self._download_dir(
                 from_info, to_info, name, no_progress_bar, jobs, **kwargs,
             )
         return self._download_file(from_info, to_info, name, no_progress_bar,)
 
+    download_file = partialmethod(download, _only_file=True)
+
     def _download_dir(
         self, from_info, to_info, name, no_progress_bar, jobs, **kwargs,
     ):
         from_infos = list(self.walk_files(from_info, **kwargs))
+        if not from_infos:
+            makedirs(to_info, exist_ok=True)
+            return None
         to_infos = (
             to_info / info.relative_to(from_info) for info in from_infos
         )

dvc/fs/dvc.py

@@ -48,9 +48,9 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         # NOTE: use string paths here for performance reasons
         key = tuple(relpath(path_info, out.path_info).split(os.sep))
         out.get_dir_cache(remote=remote)
-        file_hash = out.hash_info.dir_info.get(key)
-        if file_hash:
-            return file_hash
+        obj = out.obj.trie.get(key)
+        if obj:
+            return obj.hash_info
         raise FileNotFoundError
 
     def open(  # type: ignore
@@ -131,21 +131,14 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         # pull dir cache if needed
         out.get_dir_cache(**kwargs)
 
-        dir_cache = out.dir_cache
-        if not dir_cache:
-            raise FileNotFoundError
-
-        from dvc.objects import Tree
-
-        hash_info = Tree.save_dir_info(out.odb, dir_cache)
-        if hash_info != out.hash_info:
+        if not out.obj:
             raise FileNotFoundError
 
     def _add_dir(self, trie, out, **kwargs):
         self._fetch_dir(out, **kwargs)
 
         base = out.path_info.parts
-        for key, _ in out.dir_cache.items():  # noqa: B301
+        for key, _ in out.obj:  # noqa: B301
             trie[base + key] = None
 
     def _walk(self, root, trie, topdown=True, **kwargs):
@@ -241,9 +234,9 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         elif meta.part_of_output:
             (out,) = meta.outs
             key = path_info.relative_to(out.path_info).parts
-            hash_info = out.hash_info.dir_info.trie.get(key)
-            if hash_info:
-                ret["size"] = hash_info.size
-                ret[hash_info.name] = hash_info.value
+            obj = out.obj.trie.get(key)
+            if obj:
+                ret["size"] = obj.size
+                ret[obj.hash_info.name] = obj.hash_info.value
 
         return ret

dvc/fs/fsspec_wrapper.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+from functools import lru_cache
 
 from dvc.progress import Tqdm
 
@@ -8,27 +9,37 @@ from .base import BaseFileSystem
 
 # pylint: disable=no-member
 class FSSpecWrapper(BaseFileSystem):
+    def __init__(self, repo, config):
+        super().__init__(repo, config)
+        self.fs_args = {"skip_instance_cache": True}
+        self.fs_args.update(self._prepare_credentials(config))
+
     def fs(self):
         raise NotImplementedError
 
+    @lru_cache(512)
     def _with_bucket(self, path):
         if isinstance(path, self.PATH_CLS):
             return f"{path.bucket}/{path.path}"
         return path
 
     def _strip_bucket(self, entry):
-        _, entry = entry.split("/", 1)
-        return entry
+        try:
+            bucket, path = entry.split("/", 1)
+        except ValueError:
+            # If there is no path attached, only returns
+            # the bucket (top-level).
+            bucket, path = entry, None
 
-    def _strip_buckets(self, entries, detail, prefix=None):
+        return path or bucket
+
+    def _strip_buckets(self, entries, detail=False):
         for entry in entries:
             if detail:
                 entry = self._entry_hook(entry.copy())
                 entry["name"] = self._strip_bucket(entry["name"])
             else:
-                entry = self._strip_bucket(
-                    f"{prefix}/{entry}" if prefix else entry
-                )
+                entry = self._strip_bucket(entry)
             yield entry
 
     def _entry_hook(self, entry):
@@ -36,11 +47,38 @@ class FSSpecWrapper(BaseFileSystem):
         entries within info() and ls(detail=True) calls"""
         return entry
 
+    def _prepare_credentials(self, config):  # pylint: disable=unused-argument
+        """Prepare the arguments for authentication to the
+        host filesystem"""
+        return {}
+
     def isdir(self, path_info):
-        return self.fs.isdir(self._with_bucket(path_info))
+        if not self.exists(path_info):
+            return False
+
+        # Directory in object storages are interpreted differently
+        # among different fsspec providers, so this logic is a temporary
+        # measure for us to adapt as of now. It checks whether it is a
+        # directory (as in a prefix with contents) or whether it is an empty
+        # file where it's name ends with a forward slash
+        entry = self.info(path_info)
+        return entry["type"] == "directory" or (
+            entry["size"] == 0
+            and entry["type"] == "file"
+            and entry["name"].endswith("/")
+        )
 
     def isfile(self, path_info):
-        return self.fs.isfile(self._with_bucket(path_info))
+        if not self.exists(path_info):
+            return False
+
+        return not self.isdir(path_info)
+
+    def is_empty(self, path_info):
+        entry = self.info(path_info)
+        if entry["type"] == "directory":
+            return not self.fs.ls(self._with_bucket(path_info))
+        return entry["size"] == 0
 
     def open(
         self, path_info, mode="r", **kwargs
@@ -53,21 +91,29 @@ class FSSpecWrapper(BaseFileSystem):
     def exists(self, path_info, use_dvcignore=False):
         return self.fs.exists(self._with_bucket(path_info))
 
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
+    def ls(self, path_info, detail=False):
         path = self._with_bucket(path_info)
-        if recursive:
-            for root, _, files in self.fs.walk(path, detail=detail):
-                if detail:
-                    files = files.values()
-                yield from self._strip_buckets(files, detail, prefix=root)
+        files = self.fs.ls(path, detail=detail)
+        yield from self._strip_buckets(files, detail=detail)
+
+    def find(self, path_info, detail=False):
+        path = self._with_bucket(path_info)
+        files = self.fs.find(path, detail=detail)
+        if detail:
+            files = files.values()
+
+        # When calling find() on a file, it returns the same file in a list.
+        # For object-based storages, the same behavior applies to empty
+        # directories since they are represented as files. This condition
+        # checks whether we should yield an empty list (if it is an empty
+        # directory) or just yield the file itself.
+        if len(files) == 1 and files[0] == path and self.isdir(path_info):
             return None
 
-        yield from self._strip_buckets(self.ls(path, detail=detail), detail)
+        yield from self._strip_buckets(files, detail=detail)
 
     def walk_files(self, path_info, **kwargs):
-        for file in self.ls(path_info, recursive=True):
+        for file in self.find(path_info):
             yield path_info.replace(path=file)
 
     def remove(self, path_info):

dvc/fs/gdrive.py

@@ -126,6 +126,9 @@ class GDriveFileSystem(BaseFileSystem):
         self._client_id = config.get("gdrive_client_id")
         self._client_secret = config.get("gdrive_client_secret")
         self._validate_config()
+        self._gdrive_service_credentials_path = tmp_fname(
+            os.path.join(self.repo.tmp_dir, "")
+        )
         self._gdrive_user_credentials_path = (
             tmp_fname(os.path.join(self.repo.tmp_dir, ""))
             if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
@@ -208,8 +211,15 @@ class GDriveFileSystem(BaseFileSystem):
         from pydrive2.auth import GoogleAuth
         from pydrive2.drive import GoogleDrive
 
-        if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
-            with open(self._gdrive_user_credentials_path, "w") as cred_file:
+        temporary_save_path = self._gdrive_user_credentials_path
+        is_credentials_temp = os.getenv(
+            GDriveFileSystem.GDRIVE_CREDENTIALS_DATA
+        )
+        if self._use_service_account:
+            temporary_save_path = self._gdrive_service_credentials_path
+
+        if is_credentials_temp:
+            with open(temporary_save_path, "w") as cred_file:
                 cred_file.write(
                     os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
                 )
@@ -217,6 +227,7 @@ class GDriveFileSystem(BaseFileSystem):
         auth_settings = {
             "client_config_backend": "settings",
             "save_credentials": True,
+            "save_credentials_backend": "file",
             "save_credentials_file": self._gdrive_user_credentials_path,
             "get_refresh_token": True,
             "oauth_scope": [
@@ -226,13 +237,16 @@ class GDriveFileSystem(BaseFileSystem):
         }
 
         if self._use_service_account:
-            auth_settings["save_credentials_backend"] = "json"
             auth_settings["service_config"] = {
                 "client_user_email": self._service_account_user_email,
                 "client_json_file_path": self._service_account_json_file_path,
             }
+            if is_credentials_temp:
+                auth_settings["service_config"][
+                    "client_json_file_path"
+                ] = temporary_save_path
+
         else:
-            auth_settings["save_credentials_backend"] = "file"
             auth_settings["client_config"] = {
                 "client_id": self._client_id or self.DEFAULT_GDRIVE_CLIENT_ID,
                 "client_secret": self._client_secret
@@ -267,8 +281,8 @@ class GDriveFileSystem(BaseFileSystem):
         except Exception as exc:
             raise GDriveAuthError(self.credentials_location) from exc
         finally:
-            if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
-                os.remove(self._gdrive_user_credentials_path)
+            if is_credentials_temp:
+                os.remove(temporary_save_path)
 
         return GoogleDrive(gauth)
 
@@ -518,7 +532,7 @@ class GDriveFileSystem(BaseFileSystem):
         query = f"({query}) and trashed=false"
         return self._gdrive_list(query)
 
-    def _ls_recursive(self, path_info, detail=False):
+    def find(self, path_info, detail=False):
         root_path = path_info.path
         seen_paths = set()
 
@@ -555,13 +569,7 @@ class GDriveFileSystem(BaseFileSystem):
                 else:
                     yield item_path
 
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
-        if recursive:
-            yield from self._ls_recursive(path_info, detail=detail)
-            return None
-
+    def ls(self, path_info, detail=False):
         cached = path_info.path in self._ids_cache["dirs"]
         if cached:
             dir_ids = self._ids_cache["dirs"][path_info.path]
@@ -591,8 +599,8 @@ class GDriveFileSystem(BaseFileSystem):
             self._cache_path_id(root_path, *dir_ids)
 
     def walk_files(self, path_info, **kwargs):
-        for filename in self.ls(path_info, recursive=True):
-            yield path_info.replace(path=filename)
+        for file in self.find(path_info):
+            yield path_info.replace(path=file)
 
     def remove(self, path_info):
         item_id = self._get_item_id(path_info)

dvc/fs/git.py

@@ -158,12 +158,6 @@ class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 errno.ENOENT, os.strerror(errno.ENOENT), path
             )
 
-    @property
-    def hash_jobs(self):  # pylint: disable=invalid-overridden-method
-        # NOTE: gitpython is not threadsafe. See
-        # https://github.com/iterative/dvc/issues/4079
-        return 1
-
     def walk_files(self, top):  # pylint: disable=arguments-differ
         for root, _, files in self.walk(top):
             for file in files:

dvc/fs/gs.py

@@ -22,16 +22,15 @@ class GSFileSystem(FSSpecWrapper):
         url = config.get("url", "gs://")
         self.path_info = self.PATH_CLS(url)
 
-        self.login_info = self._prepare_credentials(config)
-
     def _prepare_credentials(self, config):
-        login_info = {}
+        login_info = {"consistency": None}
         login_info["project"] = config.get("projectname")
         login_info["token"] = config.get("credentialpath")
         return login_info
 
     def _entry_hook(self, entry):
-        entry["etag"] = base64.b64decode(entry["etag"]).hex()
+        if "etag" in entry:
+            entry["etag"] = base64.b64decode(entry["etag"]).hex()
         return entry
 
     @wrap_prop(threading.Lock())
@@ -39,4 +38,4 @@ class GSFileSystem(FSSpecWrapper):
     def fs(self):
         from gcsfs import GCSFileSystem
 
-        return GCSFileSystem(**self.login_info)
+        return GCSFileSystem(**self.fs_args)

dvc/fs/hdfs.py

@@ -84,8 +84,9 @@ class HDFSFileSystem(BaseFileSystem):
             path=parsed.path,
         )
 
-    @staticmethod
-    def hdfs(path_info):
+        self.kerb_ticket = config.get("kerb_ticket")
+
+    def hdfs(self, path_info):
         import pyarrow.fs
 
         # NOTE: HadoopFileSystem is not meant to be closed by us and doesn't
@@ -100,6 +101,7 @@ class HDFSFileSystem(BaseFileSystem):
             path_info.host,
             path_info.port,
             user=path_info.user,
+            kerb_ticket=self.kerb_ticket,
         )
 
     @contextmanager

dvc/fs/http.py

@@ -1,6 +1,7 @@
 import logging
 import os.path
 import threading
+from typing import Optional
 
 from funcy import cached_property, memoize, wrap_prop, wrap_with
 
@@ -146,10 +147,9 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         raise HTTPError(res.status_code, res.reason)
 
     def info(self, path_info):
-        url = path_info.url
-        headers = self._head(url).headers
-        etag = headers.get("ETag") or headers.get("Content-MD5")
-        size = int(headers.get("Content-Length"))
+        resp = self._head(path_info.url)
+        etag = resp.headers.get("ETag") or resp.headers.get("Content-MD5")
+        size = self._content_length(resp)
         return {"etag": etag, "size": size}
 
     def _upload_fobj(self, fobj, to_info):
@@ -195,6 +195,6 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             )
 
     @staticmethod
-    def _content_length(response):
+    def _content_length(response) -> Optional[int]:
         res = response.headers.get("Content-Length")
         return int(res) if res else None

dvc/fs/memory.py

@@ -13,13 +13,16 @@ class MemoryFileSystem(BaseFileSystem):
         self.fs = MemFS()
 
     def exists(self, path_info, use_dvcignore=True):
-        return self.fs.exists(path_info.path)
+        return self.fs.exists(path_info.fspath)
 
     def open(self, path_info, mode="r", encoding=None, **kwargs):
         return self.fs.open(
             path_info.fspath, mode=mode, encoding=encoding, **kwargs
         )
 
+    def info(self, path_info):
+        return self.fs.info(path_info.fspath)
+
     def stat(self, path_info):
         import os
 

dvc/fs/s3.py

@@ -38,6 +38,7 @@ class S3FileSystem(BaseFileSystem):
         self.endpoint_url = config.get("endpointurl")
 
         self.use_ssl = config.get("use_ssl", True)
+        self.ssl_verify = config.get("ssl_verify", True)
 
         self.extra_args = {}
 
@@ -136,6 +137,7 @@ class S3FileSystem(BaseFileSystem):
         return session.resource(
             "s3",
             endpoint_url=self.endpoint_url,
+            verify=self.ssl_verify,
             use_ssl=self.use_ssl,
             config=boto3.session.Config(
                 signature_version="s3v4", s3=s3_config
@@ -261,8 +263,13 @@ class S3FileSystem(BaseFileSystem):
         return path_info.path in self._list_paths(path_info)
 
     def info(self, path_info):
+        # temporary workaround, will be removed when migrating to s3fs
+        if self.isdir(path_info):
+            return {"type": "directory"}
+
         with self._get_obj(path_info) as obj:
             return {
+                "type": "file",
                 "etag": obj.e_tag.strip('"'),
                 "size": obj.content_length,
             }
@@ -286,11 +293,7 @@ class S3FileSystem(BaseFileSystem):
 
             yield path_info.replace(path=fname)
 
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
-        assert recursive
-
+    def find(self, path_info, detail=False):
         with self._get_bucket(path_info.bucket) as bucket:
             for obj_summary in bucket.objects.filter(Prefix=path_info.path):
                 if detail:

dvc/fs/ssh/__init__.py

@@ -33,7 +33,7 @@ def ask_password(host, user, port):
 class SSHFileSystem(BaseFileSystem):
     scheme = Schemes.SSH
     REQUIRES = {"paramiko": "paramiko"}
-    JOBS = 4
+    _JOBS = 4
 
     PARAM_CHECKSUM = "md5"
     # At any given time some of the connections will go over network and

dvc/fs/ssh/connection.py

@@ -17,7 +17,7 @@ from ..base import RemoteCmdError
 try:
     import paramiko
 except ImportError:
-    paramiko = None
+    paramiko = None  # type: ignore
 
 
 logger = logging.getLogger(__name__)
@@ -40,7 +40,9 @@ class SSHConnection:
                 host=host, **kwargs
             )
         )
-        self.timeout = kwargs.get("timeout", 1800)
+
+        kwargs.setdefault("timeout", 1800)
+        self.timeout = kwargs["timeout"]
 
         self._ssh = paramiko.SSHClient()
 

dvc/fs/webdav.py

@@ -117,7 +117,7 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             )
 
         # Check whether connection is valid (root should always exist)
-        if not client.check("/"):
+        if not client.check(self.path_info.path):
             raise WebDAVConnectionError(self.hostname)
 
         return client
@@ -151,33 +151,30 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         # Use webdav is_dir to test whether path points to a directory
         return self._client.is_dir(path_info.path)
 
-    # Yields path info to all files
     def walk_files(self, path_info, **kwargs):
-        # Check whether directory exists
         if not self.exists(path_info):
             return
 
-        # Collect directories
-        dirs = deque([path_info.path])
+        for file in self.find(path_info):
+            yield path_info.replace(path=file)
 
-        # Iterate all directories found so far
-        while dirs:
-            # Iterate directory content
-            for entry in self._client.list(dirs.pop(), get_info=True):
-                # Construct path_info to entry
-                info = path_info.replace(path=entry["path"])
-
-                # Check whether entry is a directory
+    def ls(self, path_info, detail=False):
+        for entry in self._client.list(path_info.path):
+            path = entry["path"]
+            if detail:
                 if entry["isdir"]:
-                    # Append new found directory to directory list
-                    dirs.append(info.path)
+                    yield {"type": "directory", "name": path}
                 else:
-                    # Yield path info to non directory
-                    yield info
+                    yield {
+                        "type": "file",
+                        "name": path,
+                        "size": entry["size"],
+                        "etag": entry["etag"],
+                    }
+            else:
+                yield path
 
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
+    def find(self, path_info, detail=False):
         dirs = deque([path_info.path])
 
         while dirs:
@@ -197,14 +194,6 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 else:
                     yield path
 
-            if not recursive:
-                for entry in dirs:
-                    if detail:
-                        yield {"type": "directory", "name": entry}
-                    else:
-                        yield entry
-                return None
-
     # Removes file/directory
     def remove(self, path_info):
         # Use webdav client clean (DELETE) method to remove file/directory
@@ -213,7 +202,7 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     # Creates directories
     def makedirs(self, path_info):
         # Terminate recursion
-        if path_info.path == "/" or self.exists(path_info):
+        if path_info.path == self.path_info.path or self.exists(path_info):
             return
 
         # Recursively descent to root

dvc/hash_info.py

@@ -1,11 +1,9 @@
 from collections import OrderedDict
 from dataclasses import dataclass, field
-from typing import Dict, Optional
+from typing import Optional
 
 HASH_DIR_SUFFIX = ".dir"
 
-DirInfo = Dict[str, str]
-
 
 @dataclass
 class HashInfo:
@@ -14,7 +12,6 @@ class HashInfo:
 
     name: Optional[str]
     value: Optional[str]
-    dir_info: Optional[DirInfo] = field(default=None, compare=False)
     size: Optional[int] = field(default=None, compare=False)
     nfiles: Optional[int] = field(default=None, compare=False)
 

dvc/ignore.py

@@ -183,7 +183,12 @@ class DvcIgnoreFilter:
     def __init__(self, fs, root_dir):
         from dvc.repo import Repo
 
-        default_ignore_patterns = [".hg/", ".git/", "{}/".format(Repo.DVC_DIR)]
+        default_ignore_patterns = [
+            ".hg/",
+            ".git/",
+            ".git",
+            "{}/".format(Repo.DVC_DIR),
+        ]
 
         self.fs = fs
         self.root_dir = root_dir

dvc/logger.py

@@ -23,14 +23,19 @@ def addLoggingLevel(levelName, levelNum, methodName=None):
     Adds a new logging level to the `logging` module and the
     currently configured logging class.
 
+    Uses the existing numeric levelNum if already defined.
+
     Based on https://stackoverflow.com/questions/2183233
     """
     if methodName is None:
         methodName = levelName.lower()
 
-    assert not hasattr(logging, levelName)
-    assert not hasattr(logging, methodName)
-    assert not hasattr(logging.getLoggerClass(), methodName)
+    # If the level name is already defined as a top-level `logging`
+    # constant, then adopt the existing numeric level.
+    if hasattr(logging, levelName):
+        existingLevelNum = getattr(logging, levelName)
+        assert isinstance(existingLevelNum, int)
+        levelNum = existingLevelNum
 
     def logForLevel(self, message, *args, **kwargs):
         if self.isEnabledFor(levelNum):
@@ -40,10 +45,19 @@ def addLoggingLevel(levelName, levelNum, methodName=None):
     def logToRoot(message, *args, **kwargs):
         logging.log(levelNum, message, *args, **kwargs)
 
-    logging.addLevelName(levelNum, levelName)
-    setattr(logging, levelName, levelNum)
-    setattr(logging.getLoggerClass(), methodName, logForLevel)
-    setattr(logging, methodName, logToRoot)
+    # getLevelName resolves the numeric log level if already defined,
+    # otherwise returns a string
+    if not isinstance(logging.getLevelName(levelName), int):
+        logging.addLevelName(levelNum, levelName)
+
+    if not hasattr(logging, levelName):
+        setattr(logging, levelName, levelNum)
+
+    if not hasattr(logging.getLoggerClass(), methodName):
+        setattr(logging.getLoggerClass(), methodName, logForLevel)
+
+    if not hasattr(logging, methodName):
+        setattr(logging, methodName, logToRoot)
 
 
 class LoggingException(Exception):

dvc/main.py

@@ -45,6 +45,11 @@ def main(argv=None):  # noqa: C901
 
         logger.trace(args)
 
+        if not args.quiet:
+            from dvc.ui import ui
+
+            ui.enable()
+
         with debugtools(args):
             cmd = args.func(args)
             ret = cmd.run()

dvc/objects/__init__.py

@@ -1,315 +1,45 @@
-import itertools
-import json
 import logging
-from concurrent import futures
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
-from dvc.dir_info import DirInfo
-from dvc.exceptions import DvcException
 from dvc.progress import Tqdm
 
-from .stage import get_hash
+from .tree import Tree
 
 logger = logging.getLogger(__name__)
 
 
-class ObjectError(DvcException):
-    pass
+def save(odb, obj, jobs=None, **kwargs):
+    if isinstance(obj, Tree):
+        with ThreadPoolExecutor(max_workers=jobs) as executor:
+            for future in Tqdm(
+                as_completed(
+                    executor.submit(
+                        odb.add,
+                        entry.path_info,
+                        entry.fs,
+                        entry.hash_info,
+                        **kwargs
+                    )
+                    for _, entry in obj
+                ),
+                desc="Saving files",
+                total=len(obj),
+                unit="file",
+            ):
+                future.result()
 
-
-class ObjectFormatError(ObjectError):
-    pass
-
-
-class HashFile:
-    def __init__(self, path_info, fs, hash_info):
-        self.path_info = path_info
-        self.fs = fs
-        self.hash_info = hash_info
-
-    def __len__(self):
-        return 1
-
-    def __str__(self):
-        return f"object {self.hash_info}"
-
-    def __bool__(self):
-        return bool(self.hash_info)
-
-    def check(self, odb):
-        actual = get_hash(self.path_info, self.fs, odb.fs.PARAM_CHECKSUM, odb)
-
-        logger.trace(
-            "cache '%s' expected '%s' actual '%s'",
-            self.path_info,
-            self.hash_info,
-            actual,
-        )
-
-        assert actual.name == self.hash_info.name
-        if actual.value.split(".")[0] != self.hash_info.value.split(".")[0]:
-            raise ObjectFormatError(f"{self} is corrupted")
-
-    @classmethod
-    def load(cls, odb, hash_info):
-        return odb.get(hash_info)
-
-    def save(self, odb, **kwargs):
-        odb.add(self.path_info, self.fs, self.hash_info, **kwargs)
-
-
-class File(HashFile):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.src = None
-
-    @classmethod
-    def stage(cls, odb, path_info, fs, **kwargs):
-        hash_info = get_hash(
-            path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs
-        )
-        raw = odb.get(hash_info)
-        obj = cls(raw.path_info, raw.fs, hash_info)
-        obj.src = HashFile(path_info, fs, hash_info)
-        return obj
-
-    def save(self, odb, **kwargs):
-        self.src.save(odb, **kwargs)
-
-
-class Tree(HashFile):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.src_hash_info = None
-        self.src_path_info = None
-        self.src_fs = None
-
-    def __len__(self):
-        return self.hash_info.dir_info.nfiles
-
-    def __iter__(self):
-        yield from self.hash_info.dir_info.items()
-
-    @classmethod
-    def save_dir_info(cls, odb, dir_info, hash_info=None):
-        if hash_info and hash_info.name == odb.fs.PARAM_CHECKSUM:
-            try:
-                odb.check(hash_info)
-                assert hash_info.dir_info == dir_info
-                return hash_info
-            except (FileNotFoundError, ObjectFormatError):
-                pass
-
-        from dvc.fs.memory import MemoryFileSystem
-        from dvc.path_info import PathInfo
-        from dvc.utils import tmp_fname
-
-        fs = MemoryFileSystem(None, {})
-        path_info = PathInfo(tmp_fname(""))
-        with fs.open(path_info, "w") as fobj:
-            json.dump(dir_info.to_list(), fobj, sort_keys=True)
-
-        tmp_info = odb.fs.path_info / tmp_fname("")
-        with fs.open(path_info, "rb") as fobj:
-            odb.fs.upload_fobj(fobj, tmp_info)
-
-        hash_info = get_hash(tmp_info, odb.fs, odb.fs.PARAM_CHECKSUM, odb)
-        hash_info.value += odb.fs.CHECKSUM_DIR_SUFFIX
-        hash_info.dir_info = dir_info
-        hash_info.nfiles = dir_info.nfiles
-
-        odb.add(tmp_info, odb.fs, hash_info)
-
-        return hash_info
-
-    @classmethod
-    def stage(cls, odb, path_info, fs, **kwargs):
-        hash_info = get_hash(
-            path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs
-        )
-        hi = cls.save_dir_info(odb, hash_info.dir_info, hash_info)
-        hi.size = hash_info.size
-        raw = odb.get(hi)
-        obj = cls(raw.path_info, raw.fs, hi)
-        obj.src_hash_info = hash_info
-        obj.src_path_info = path_info
-        obj.src_fs = fs
-        return obj
-
-    @classmethod
-    def load(cls, odb, hash_info):
-
-        obj = odb.get(hash_info)
-
-        try:
-            with obj.fs.open(obj.path_info, "r") as fobj:
-                raw = json.load(fobj)
-        except ValueError as exc:
-            raise ObjectFormatError(f"{obj} is corrupted") from exc
-
-        if not isinstance(raw, list):
-            logger.error(
-                "dir cache file format error '%s' [skipping the file]",
-                obj.path_info,
-            )
-            raise ObjectFormatError(f"{obj} is corrupted")
-
-        dir_info = DirInfo.from_list(raw)
-        hash_info.dir_info = dir_info
-        hash_info.nfiles = dir_info.nfiles
-
-        return cls(obj.path_info, obj.fs, hash_info)
-
-    def save(self, odb, **kwargs):
-        assert self.src_hash_info.dir_info
-        hi = self.save_dir_info(
-            odb, self.src_hash_info.dir_info, self.hash_info
-        )
-        for entry_key, entry_hash in Tqdm(
-            hi.dir_info.items(),
-            desc="Saving " + self.src_path_info.name,
-            unit="file",
-        ):
-            entry_info = self.src_path_info.joinpath(*entry_key)
-            entry_obj = HashFile(entry_info, self.src_fs, entry_hash)
-            entry_obj.save(odb, **kwargs)
-        self.src_fs.repo.state.save(self.src_path_info, self.src_fs, hi)
-
-    def filter(self, odb, prefix):
-        hash_info = self.hash_info.dir_info.get(prefix)
-        if hash_info:
-            return load(odb, hash_info)
-
-        depth = len(prefix)
-        dir_info = DirInfo()
-        try:
-            for key, value in self.hash_info.dir_info.trie.items(prefix):
-                dir_info.add(key[depth:], value)
-        except KeyError:
-            return None
-
-        return load(odb, self.save_dir_info(odb, dir_info))
-
-
-def save(odb, obj, **kwargs):
-    obj.save(odb, **kwargs)
+    odb.add(obj.path_info, obj.fs, obj.hash_info, **kwargs)
 
 
 def check(odb, obj):
     odb.check(obj.hash_info)
 
     if isinstance(obj, Tree):
-        for _, hash_info in obj:
-            odb.check(hash_info)
+        for _, entry in obj:
+            odb.check(entry.hash_info)
 
 
 def load(odb, hash_info):
     if hash_info.isdir:
         return Tree.load(odb, hash_info)
-    return File.load(odb, hash_info)
-
-
-def _get_dir_size(odb, dir_info):
-    try:
-        return sum(
-            odb.fs.getsize(odb.hash_to_path_info(hi.value))
-            for _, hi in dir_info.items()
-        )
-    except FileNotFoundError:
-        return None
-
-
-def merge(odb, ancestor_info, our_info, their_info):
-    assert our_info
-    assert their_info
-
-    if ancestor_info:
-        ancestor = load(odb, ancestor_info).hash_info.dir_info
-    else:
-        ancestor = DirInfo()
-
-    our = load(odb, our_info).hash_info.dir_info
-    their = load(odb, their_info).hash_info.dir_info
-
-    merged = our.merge(ancestor, their)
-    hash_info = Tree.save_dir_info(odb, merged)
-    hash_info.size = _get_dir_size(odb, merged)
-    return hash_info
-
-
-def _transfer_file(odb, from_fs, from_info):
-    from dvc.utils import tmp_fname
-    from dvc.utils.stream import HashedStreamReader
-
-    tmp_info = odb.fs.path_info / tmp_fname()
-    with from_fs.open(
-        from_info, mode="rb", chunk_size=from_fs.CHUNK_SIZE
-    ) as stream:
-        stream_reader = HashedStreamReader(stream)
-        # Since we don't know the hash beforehand, we'll
-        # upload it to a temporary location and then move
-        # it.
-        odb.fs.upload_fobj(
-            stream_reader,
-            tmp_info,
-            total=from_fs.getsize(from_info),
-            desc=from_info.name,
-        )
-
-    hash_info = stream_reader.hash_info
-    return tmp_info, hash_info
-
-
-def _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
-    rel_path_infos = {}
-    from_infos = from_fs.walk_files(from_info)
-
-    def create_tasks(executor, amount):
-        for entry_info in itertools.islice(from_infos, amount):
-            pbar.total += 1
-            task = executor.submit(
-                pbar.wrap_fn(_transfer_file), odb, from_fs, entry_info
-            )
-            rel_path_infos[task] = entry_info.relative_to(from_info)
-            yield task
-
-    pbar.total = 0
-    with futures.ThreadPoolExecutor(max_workers=jobs) as executor:
-        tasks = set(create_tasks(executor, jobs * 5))
-
-        while tasks:
-            done, tasks = futures.wait(
-                tasks, return_when=futures.FIRST_COMPLETED
-            )
-            tasks.update(create_tasks(executor, len(done)))
-            for task in done:
-                yield rel_path_infos.pop(task), task.result()
-
-
-def _transfer_directory(odb, from_fs, from_info, jobs, no_progress_bar=False):
-    dir_info = DirInfo()
-
-    with Tqdm(total=1, unit="Files", disable=no_progress_bar) as pbar:
-        for (
-            entry_info,
-            (entry_tmp_info, entry_hash),
-        ) in _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
-            odb.add(entry_tmp_info, odb.fs, entry_hash)
-            dir_info.add(entry_info.parts, entry_hash)
-
-    return Tree.save_dir_info(odb, dir_info)
-
-
-def transfer(odb, from_fs, from_info, jobs=None, no_progress_bar=False):
-    jobs = jobs or min((from_fs.jobs, odb.fs.jobs))
-
-    if from_fs.isdir(from_info):
-        return _transfer_directory(
-            odb,
-            from_fs,
-            from_info,
-            jobs=jobs,
-            no_progress_bar=no_progress_bar,
-        )
-    tmp_info, hash_info = _transfer_file(odb, from_fs, from_info)
-    odb.add(tmp_info, odb.fs, hash_info)
-    return hash_info
+    return odb.get(hash_info)

dvc/objects/db/base.py

@@ -1,13 +1,14 @@
 import itertools
 import logging
+import os
 from concurrent.futures import ThreadPoolExecutor
 from copy import copy
 from typing import Optional
 
+from dvc.objects.errors import ObjectFormatError
+from dvc.objects.file import HashFile
 from dvc.progress import Tqdm
 
-from .. import HashFile, ObjectFormatError
-
 logger = logging.getLogger(__name__)
 
 
@@ -42,9 +43,9 @@ class ObjectDB:
             hash_info,
         )
 
-    def add(self, path_info, fs, hash_info, **kwargs):
+    def add(self, path_info, fs, hash_info, move=True, **kwargs):
         try:
-            self.check(hash_info)
+            self.check(hash_info, check_hash=self.verify)
             return
         except (ObjectFormatError, FileNotFoundError):
             pass
@@ -52,11 +53,32 @@ class ObjectDB:
         cache_info = self.hash_to_path_info(hash_info.value)
         # using our makedirs to create dirs with proper permissions
         self.makedirs(cache_info.parent)
-        if isinstance(fs, type(self.fs)):
-            self.fs.move(path_info, cache_info)
-        else:
-            with fs.open(path_info, mode="rb") as fobj:
-                self.fs.upload_fobj(fobj, cache_info)
+        use_move = isinstance(fs, type(self.fs)) and move
+        try:
+            if use_move:
+                self.fs.move(path_info, cache_info)
+            else:
+                with fs.open(path_info, mode="rb") as fobj:
+                    self.fs.upload_fobj(fobj, cache_info)
+        except OSError as exc:
+            # If the target file already exists, we are going to simply
+            # ignore the exception (#4992).
+            #
+            # On Windows, it is not always guaranteed that you'll get
+            # FileExistsError (it might be PermissionError or a bare OSError)
+            # but all of those exceptions raised from the original
+            # FileExistsError so we have a separate check for that.
+            if isinstance(exc, FileExistsError) or (
+                os.name == "nt"
+                and exc.__context__
+                and isinstance(exc.__context__, FileExistsError)
+            ):
+                logger.debug("'%s' file already exists, skipping", path_info)
+                if use_move:
+                    fs.remove(path_info)
+            else:
+                raise
+
         self.protect(cache_info)
         self.fs.repo.state.save(cache_info, self.fs, hash_info)
 
@@ -84,8 +106,10 @@ class ObjectDB:
     def set_exec(self, path_info):  # pylint: disable=unused-argument
         pass
 
-    def check(self, hash_info):
-        """Compare the given hash with the (corresponding) actual one.
+    def check(self, hash_info, check_hash=True):
+        """Compare the given hash with the (corresponding) actual one if
+        check_hash is specified, or just verify the existence of the cache
+        files on the filesystem.
 
         - Use `State` as a cache for computed hashes
             + The entries are invalidated by taking into account the following:
@@ -106,7 +130,7 @@ class ObjectDB:
             return
 
         try:
-            obj.check(self)
+            obj.check(self, check_hash=check_hash)
         except ObjectFormatError:
             logger.warning("corrupted cache file '%s'.", obj.path_info)
             self.fs.remove(obj.path_info)
@@ -229,7 +253,7 @@ class ObjectDB:
         keeping all of it in memory.
         """
         num_pages = remote_size / self.fs.LIST_OBJECT_PAGE_SIZE
-        if num_pages < 256 / self.fs.JOBS:
+        if num_pages < 256 / self.fs.jobs:
             # Fetching prefixes in parallel requires at least 255 more
             # requests, for small enough remotes it will be faster to fetch
             # entire cache without splitting it into prefixes.
@@ -263,7 +287,7 @@ class ObjectDB:
                 )
 
             with ThreadPoolExecutor(
-                max_workers=jobs or self.fs.JOBS
+                max_workers=jobs or self.fs.jobs
             ) as executor:
                 in_remote = executor.map(list_with_update, traverse_prefixes,)
                 yield from itertools.chain.from_iterable(in_remote)
@@ -331,7 +355,7 @@ class ObjectDB:
                 return ret
 
             with ThreadPoolExecutor(
-                max_workers=jobs or self.fs.JOBS
+                max_workers=jobs or self.fs.jobs
             ) as executor:
                 path_infos = map(self.hash_to_path_info, hashes)
                 in_remote = executor.map(exists_with_progress, path_infos)

dvc/objects/db/local.py

@@ -6,7 +6,7 @@ from funcy import cached_property
 from shortuuid import uuid
 
 from dvc.hash_info import HashInfo
-from dvc.objects import ObjectFormatError
+from dvc.objects.errors import ObjectFormatError
 from dvc.path_info import PathInfo
 from dvc.progress import Tqdm
 from dvc.utils import relpath

dvc/objects/db/ssh.py

@@ -64,10 +64,10 @@ class SSHObjectDB(ObjectDB):
                 return self.batch_exists(chunks, callback=pbar.update_msg)
 
             with ThreadPoolExecutor(
-                max_workers=jobs or self.fs.JOBS
+                max_workers=jobs or self.fs.jobs
             ) as executor:
                 path_infos = [self.hash_to_path_info(x) for x in hashes]
-                chunks = to_chunks(path_infos, num_chunks=self.fs.JOBS)
+                chunks = to_chunks(path_infos, num_chunks=self.fs.jobs)
                 results = executor.map(exists_with_progress, chunks)
                 in_remote = itertools.chain.from_iterable(results)
                 ret = list(itertools.compress(hashes, in_remote))

dvc/objects/errors.py

@@ -0,0 +1,9 @@
+from dvc.exceptions import DvcException
+
+
+class ObjectError(DvcException):
+    pass
+
+
+class ObjectFormatError(ObjectError):
+    pass

dvc/objects/file.py

@@ -0,0 +1,64 @@
+import errno
+import logging
+import os
+
+from .errors import ObjectFormatError
+
+logger = logging.getLogger(__name__)
+
+
+class HashFile:
+    def __init__(self, path_info, fs, hash_info):
+        self.path_info = path_info
+        self.fs = fs
+        self.hash_info = hash_info
+
+    @property
+    def size(self):
+        if not (self.path_info and self.fs):
+            return None
+        return self.fs.getsize(self.path_info)
+
+    def __len__(self):
+        return 1
+
+    def __str__(self):
+        return f"object {self.hash_info}"
+
+    def __bool__(self):
+        return bool(self.hash_info)
+
+    def __eq__(self, other):
+        if not isinstance(other, HashFile):
+            return False
+        return (
+            self.path_info == other.path_info
+            and self.fs == other.fs
+            and self.hash_info == other.hash_info
+        )
+
+    def check(self, odb, check_hash=True):
+        from .stage import get_file_hash
+
+        if not check_hash:
+            if not self.fs.exists(self.path_info):
+                raise FileNotFoundError(
+                    errno.ENOENT, os.strerror(errno.ENOENT), self.path_info
+                )
+            else:
+                return None
+
+        actual = get_file_hash(
+            self.path_info, self.fs, self.hash_info.name, odb.repo.state
+        )
+
+        logger.trace(
+            "cache '%s' expected '%s' actual '%s'",
+            self.path_info,
+            self.hash_info,
+            actual,
+        )
+
+        assert actual.name == self.hash_info.name
+        if actual.value.split(".")[0] != self.hash_info.value.split(".")[0]:
+            raise ObjectFormatError(f"{self} is corrupted")

dvc/objects/stage.py

@@ -1,15 +1,32 @@
 import errno
 import os
 from concurrent.futures import ThreadPoolExecutor
+from functools import partial
 
 from dvc.exceptions import DvcIgnoreInCollectedDirError
 from dvc.hash_info import HashInfo
 from dvc.ignore import DvcIgnore
+from dvc.objects.file import HashFile
 from dvc.progress import Tqdm
 from dvc.utils import file_md5
 
 
-def get_file_hash(path_info, fs, name):
+def _upload_file(path_info, fs, odb):
+    from dvc.utils import tmp_fname
+    from dvc.utils.stream import HashedStreamReader
+
+    tmp_info = odb.fs.path_info / tmp_fname()
+    with fs.open(path_info, mode="rb", chunk_size=fs.CHUNK_SIZE) as stream:
+        stream = HashedStreamReader(stream)
+        odb.fs.upload_fobj(
+            stream, tmp_info, desc=path_info.name, total=fs.getsize(path_info)
+        )
+
+    obj = HashFile(tmp_info, odb.fs, stream.hash_info)
+    return path_info, obj
+
+
+def _get_file_hash(path_info, fs, name):
     info = fs.info(path_info)
     if name in info:
         assert not info[name].endswith(".dir")
@@ -27,46 +44,86 @@ def get_file_hash(path_info, fs, name):
     raise NotImplementedError
 
 
-def _calculate_hashes(path_info, fs, name, state, **kwargs):
-    def _get_file_hash(file_info):
+def get_file_hash(path_info, fs, name, state=None):
+    if state:
         hash_info = state.get(  # pylint: disable=assignment-from-none
-            file_info, fs,
+            path_info, fs,
         )
-        if not hash_info:
-            hash_info = get_file_hash(file_info, fs, name)
-            state.save(file_info, fs, hash_info)
-        return file_info, hash_info
+        if hash_info:
+            return hash_info
 
+    if not fs.exists(path_info):
+        raise FileNotFoundError(
+            errno.ENOENT, os.strerror(errno.ENOENT), path_info
+        )
+
+    hash_info = _get_file_hash(path_info, fs, name)
+
+    if state:
+        assert ".dir" not in hash_info.value
+        state.save(path_info, fs, hash_info)
+
+    return hash_info
+
+
+def _get_file_obj(path_info, fs, name, odb=None, state=None, upload=False):
+    if upload:
+        assert odb and name == "md5"
+        return _upload_file(path_info, fs, odb)
+
+    obj = HashFile(
+        path_info, fs, get_file_hash(path_info, fs, name, state=state)
+    )
+    return path_info, obj
+
+
+def _build_objects(path_info, fs, name, odb, state, upload, **kwargs):
     with Tqdm(
-        unit="md5", desc="Computing file/dir hashes (only done once)",
+        unit="md5",
+        desc="Computing file/dir hashes (only done once)",
+        disable=kwargs.pop("no_progress_bar", False),
     ) as pbar:
-        worker = pbar.wrap_fn(_get_file_hash)
-        with ThreadPoolExecutor(max_workers=fs.hash_jobs) as executor:
-            pairs = executor.map(worker, fs.walk_files(path_info, **kwargs))
-            return dict(pairs)
+        worker = pbar.wrap_fn(
+            partial(
+                _get_file_obj,
+                fs=fs,
+                name=name,
+                odb=odb,
+                state=state,
+                upload=upload,
+            )
+        )
+        with ThreadPoolExecutor(
+            max_workers=kwargs.pop("jobs", fs.hash_jobs)
+        ) as executor:
+            yield from executor.map(worker, fs.walk_files(path_info, **kwargs))
 
 
-def _iter_hashes(path_info, fs, name, state, **kwargs):
-    if name in fs.DETAIL_FIELDS:
-        for details in fs.ls(path_info, recursive=True, detail=True):
+def _iter_objects(path_info, fs, name, odb, state, upload, **kwargs):
+    if not upload and name in fs.DETAIL_FIELDS:
+        for details in fs.find(path_info, detail=True):
             file_info = path_info.replace(path=details["name"])
             hash_info = HashInfo(
                 name, details[name], size=details.get("size"),
             )
-            yield file_info, hash_info
+            yield file_info, HashFile(file_info, fs, hash_info)
 
         return None
 
-    yield from _calculate_hashes(path_info, fs, name, state, **kwargs).items()
+    yield from _build_objects(
+        path_info, fs, name, odb, state, upload, **kwargs
+    )
 
 
-def _collect_dir(path_info, fs, name, state, **kwargs):
-    from dvc.dir_info import DirInfo
+def _build_tree(path_info, fs, name, odb, state, upload, **kwargs):
+    from .tree import Tree
 
-    dir_info = DirInfo()
-    for fi, hi in _iter_hashes(path_info, fs, name, state, **kwargs):
-        if DvcIgnore.DVCIGNORE_FILE == fi.name:
-            raise DvcIgnoreInCollectedDirError(fi.parent)
+    tree = Tree(None, None, None)
+    for file_info, obj in _iter_objects(
+        path_info, fs, name, odb, state, upload, **kwargs
+    ):
+        if DvcIgnore.DVCIGNORE_FILE == file_info.name:
+            raise DvcIgnoreInCollectedDirError(file_info.parent)
 
         # NOTE: this is lossy transformation:
         #   "hey\there" -> "hey/there"
@@ -76,22 +133,44 @@ def _collect_dir(path_info, fs, name, state, **kwargs):
         #
         # Yes, this is a BUG, as long as we permit "/" in
         # filenames on Windows and "\" on Unix
-        dir_info.add(fi.relative_to(path_info).parts, hi)
+        tree.add(file_info.relative_to(path_info).parts, obj)
 
-    return dir_info
+    tree.digest()
+
+    return tree
 
 
-def get_dir_hash(path_info, fs, name, state, **kwargs):
-    from . import Tree
+def _get_tree_obj(path_info, fs, name, odb, state, upload, **kwargs):
+    from .tree import Tree
 
-    dir_info = _collect_dir(path_info, fs, name, state, **kwargs)
-    hash_info = Tree.save_dir_info(fs.repo.odb.local, dir_info)
-    hash_info.size = dir_info.size
-    hash_info.dir_info = dir_info
-    return hash_info
+    value = fs.info(path_info).get(name)
+    if value:
+        hash_info = HashInfo(name, value)
+        try:
+            return Tree.load(odb, hash_info)
+        except FileNotFoundError:
+            pass
+
+    tree = _build_tree(path_info, fs, name, odb, state, upload, **kwargs)
+
+    odb.add(tree.path_info, tree.fs, tree.hash_info)
+    if name != "md5":
+        # NOTE: used only for external outputs. Initial reasoning was to be
+        # able to validate .dir files right in the workspace (e.g. check s3
+        # etag), but could be dropped for manual validation with regular md5,
+        # that would be universal for all clouds.
+        raw = odb.get(tree.hash_info)
+        hash_info = get_file_hash(raw.path_info, raw.fs, name, state)
+        tree.hash_info.name = hash_info.name
+        tree.hash_info.value = hash_info.value
+        if not tree.hash_info.value.endswith(".dir"):
+            tree.hash_info.value += ".dir"
+        odb.add(tree.path_info, tree.fs, tree.hash_info)
+
+    return tree
 
 
-def get_hash(path_info, fs, name, odb, **kwargs):
+def stage(odb, path_info, fs, name, upload=False, **kwargs):
     assert path_info and (
         isinstance(path_info, str) or path_info.scheme == fs.scheme
     )
@@ -116,28 +195,28 @@ def get_hash(path_info, fs, name, odb, **kwargs):
         hash_info = None
 
     if hash_info:
-        if hash_info.isdir:
-            from . import Tree
+        from . import load
+        from .tree import Tree
 
-            # NOTE: loading the fs will restore hash_info.dir_info
-            Tree.load(odb, hash_info)
-        assert hash_info.name == name
-        return hash_info
+        obj = load(odb, hash_info)
+        if isinstance(obj, Tree):
+            obj.hash_info.nfiles = len(obj)
+            for key, entry in obj:
+                entry.fs = fs
+                entry.path_info = path_info.joinpath(*key)
+        else:
+            obj.fs = fs
+            obj.path_info = path_info
+        assert obj.hash_info.name == name
+        obj.hash_info.size = hash_info.size
+        return obj
 
     if fs.isdir(path_info):
-        hash_info = get_dir_hash(path_info, fs, name, state, **kwargs)
+        obj = _get_tree_obj(path_info, fs, name, odb, state, upload, **kwargs)
     else:
-        hash_info = get_file_hash(path_info, fs, name)
+        _, obj = _get_file_obj(path_info, fs, name, odb, state, upload)
 
-    if hash_info and fs.exists(path_info):
-        state.save(path_info, fs, hash_info)
+    if obj.hash_info and fs.exists(path_info):
+        state.save(path_info, fs, obj.hash_info)
 
-    return hash_info
-
-
-def stage(odb, path_info, fs, **kwargs):
-    from . import File, Tree
-
-    if fs.isdir(path_info):
-        return Tree.stage(odb, path_info, fs, **kwargs)
-    return File.stage(odb, path_info, fs, **kwargs)
+    return obj

dvc/objects/tree.py

@@ -0,0 +1,211 @@
+import json
+import logging
+import posixpath
+
+from funcy import cached_property
+
+from .errors import ObjectFormatError
+from .file import HashFile
+from .stage import get_file_hash
+
+logger = logging.getLogger(__name__)
+
+
+class Tree(HashFile):
+    PARAM_RELPATH = "relpath"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._dict = {}
+
+    @cached_property
+    def trie(self):
+        from pygtrie import Trie
+
+        return Trie(self._dict)
+
+    @property
+    def size(self):
+        try:
+            return sum(obj.size for _, obj in self)
+        except TypeError:
+            return None
+
+    def add(self, key, obj):
+        self.__dict__.pop("trie", None)
+        self._dict[key] = obj
+
+    def digest(self):
+        from dvc.fs.memory import MemoryFileSystem
+        from dvc.path_info import PathInfo
+        from dvc.utils import tmp_fname
+
+        memfs = MemoryFileSystem(None, {})
+        path_info = PathInfo(tmp_fname(""))
+        with memfs.open(path_info, "wb") as fobj:
+            fobj.write(self.as_bytes())
+        self.fs = memfs
+        self.path_info = path_info
+        self.hash_info = get_file_hash(path_info, memfs, "md5")
+        self.hash_info.value += ".dir"
+        self.hash_info.size = self.size
+        self.hash_info.nfiles = len(self)
+
+    def __len__(self):
+        return len(self._dict)
+
+    def __iter__(self):
+        yield from self._dict.items()
+
+    def as_dict(self):
+        return self._dict.copy()
+
+    def as_list(self):
+        from operator import itemgetter
+
+        # Sorting the list by path to ensure reproducibility
+        return sorted(
+            (
+                {
+                    # NOTE: not using hash_info.to_dict() because we don't want
+                    # size/nfiles fields at this point.
+                    obj.hash_info.name: obj.hash_info.value,
+                    self.PARAM_RELPATH: posixpath.sep.join(parts),
+                }
+                for parts, obj in self._dict.items()  # noqa: B301
+            ),
+            key=itemgetter(self.PARAM_RELPATH),
+        )
+
+    def as_bytes(self):
+        return json.dumps(self.as_list(), sort_keys=True).encode("utf-8")
+
+    @classmethod
+    def from_list(cls, lst):
+        from dvc.hash_info import HashInfo
+
+        tree = cls(None, None, None)
+        for _entry in lst:
+            entry = _entry.copy()
+            relpath = entry.pop(cls.PARAM_RELPATH)
+            parts = tuple(relpath.split(posixpath.sep))
+            hash_info = HashInfo.from_dict(entry)
+            obj = HashFile(None, None, hash_info)
+            tree.add(parts, obj)
+        return tree
+
+    @classmethod
+    def load(cls, odb, hash_info):
+
+        obj = odb.get(hash_info)
+
+        try:
+            with obj.fs.open(obj.path_info, "r") as fobj:
+                raw = json.load(fobj)
+        except ValueError as exc:
+            raise ObjectFormatError(f"{obj} is corrupted") from exc
+
+        if not isinstance(raw, list):
+            logger.error(
+                "dir cache file format error '%s' [skipping the file]",
+                obj.path_info,
+            )
+            raise ObjectFormatError(f"{obj} is corrupted")
+
+        tree = cls.from_list(raw)
+        tree.path_info = obj.path_info
+        tree.fs = obj.fs
+        tree.hash_info = hash_info
+
+        return tree
+
+    def filter(self, odb, prefix):
+        obj = self._dict.get(prefix)
+        if obj:
+            return obj
+
+        depth = len(prefix)
+        tree = Tree(None, None, None)
+        try:
+            for key, obj in self.trie.items(prefix):
+                tree.add(key[depth:], obj)
+        except KeyError:
+            return None
+        tree.digest()
+        odb.add(tree.path_info, tree.fs, tree.hash_info)
+        return tree
+
+
+def _get_dir_size(odb, tree):
+    try:
+        return sum(
+            odb.fs.getsize(odb.hash_to_path_info(obj.hash_info.value))
+            for _, obj in tree
+        )
+    except FileNotFoundError:
+        return None
+
+
+def _diff(ancestor, other, allow_removed=False):
+    from dictdiffer import diff
+
+    from dvc.exceptions import MergeError
+
+    allowed = ["add"]
+    if allow_removed:
+        allowed.append("remove")
+
+    result = list(diff(ancestor, other))
+    for typ, _, _ in result:
+        if typ not in allowed:
+            raise MergeError(
+                "unable to auto-merge directories with diff that contains "
+                f"'{typ}'ed files"
+            )
+    return result
+
+
+def _merge(ancestor, our, their):
+    import copy
+
+    from dictdiffer import patch
+
+    our_diff = _diff(ancestor, our)
+    if not our_diff:
+        return copy.deepcopy(their)
+
+    their_diff = _diff(ancestor, their)
+    if not their_diff:
+        return copy.deepcopy(our)
+
+    # make sure there are no conflicting files
+    _diff(our, their, allow_removed=True)
+
+    return patch(our_diff + their_diff, ancestor)
+
+
+def merge(odb, ancestor_info, our_info, their_info):
+    from . import load
+
+    assert our_info
+    assert their_info
+
+    if ancestor_info:
+        ancestor = load(odb, ancestor_info)
+    else:
+        ancestor = Tree(None, None, None)
+
+    our = load(odb, our_info)
+    their = load(odb, their_info)
+
+    merged_dict = _merge(ancestor.as_dict(), our.as_dict(), their.as_dict(),)
+
+    merged = Tree(None, None, None)
+    for key, hi in merged_dict.items():
+        merged.add(key, hi)
+    merged.digest()
+
+    odb.add(merged.path_info, merged.fs, merged.hash_info)
+    hash_info = merged.hash_info
+    hash_info.size = _get_dir_size(odb, merged)
+    return hash_info

dvc/output/base.py

@@ -17,8 +17,9 @@ from dvc.exceptions import (
     RemoteCacheRequiredError,
 )
 from dvc.hash_info import HashInfo
+from dvc.objects import save as osave
 from dvc.objects.db import NamedCache
-from dvc.objects.stage import get_hash
+from dvc.objects.errors import ObjectFormatError
 from dvc.objects.stage import stage as ostage
 
 from ..fs.base import BaseFileSystem
@@ -183,23 +184,21 @@ class BaseOutput:
     def odb(self):
         return getattr(self.repo.odb, self.scheme)
 
-    @property
-    def dir_cache(self):
-        return self.hash_info.dir_info
-
     @property
     def cache_path(self):
         return self.odb.hash_to_path_info(self.hash_info.value).url
 
     def get_hash(self):
         if not self.use_cache:
-            return get_hash(
+            return ostage(
+                self.repo.odb.local,
                 self.path_info,
                 self.fs,
                 self.fs.PARAM_CHECKSUM,
-                self.repo.odb.local,
-            )
-        return ostage(self.odb, self.path_info, self.fs).hash_info
+            ).hash_info
+        return ostage(
+            self.odb, self.path_info, self.fs, self.odb.fs.PARAM_CHECKSUM
+        ).hash_info
 
     @property
     def is_dir_checksum(self):
@@ -223,7 +222,7 @@ class BaseOutput:
         try:
             objects.check(self.odb, obj)
             return False
-        except (FileNotFoundError, objects.ObjectFormatError):
+        except (FileNotFoundError, ObjectFormatError):
             return True
 
     def workspace_status(self):
@@ -307,7 +306,9 @@ class BaseOutput:
             logger.debug("Output '%s' didn't change. Skipping saving.", self)
             return
 
-        self.obj = ostage(self.odb, self.path_info, self.fs)
+        self.obj = ostage(
+            self.odb, self.path_info, self.fs, self.odb.fs.PARAM_CHECKSUM
+        )
         self.hash_info = self.obj.hash_info
         self.isexec = self.isfile() and self.fs.isexec(self.path_info)
 
@@ -322,7 +323,12 @@ class BaseOutput:
         assert self.hash_info
 
         if self.use_cache:
-            obj = ostage(self.odb, filter_info or self.path_info, self.fs)
+            obj = ostage(
+                self.odb,
+                filter_info or self.path_info,
+                self.fs,
+                self.odb.fs.PARAM_CHECKSUM,
+            )
             objects.save(self.odb, obj)
             checkout(
                 filter_info or self.path_info,
@@ -403,6 +409,7 @@ class BaseOutput:
         relink=False,
         filter_info=None,
         allow_missing=False,
+        checkpoint_reset=False,
         **kwargs,
     ):
         if not self.use_cache:
@@ -417,6 +424,11 @@ class BaseOutput:
             # backward compatibility
             return None
 
+        if self.checkpoint and checkpoint_reset:
+            if self.exists:
+                self.remove()
+            return None
+
         added = not self.exists
 
         try:
@@ -459,9 +471,43 @@ class BaseOutput:
         if self.scheme == "local" and self.use_scm_ignore:
             self.repo.scm.ignore(self.fspath)
 
-    def get_files_number(self, filter_info=None):
-        from funcy import ilen
+    def transfer(
+        self, source, odb=None, jobs=None, update=False, no_progress_bar=False,
+    ):
+        from dvc.fs import get_cloud_fs
 
+        if odb is None:
+            odb = self.odb
+
+        from_fs = get_cloud_fs(self.repo, url=source)
+        from_info = from_fs.path_info
+
+        # When running import-url --to-remote / add --to-remote/-o ... we
+        # assume that it is unlikely that the odb will contain majority of the
+        # hashes, so we transfer everything as is (even if that file might
+        # already be in the cache) and don't waste an upload to scan the layout
+        # of the source location. But when doing update --to-remote, there is
+        # a high probability that the odb might contain some of the hashes, so
+        # we first calculate all the hashes (but don't transfer anything) and
+        # then only update the missing cache files.
+
+        upload = not (update and from_fs.isdir(from_info))
+        jobs = jobs or min((from_fs.jobs, odb.fs.jobs))
+        obj = ostage(
+            odb,
+            from_info,
+            from_fs,
+            "md5",
+            upload=upload,
+            jobs=jobs,
+            no_progress_bar=no_progress_bar,
+        )
+        osave(odb, obj, jobs=jobs, move=upload)
+
+        self.hash_info = obj.hash_info
+        return obj
+
+    def get_files_number(self, filter_info=None):
         if not self.use_cache or not self.hash_info:
             return 0
 
@@ -471,13 +517,8 @@ class BaseOutput:
         if not filter_info or filter_info == self.path_info:
             return self.hash_info.nfiles or 0
 
-        if not self.hash_info.dir_info:
-            return 0
-
-        return ilen(
-            filter_info.isin_or_eq(self.path_info / relpath)
-            for relpath, _ in self.hash_info.dir_info.items()
-        )
+        obj = self.get_obj(filter_info=filter_info)
+        return len(obj) if obj else 0
 
     def unprotect(self):
         if self.exists:
@@ -490,7 +531,7 @@ class BaseOutput:
 
         try:
             objects.check(self.odb, self.odb.get(self.hash_info))
-        except (FileNotFoundError, objects.ObjectFormatError):
+        except (FileNotFoundError, ObjectFormatError):
             self.repo.cloud.pull(
                 NamedCache.make("local", self.hash_info.value, str(self)),
                 show_checksums=False,
@@ -498,12 +539,11 @@ class BaseOutput:
             )
 
         try:
-            objects.load(self.odb, self.hash_info)
-            assert self.hash_info.dir_info
-        except (objects.ObjectFormatError, FileNotFoundError):
-            self.hash_info.dir_info = None
+            self.obj = objects.load(self.odb, self.hash_info)
+        except (FileNotFoundError, ObjectFormatError):
+            self.obj = None
 
-        return self.dir_cache
+        return self.obj
 
     def collect_used_dir_cache(
         self, remote=None, force=False, jobs=None, filter_info=None
@@ -536,7 +576,7 @@ class BaseOutput:
 
         try:
             objects.check(self.odb, self.odb.get(self.hash_info))
-        except (FileNotFoundError, objects.ObjectFormatError):
+        except (FileNotFoundError, ObjectFormatError):
             msg = (
                 "Missing cache for directory '{}'. "
                 "Cache for files inside will be lost. "
@@ -551,14 +591,14 @@ class BaseOutput:
 
         path = str(self.path_info)
         filter_path = str(filter_info) if filter_info else None
-        for entry_key, entry_hash_info in self.dir_cache.items():
+        for entry_key, entry_obj in self.obj:
             entry_path = os.path.join(path, *entry_key)
             if (
                 not filter_path
                 or entry_path == filter_path
                 or entry_path.startswith(filter_path + os.sep)
             ):
-                cache.add(self.scheme, entry_hash_info.value, entry_path)
+                cache.add(self.scheme, entry_obj.hash_info.value, entry_path)
 
         return cache
 
@@ -651,6 +691,8 @@ class BaseOutput:
             )
 
     def merge(self, ancestor, other):
+        from dvc.objects.tree import merge
+
         assert other
 
         if ancestor:
@@ -662,6 +704,6 @@ class BaseOutput:
         self._check_can_merge(self)
         self._check_can_merge(other)
 
-        self.hash_info = objects.merge(
+        self.hash_info = merge(
             self.odb, ancestor_info, self.hash_info, other.hash_info
         )

dvc/output/local.py

@@ -48,7 +48,11 @@ class LocalOutput(BaseOutput):
         return self.FS_CLS.PATH_CLS(abs_p)
 
     def __str__(self):
-        if not self.repo or not self.is_in_repo:
+        if (
+            not self.repo
+            or urlparse(self.def_path).scheme == "remote"
+            or os.path.isabs(self.def_path)
+        ):
             return str(self.def_path)
 
         cur_dir = os.getcwd()
@@ -63,8 +67,15 @@ class LocalOutput(BaseOutput):
 
     @property
     def is_in_repo(self):
-        def_scheme = urlparse(self.def_path).scheme
-        return def_scheme != "remote" and not os.path.isabs(self.def_path)
+        if urlparse(self.def_path).scheme == "remote":
+            return False
+
+        if os.path.isabs(self.def_path):
+            return False
+
+        if self.repo and not self.path_info.isin(self.repo.root_dir):
+            return False
+        return True
 
     def dumpd(self):
         ret = super().dumpd()

dvc/parsing/context.py

@@ -401,9 +401,9 @@ class Context(CtxDict):
     ):
         path, _, keys_str = item.partition(":")
         select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
-        path_info = wdir / path
 
-        abspath = os.path.abspath(path_info)
+        abspath = os.path.abspath(wdir / path)
+        path_info = PathInfo(abspath)
         if abspath in self.imports:
             if not select_keys and self.imports[abspath] is None:
                 return  # allow specifying complete filepath multiple times

dvc/remote/base.py

@@ -309,7 +309,7 @@ class Remote:
         )
 
         if download:
-            func = _log_exceptions(self.fs.download, "download")
+            func = _log_exceptions(self.fs.download_file, "download")
             status = STATUS_DELETED
             desc = "Downloading"
         else:
@@ -318,7 +318,7 @@ class Remote:
             desc = "Uploading"
 
         if jobs is None:
-            jobs = self.fs.JOBS
+            jobs = self.fs.jobs
 
         dir_status, file_status, dir_contents = self._status(
             cache,
@@ -505,17 +505,6 @@ class Remote:
 
         return ret
 
-    def transfer(self, from_fs, from_info, jobs=None, no_progress_bar=False):
-        from dvc.objects import transfer
-
-        return transfer(
-            self.odb,
-            from_fs,
-            from_info,
-            jobs=jobs,
-            no_progress_bar=no_progress_bar,
-        )
-
     @staticmethod
     def _log_missing_caches(hash_info_dict):
         missing_caches = [

dvc/repo/__init__.py

@@ -141,6 +141,7 @@ class Repo:
         from dvc.repo.params import Params
         from dvc.repo.plots import Plots
         from dvc.repo.stage import StageLoad
+        from dvc.scm import SCM
         from dvc.stage.cache import StageCache
         from dvc.state import State, StateNoop
 
@@ -149,6 +150,9 @@ class Repo:
             "repo_factory": repo_factory,
         }
 
+        if rev and not scm:
+            scm = SCM(root_dir or os.curdir)
+
         self.root_dir, self.dvc_dir, self.tmp_dir = self._get_repo_dirs(
             root_dir=root_dir, scm=scm, rev=rev, uninitialized=uninitialized
         )
@@ -463,6 +467,12 @@ class Repo:
         path_parts = os.path.normpath(path).split(os.path.sep)
         return self.DVC_DIR in path_parts
 
+    @cached_property
+    def dvcfs(self):
+        from dvc.fs.dvc import DvcFileSystem
+
+        return DvcFileSystem(self)
+
     @cached_property
     def repo_fs(self):
         from dvc.fs.repo import RepoFileSystem

dvc/repo/add.py

@@ -164,21 +164,14 @@ def _process_stages(
         (target,) = sub_targets
         (out,) = stage.outs
 
+        odb = None
         if to_remote:
-            out.hash_info = repo.cloud.transfer(
-                target,
-                jobs=kwargs.get("jobs"),
-                remote=kwargs.get("remote"),
-                command="add",
-            )
-        else:
-            from dvc.fs import get_cloud_fs
-            from dvc.objects import transfer
+            remote = repo.cloud.get_remote(kwargs.get("remote"), "add")
+            odb = remote.odb
 
-            from_fs = get_cloud_fs(repo, url=target)
-            out.hash_info = transfer(
-                out.odb, from_fs, from_fs.path_info, jobs=kwargs.get("jobs"),
-            )
+        out.transfer(target, odb=odb, jobs=kwargs.get("jobs"))
+
+        if to_cache:
             out.checkout()
 
         Dvcfile(repo, stage.path).dump(stage)

dvc/repo/commit.py

@@ -37,20 +37,32 @@ def prompt_to_commit(stage, changes, force=False):
 
 @locked
 def commit(
-    self, target, with_deps=False, recursive=False, force=False,
+    self,
+    target,
+    with_deps=False,
+    recursive=False,
+    force=False,
+    allow_missing=False,
+    data_only=False,
 ):
     from dvc.dvcfile import Dvcfile
 
-    stages_info = self.stage.collect_granular(
-        target, with_deps=with_deps, recursive=recursive
-    )
+    stages_info = [
+        info
+        for info in self.stage.collect_granular(
+            target, with_deps=with_deps, recursive=recursive
+        )
+        if not data_only or info.stage.is_data_source
+    ]
     for stage_info in stages_info:
         stage = stage_info.stage
         changes = stage.changed_entries()
         if any(changes):
             prompt_to_commit(stage, changes, force=force)
-            stage.save()
-        stage.commit(filter_info=stage_info.filter_info)
+            stage.save(allow_missing=allow_missing)
+        stage.commit(
+            filter_info=stage_info.filter_info, allow_missing=allow_missing
+        )
 
         Dvcfile(self, stage.path).dump(stage, update_pipeline=False)
     return [s.stage for s in stages_info]

dvc/repo/diff.py

@@ -1,10 +1,13 @@
 import logging
 import os
+from collections import defaultdict
+from typing import Dict, List
 
 from dvc.exceptions import PathMissingError
 from dvc.objects.stage import get_file_hash
 from dvc.objects.stage import stage as ostage
 from dvc.repo import locked
+from dvc.repo.experiments.utils import fix_exp_head
 
 logger = logging.getLogger(__name__)
 
@@ -26,7 +29,8 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
 
     repo_fs = RepoFileSystem(self)
 
-    b_rev = b_rev if b_rev else "workspace"
+    a_rev = fix_exp_head(self.scm, a_rev)
+    b_rev = fix_exp_head(self.scm, b_rev) if b_rev else "workspace"
     results = {}
     missing_targets = {}
     for rev in self.brancher(revs=[a_rev, b_rev]):
@@ -130,7 +134,7 @@ def _output_paths(repo, repo_fs, targets):
     def _to_checksum(output):
         if on_working_fs:
             return ostage(
-                repo.odb.local, output.path_info, repo.odb.local.fs,
+                repo.odb.local, output.path_info, repo.odb.local.fs, "md5",
             ).hash_info.value
         return output.hash_info.value
 
@@ -159,6 +163,7 @@ def _dir_output_paths(repo_fs, output, targets=None):
             if targets is None or any(
                 fname.isin_or_eq(target) for target in targets
             ):
+                # pylint: disable=no-member
                 yield str(fname), get_file_hash(fname, repo_fs, "md5").value
     except NoRemoteError:
         logger.warning("dir cache entry for '%s' is missing", output)
@@ -190,29 +195,23 @@ def _targets_to_path_infos(repo_fs, targets):
 
 
 def _calculate_renamed(new, old, added, deleted):
-    old_inverted = {}
+    old_inverted: Dict[str, List[str]] = defaultdict(list)
     # It is needed to be dict of lists to cover cases
     # when repo has paths with same hash
     for path, path_hash in old.items():
-        bucket = old_inverted.get(path_hash, None)
-        if bucket is None:
-            old_inverted[path_hash] = [path]
-        else:
-            bucket.append(path)
+        old_inverted[path_hash].append(path)
 
     renamed = []
     for path in added:
         path_hash = new[path]
-        old_paths = old_inverted.get(path_hash, None)
-        if not old_paths:
-            continue
-        old_path = None
-        for tmp_old_path in old_paths:
-            if tmp_old_path in deleted:
-                old_path = tmp_old_path
-                break
-        if not old_path:
+        old_paths = old_inverted[path_hash]
+        try:
+            iterator = enumerate(old_paths)
+            index = next(idx for idx, path in iterator if path in deleted)
+        except StopIteration:
             continue
+
+        old_path = old_paths.pop(index)
         renamed.append(
             {"path": {"old": old_path, "new": path}, "hash": path_hash}
         )

dvc/repo/experiments/__init__.py

@@ -182,17 +182,19 @@ class Experiments:
                         )
                         self.scm.reset()
 
-                    self._prune_lockfiles()
-
                     # update experiment params from command line
                     if params:
                         self._update_params(params)
 
+                    # DVC commit data deps to preserve state across workspace
+                    # & tempdir runs
+                    self._stash_commit_deps(*args, **kwargs)
+
                     if resume_rev:
                         if branch:
                             branch_name = ExpRefInfo.from_ref(branch).name
                         else:
-                            branch_name = ""
+                            branch_name = f"{resume_rev[:7]}"
                         if self.scm.is_dirty():
                             logger.info(
                                 "Modified checkpoint experiment based on "
@@ -200,6 +202,39 @@ class Experiments:
                                 branch_name,
                             )
                             branch = None
+                        elif (
+                            not branch
+                            or self.scm.get_ref(branch) != resume_rev
+                        ):
+                            msg = [
+                                (
+                                    "Nothing to do for unchanged checkpoint "
+                                    f"'{resume_rev[:7]}'. "
+                                )
+                            ]
+                            if branch:
+                                msg.append(
+                                    "To resume from the head of this "
+                                    "experiment, use "
+                                    f"'dvc exp apply {branch_name}'."
+                                )
+                            else:
+                                names = [
+                                    ref_info.name
+                                    for ref_info in exp_refs_by_rev(
+                                        self.scm, resume_rev
+                                    )
+                                ]
+                                if len(names) > 3:
+                                    names[3:] = [
+                                        f"... ({len(names) - 3} more)"
+                                    ]
+                                msg.append(
+                                    "To resume an experiment containing this "
+                                    "checkpoint, apply one of these heads:\n"
+                                    "\t{}".format(", ".join(names))
+                                )
+                            raise DvcException("".join(msg))
                         else:
                             logger.info(
                                 "Existing checkpoint experiment '%s' will be "
@@ -235,42 +270,37 @@ class Experiments:
                         baseline_rev[:7],
                     )
                 finally:
-                    # Reset any of our changes before prior unstashing
                     if resume_rev:
+                        # NOTE: this set_ref + reset() is equivalent to
+                        # `git reset orig_head` (our SCM reset() only operates
+                        # on HEAD rather than any arbitrary commit)
                         self.scm.set_ref(
                             "HEAD", orig_head, message="dvc: restore HEAD"
                         )
+                        self.scm.reset()
+                    # Revert any of our changes before prior unstashing
                     self.scm.reset(hard=True)
 
         return stash_rev
 
-    def _prune_lockfiles(self):
-        from dvc.dvcfile import is_lock_file
-
-        # NOTE: dirty DVC lock files must be restored to index state to
-        # avoid checking out incorrect persist or checkpoint outs
-        fs = self.scm.get_fs("HEAD")
-        lock_files = [
-            str(fname)
-            for fname in fs.walk_files(self.scm.root_dir)
-            if is_lock_file(fname)
-        ]
-        if lock_files:
-
-            self.scm.reset(paths=lock_files)
-            self.scm.checkout_index(paths=lock_files, force=True)
-
-    def _prune_untracked_lockfiles(self):
-        from dvc.dvcfile import is_lock_file
-        from dvc.utils.fs import remove
-
-        untracked = [
-            fname
-            for fname in self.scm.untracked_files()
-            if is_lock_file(fname)
-        ]
-        for fname in untracked:
-            remove(fname)
+    def _stash_commit_deps(self, *args, **kwargs):
+        if len(args):
+            targets = args[0]
+        else:
+            targets = kwargs.get("targets")
+        if isinstance(targets, str):
+            targets = [targets]
+        elif not targets:
+            targets = [None]
+        for target in targets:
+            self.repo.commit(
+                target,
+                with_deps=True,
+                recursive=kwargs.get("recursive", False),
+                force=True,
+                allow_missing=True,
+                data_only=True,
+            )
 
     def _stash_msg(
         self,
@@ -345,9 +375,16 @@ class Experiments:
         queue: bool = False,
         tmp_dir: bool = False,
         checkpoint_resume: Optional[str] = None,
+        reset: bool = False,
         **kwargs,
     ):
         """Reproduce and checkout a single experiment."""
+        if queue and not checkpoint_resume:
+            reset = True
+
+        if reset:
+            self.reset_checkpoints()
+
         if not (queue or tmp_dir):
             staged, _, _ = self.scm.status()
             if staged:
@@ -370,7 +407,9 @@ class Experiments:
         else:
             checkpoint_resume = self._workspace_resume_rev()
 
-        stash_rev = self.new(checkpoint_resume=checkpoint_resume, **kwargs)
+        stash_rev = self.new(
+            checkpoint_resume=checkpoint_resume, reset=reset, **kwargs
+        )
         if queue:
             logger.info(
                 "Queued experiment '%s' for future execution.", stash_rev[:7],
@@ -445,22 +484,34 @@ class Experiments:
         """
         assert resume_rev
 
-        allow_multiple = "params" in kwargs
-        branch: Optional[str] = self.get_branch_by_rev(
-            resume_rev, allow_multiple=allow_multiple
-        )
-        if not branch:
-            raise DvcException(
-                "Could not find checkpoint experiment " f"'{resume_rev[:7]}'"
+        branch: Optional[str] = None
+        try:
+            allow_multiple = bool(kwargs.get("params", None))
+            branch = self.get_branch_by_rev(
+                resume_rev, allow_multiple=allow_multiple
             )
+            if not branch:
+                raise DvcException(
+                    "Could not find checkpoint experiment "
+                    f"'{resume_rev[:7]}'"
+                )
+            baseline_rev = self._get_baseline(branch)
+        except MultipleBranchError as exc:
+            baselines = {
+                info.baseline_sha
+                for info in exc.ref_infos
+                if info.baseline_sha
+            }
+            if len(baselines) == 1:
+                baseline_rev = baselines.pop()
+            else:
+                raise
 
-        baseline_rev = self._get_baseline(branch)
         logger.debug(
-            "Resume from checkpoint '%s' with baseline '%s'",
-            resume_rev,
+            "Checkpoint run from '%s' with baseline '%s'",
+            resume_rev[:7],
             baseline_rev,
         )
-
         return self._stash_exp(
             *args,
             resume_rev=resume_rev,
@@ -705,11 +756,10 @@ class Experiments:
 
         # NOTE: the stash commit to be popped already contains all the current
         # workspace changes plus CLI modifed --params changes.
-        # `reset --hard` here will not lose any data (pop without reset would
-        # result in conflict between workspace params and stashed CLI params).
-        self.scm.reset(hard=True)
-        with self.scm.detach_head(entry.rev):
-            self._prune_untracked_lockfiles()
+        # `checkout --force` here will not lose any data (popping stash commit
+        # will result in conflict between workspace params and stashed CLI
+        # params, but we always want the stashed version).
+        with self.scm.detach_head(entry.rev, force=True):
             rev = self.stash.pop()
             self.scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
             if entry.branch:
@@ -723,12 +773,17 @@ class Experiments:
                     rev,
                     name=entry.name,
                     rel_cwd=relpath(os.getcwd(), self.scm.root_dir),
+                    log_errors=False,
                 )
 
-                if not exec_result.exp_hash or not exec_result.ref_info:
+                if not exec_result.exp_hash:
                     raise DvcException(
                         f"Failed to reproduce experiment '{rev[:7]}'"
                     )
+                if not exec_result.ref_info:
+                    # repro succeeded but result matches baseline
+                    # (no experiment generated or applied)
+                    return {}
                 exp_rev = self.scm.get_ref(str(exec_result.ref_info))
                 self.scm.set_ref(EXEC_APPLY, exp_rev)
                 return {exp_rev: exec_result.exp_hash}
@@ -791,7 +846,10 @@ class Experiments:
         if not ref_infos:
             return None
         if len(ref_infos) > 1 and not allow_multiple:
-            raise MultipleBranchError(rev)
+            for ref_info in ref_infos:
+                if self.scm.get_ref(str(ref_info)) == rev:
+                    return str(ref_info)
+            raise MultipleBranchError(rev, ref_infos)
         return str(ref_infos[0])
 
     def get_exact_name(self, rev: str):

dvc/repo/experiments/apply.py

@@ -32,7 +32,9 @@ def apply(repo, rev, force=True, **kwargs):
         raise InvalidExpRevError(rev) from exc
 
     stash_rev = exp_rev in exps.stash_revs
-    if not stash_rev and not exps.get_branch_by_rev(exp_rev):
+    if not stash_rev and not exps.get_branch_by_rev(
+        exp_rev, allow_multiple=True
+    ):
         raise InvalidExpRevError(exp_rev)
 
     # Note that we don't use stash_workspace() here since we need finer control

dvc/repo/experiments/base.py

@@ -74,12 +74,13 @@ class InvalidExpRevError(InvalidArgumentError):
 
 
 class MultipleBranchError(DvcException):
-    def __init__(self, rev):
+    def __init__(self, rev, ref_infos):
         super().__init__(
             f"Ambiguous commit '{rev[:7]}' belongs to multiple experiment "
             "branches."
         )
         self.rev = rev
+        self.ref_infos = ref_infos
 
 
 class ApplyConflictError(InvalidArgumentError):

dvc/repo/experiments/diff.py

@@ -1,26 +1,35 @@
 import logging
 
+from dvc.repo.experiments.utils import fix_exp_head
 from dvc.utils.diff import diff as _diff
 from dvc.utils.diff import format_dict
 
 logger = logging.getLogger(__name__)
 
 
-def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
+def diff(repo, *args, a_rev=None, b_rev=None, param_deps=False, **kwargs):
     from dvc.repo.experiments.show import _collect_experiment_commit
 
     if repo.scm.no_commits:
         return {}
 
     if a_rev:
-        old = _collect_experiment_commit(repo, a_rev)
+        a_rev = fix_exp_head(repo.scm, a_rev)
+        rev = repo.scm.resolve_rev(a_rev)
+        old = _collect_experiment_commit(repo, rev, param_deps=param_deps)
     else:
-        old = _collect_experiment_commit(repo, "HEAD")
+        old = _collect_experiment_commit(
+            repo, fix_exp_head(repo.scm, "HEAD"), param_deps=param_deps
+        )
 
     if b_rev:
-        new = _collect_experiment_commit(repo, b_rev)
+        b_rev = fix_exp_head(repo.scm, b_rev)
+        rev = repo.scm.resolve_rev(b_rev)
+        new = _collect_experiment_commit(repo, rev, param_deps=param_deps)
     else:
-        new = _collect_experiment_commit(repo, "workspace")
+        new = _collect_experiment_commit(
+            repo, "workspace", param_deps=param_deps
+        )
 
     with_unchanged = kwargs.pop("all", False)
 

dvc/repo/experiments/executor/base.py

@@ -181,16 +181,16 @@ class BaseExecutor(ABC):
                 refs.append(ref)
 
         def on_diverged_ref(orig_ref: str, new_rev: str):
-            orig_rev = dest_scm.get_ref(orig_ref)
-            if dest_scm.diff(orig_rev, new_rev):
-                if force:
-                    logger.debug(
-                        "Replacing existing experiment '%s'", orig_ref,
-                    )
-                    return True
-                if on_diverged:
-                    checkpoint = self.scm.get_ref(EXEC_CHECKPOINT) is not None
-                    on_diverged(orig_ref, checkpoint)
+            if force:
+                logger.debug(
+                    "Replacing existing experiment '%s'", orig_ref,
+                )
+                return True
+
+            checkpoint = self.scm.get_ref(EXEC_CHECKPOINT) is not None
+            self._raise_ref_conflict(dest_scm, orig_ref, new_rev, checkpoint)
+            if on_diverged:
+                on_diverged(orig_ref, checkpoint)
             logger.debug("Reproduced existing experiment '%s'", orig_ref)
             return False
 
@@ -218,6 +218,7 @@ class BaseExecutor(ABC):
         queue: Optional["Queue"] = None,
         rel_cwd: Optional[str] = None,
         name: Optional[str] = None,
+        log_errors: bool = True,
         log_level: Optional[int] = None,
     ) -> "ExecutorResult":
         """Run dvc repro and return the result.
@@ -234,7 +235,7 @@ class BaseExecutor(ABC):
 
         if queue is not None:
             queue.put((rev, os.getpid()))
-        if log_level is not None:
+        if log_errors and log_level is not None:
             cls._set_log_level(log_level)
 
         def filter_pipeline(stages):
@@ -246,7 +247,7 @@ class BaseExecutor(ABC):
         exp_ref: Optional["ExpRefInfo"] = None
         repro_force: bool = False
 
-        with cls._repro_dvc(dvc_dir, rel_cwd) as dvc:
+        with cls._repro_dvc(dvc_dir, rel_cwd, log_errors) as dvc:
             args, kwargs = cls._repro_args(dvc)
             if args:
                 targets: Optional[Union[list, str]] = args[0]
@@ -258,28 +259,37 @@ class BaseExecutor(ABC):
                 "Executor repro with force = '%s'", str(repro_force)
             )
 
-            # NOTE: for checkpoint experiments we handle persist outs slightly
-            # differently than normal:
+            repro_dry = kwargs.get("dry")
+
+            # NOTE: checkpoint outs are handled as a special type of persist
+            # out:
             #
             # - checkpoint out may not yet exist if this is the first time this
             #   experiment has been run, this is not an error condition for
             #   experiments
-            # - at the start of a repro run, we need to remove the persist out
-            #   and restore it to its last known (committed) state (which may
-            #   be removed/does not yet exist) so that our executor workspace
-            #   is not polluted with the (persistent) out from an unrelated
-            #   experiment run
-            dvc_checkout(
-                dvc,
-                targets=targets,
-                with_deps=targets is not None,
-                force=True,
-                quiet=True,
-                allow_missing=True,
-            )
+            # - if experiment was run with --reset, the checkpoint out will be
+            #   removed at the start of the experiment (regardless of any
+            #   dvc.lock entry for the checkpoint out)
+            # - if run without --reset, the checkpoint out will be checked out
+            #   using any hash present in dvc.lock (or removed if no entry
+            #   exists in dvc.lock)
+            checkpoint_reset: bool = kwargs.pop("reset", False)
+            if not repro_dry:
+                dvc_checkout(
+                    dvc,
+                    targets=targets,
+                    with_deps=targets is not None,
+                    force=True,
+                    quiet=True,
+                    allow_missing=True,
+                    checkpoint_reset=checkpoint_reset,
+                )
 
             checkpoint_func = partial(
-                cls.checkpoint_callback, dvc.scm, name, repro_force
+                cls.checkpoint_callback,
+                dvc.scm,
+                name,
+                repro_force or checkpoint_reset,
             )
             stages = dvc_reproduce(
                 dvc,
@@ -290,29 +300,38 @@ class BaseExecutor(ABC):
             )
 
             exp_hash = cls.hash_exp(stages)
-            try:
-                cls.commit(
-                    dvc.scm,
-                    exp_hash,
-                    exp_name=name,
-                    force=repro_force,
-                    checkpoint=any(stage.is_checkpoint for stage in stages),
-                )
-            except UnchangedExperimentError:
-                pass
-            ref = dvc.scm.get_ref(EXEC_BRANCH, follow=False)
-            if ref:
-                exp_ref = ExpRefInfo.from_ref(ref)
-            if cls.WARN_UNTRACKED:
-                untracked = dvc.scm.untracked_files()
-                if untracked:
-                    logger.warning(
-                        "The following untracked files were present in the "
-                        "experiment directory after reproduction but will "
-                        "not be included in experiment commits:\n"
-                        "\t%s",
-                        ", ".join(untracked),
+            if not repro_dry:
+                try:
+                    is_checkpoint = any(
+                        stage.is_checkpoint for stage in stages
                     )
+                    if is_checkpoint and checkpoint_reset:
+                        # For reset checkpoint stages, we need to force
+                        # overwriting existing checkpoint refs even though
+                        # repro may not have actually been run with --force
+                        repro_force = True
+                    cls.commit(
+                        dvc.scm,
+                        exp_hash,
+                        exp_name=name,
+                        force=repro_force,
+                        checkpoint=is_checkpoint,
+                    )
+                except UnchangedExperimentError:
+                    pass
+                ref = dvc.scm.get_ref(EXEC_BRANCH, follow=False)
+                if ref:
+                    exp_ref = ExpRefInfo.from_ref(ref)
+                if cls.WARN_UNTRACKED:
+                    untracked = dvc.scm.untracked_files()
+                    if untracked:
+                        logger.warning(
+                            "The following untracked files were present in "
+                            "the experiment directory after reproduction but "
+                            "will not be included in experiment commits:\n"
+                            "\t%s",
+                            ", ".join(untracked),
+                        )
 
         # ideally we would return stages here like a normal repro() call, but
         # stages is not currently picklable and cannot be returned across
@@ -321,7 +340,9 @@ class BaseExecutor(ABC):
 
     @classmethod
     @contextmanager
-    def _repro_dvc(cls, dvc_dir: Optional[str], rel_cwd: Optional[str]):
+    def _repro_dvc(
+        cls, dvc_dir: Optional[str], rel_cwd: Optional[str], log_errors: bool
+    ):
         from dvc.repo import Repo
 
         dvc = Repo(dvc_dir)
@@ -342,10 +363,12 @@ class BaseExecutor(ABC):
         except CheckpointKilledError:
             raise
         except DvcException:
-            logger.exception("")
+            if log_errors:
+                logger.exception("")
             raise
         except Exception:
-            logger.exception("unexpected error")
+            if log_errors:
+                logger.exception("unexpected error")
             raise
         finally:
             dvc.close()
@@ -358,6 +381,8 @@ class BaseExecutor(ABC):
         if os.path.exists(args_path):
             args, kwargs = cls.unpack_repro_args(args_path)
             remove(args_path)
+            # explicitly git rm/unstage the args file
+            dvc.scm.add([args_path])
         else:
             args = []
             kwargs = {}
@@ -396,6 +421,7 @@ class BaseExecutor(ABC):
             logger.debug("No changes to commit")
             raise UnchangedExperimentError(rev)
 
+        check_conflict = False
         branch = scm.get_ref(EXEC_BRANCH, follow=False)
         if branch:
             old_ref = rev
@@ -406,21 +432,41 @@ class BaseExecutor(ABC):
             ref_info = ExpRefInfo(baseline_rev, name)
             branch = str(ref_info)
             old_ref = None
-            if not force and scm.get_ref(branch):
-                if checkpoint:
-                    raise CheckpointExistsError(ref_info.name)
-                raise ExperimentExistsError(ref_info.name)
-            logger.debug("Commit to new experiment branch '%s'", branch)
+            if scm.get_ref(branch):
+                if not force:
+                    check_conflict = True
+                logger.debug(
+                    "%s existing experiment branch '%s'",
+                    "Replace" if force else "Reuse",
+                    branch,
+                )
+            else:
+                logger.debug("Commit to new experiment branch '%s'", branch)
 
         scm.add([], update=True)
         scm.commit(f"dvc: commit experiment {exp_hash}", no_verify=True)
         new_rev = scm.get_rev()
-        scm.set_ref(branch, new_rev, old_ref=old_ref)
+        if check_conflict:
+            new_rev = cls._raise_ref_conflict(scm, branch, new_rev, checkpoint)
+        else:
+            scm.set_ref(branch, new_rev, old_ref=old_ref)
         scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
         if checkpoint:
             scm.set_ref(EXEC_CHECKPOINT, new_rev)
         return new_rev
 
+    @staticmethod
+    def _raise_ref_conflict(scm, ref, new_rev, checkpoint):
+        # If this commit is a duplicate of the existing commit at 'ref', return
+        # the existing commit. Otherwise, error out and require user to re-run
+        # with --force as needed
+        orig_rev = scm.get_ref(ref)
+        if scm.diff(orig_rev, new_rev):
+            if checkpoint:
+                raise CheckpointExistsError(ref)
+            raise ExperimentExistsError(ref)
+        return orig_rev
+
     @staticmethod
     def _set_log_level(level):
         from dvc.logger import disable_other_loggers

dvc/repo/experiments/executor/local.py

@@ -3,13 +3,29 @@ import os
 from tempfile import TemporaryDirectory
 from typing import Optional
 
-from dvc.utils.fs import remove
-
 from .base import BaseExecutor
 
 logger = logging.getLogger(__name__)
 
 
+class ExpTemporaryDirectory(TemporaryDirectory):
+    # Python's TemporaryDirectory cleanup shutil.rmtree wrapper does not handle
+    # git read-only dirs cleanly in Windows on Python <3.8, so we use our own
+    # remove(). See:
+    # https://github.com/iterative/dvc/pull/5425
+    # https://bugs.python.org/issue26660
+
+    @classmethod
+    def _rmtree(cls, name):
+        from dvc.utils.fs import remove
+
+        remove(name)
+
+    def cleanup(self):
+        if self._finalizer.detach():
+            self._rmtree(self.name)
+
+
 class BaseLocalExecutor(BaseExecutor):
     """Base local machine executor."""
 
@@ -37,7 +53,7 @@ class TempDirExecutor(BaseLocalExecutor):
         cache_dir: Optional[str] = None,
         **kwargs,
     ):
-        self._tmp_dir = TemporaryDirectory(dir=tmp_dir)
+        self._tmp_dir = ExpTemporaryDirectory(dir=tmp_dir)
         kwargs["root_dir"] = self._tmp_dir.name
         super().__init__(*args, **kwargs)
         if cache_dir:
@@ -55,4 +71,4 @@ class TempDirExecutor(BaseLocalExecutor):
     def cleanup(self):
         super().cleanup()
         logger.debug("Removing tmpdir '%s'", self._tmp_dir)
-        remove(self._tmp_dir.name)
+        self._tmp_dir.cleanup()

dvc/repo/experiments/run.py

@@ -15,7 +15,6 @@ def run(
     run_all: bool = False,
     jobs: int = 1,
     tmp_dir: bool = False,
-    reset: bool = False,
     **kwargs,
 ) -> dict:
     """Reproduce the specified targets as an experiment.
@@ -25,10 +24,6 @@ def run(
     Returns a dict mapping new experiment SHAs to the results
     of `repro` for that experiment.
     """
-    if reset:
-        repo.experiments.reset_checkpoints()
-        kwargs["force"] = True
-
     if run_all:
         return repo.experiments.reproduce_queued(jobs=jobs)
 

dvc/repo/experiments/show.py

@@ -5,6 +5,7 @@ from datetime import datetime
 from dvc.exceptions import InvalidArgumentError
 from dvc.repo import locked
 from dvc.repo.experiments.base import ExpRefInfo
+from dvc.repo.experiments.utils import fix_exp_head
 from dvc.repo.metrics.show import _collect_metrics, _read_metrics
 from dvc.repo.params.show import _collect_configs, _read_params
 from dvc.scm.base import SCMError
@@ -12,17 +13,23 @@ from dvc.scm.base import SCMError
 logger = logging.getLogger(__name__)
 
 
-def _collect_experiment_commit(repo, rev, stash=False, sha_only=True):
+def _collect_experiment_commit(
+    repo, exp_rev, stash=False, sha_only=True, param_deps=False
+):
     res = defaultdict(dict)
-    for rev in repo.brancher(revs=[rev]):
+    for rev in repo.brancher(revs=[exp_rev]):
         if rev == "workspace":
+            if exp_rev != "workspace":
+                continue
             res["timestamp"] = None
         else:
             commit = repo.scm.resolve_commit(rev)
             res["timestamp"] = datetime.fromtimestamp(commit.commit_time)
 
-        configs = _collect_configs(repo, rev=rev)
-        params = _read_params(repo, configs, rev)
+        params, params_path_infos = _collect_configs(repo, rev=rev)
+        params = _read_params(
+            repo, params, params_path_infos, rev, deps=param_deps
+        )
         if params:
             res["params"] = params
 
@@ -83,6 +90,7 @@ def show(
     all_commits=False,
     sha_only=False,
     num=1,
+    param_deps=False,
 ):
     res = defaultdict(OrderedDict)
 
@@ -93,7 +101,8 @@ def show(
         revs = []
         for n in range(num):
             try:
-                revs.append(repo.scm.resolve_rev(f"HEAD~{n}"))
+                head = fix_exp_head(repo.scm, f"HEAD~{n}")
+                revs.append(repo.scm.resolve_rev(head))
             except SCMError:
                 break
 
@@ -110,7 +119,7 @@ def show(
 
     for rev in revs:
         res[rev]["baseline"] = _collect_experiment_commit(
-            repo, rev, sha_only=sha_only
+            repo, rev, sha_only=sha_only, param_deps=param_deps
         )
 
         if rev == "workspace":
@@ -127,14 +136,19 @@ def show(
             ref_info = ExpRefInfo.from_ref(exp_ref)
             assert ref_info.baseline_sha == rev
             _collect_experiment_branch(
-                res[rev], repo, exp_ref, rev, sha_only=sha_only
+                res[rev],
+                repo,
+                exp_ref,
+                rev,
+                sha_only=sha_only,
+                param_deps=param_deps,
             )
 
     # collect queued (not yet reproduced) experiments
     for stash_rev, entry in repo.experiments.stash_revs.items():
         if entry.baseline_rev in revs:
             experiment = _collect_experiment_commit(
-                repo, stash_rev, stash=True
+                repo, stash_rev, stash=True, param_deps=param_deps
             )
             res[entry.baseline_rev][stash_rev] = experiment
 

dvc/repo/experiments/utils.py

@@ -1,9 +1,14 @@
-from typing import TYPE_CHECKING, Generator, Iterable, Set
+from typing import Generator, Iterable, Optional, Set
 
-from .base import EXEC_NAMESPACE, EXPS_NAMESPACE, EXPS_STASH, ExpRefInfo
+from dvc.scm.git import Git
 
-if TYPE_CHECKING:
-    from dvc.scm.git import Git
+from .base import (
+    EXEC_BASELINE,
+    EXEC_NAMESPACE,
+    EXPS_NAMESPACE,
+    EXPS_STASH,
+    ExpRefInfo,
+)
 
 
 def exp_refs(scm: "Git") -> Generator["ExpRefInfo", None, None]:
@@ -102,3 +107,11 @@ def remove_exp_refs(scm: "Git", ref_infos: Iterable["ExpRefInfo"]):
         if exec_checkpoint and exec_checkpoint == ref:
             scm.remove_ref(EXEC_CHECKPOINT)
         scm.remove_ref(str(ref_info))
+
+
+def fix_exp_head(scm: "Git", ref: Optional[str]) -> Optional[str]:
+    if ref:
+        name, tail = Git.split_ref_pattern(ref)
+        if name == "HEAD" and scm.get_ref(EXEC_BASELINE):
+            return "".join((EXEC_BASELINE, tail))
+    return ref

dvc/repo/fetch.py

@@ -115,6 +115,7 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs):
                     odb,
                     path_info,
                     repo.repo_fs,
+                    "md5",
                     jobs=jobs,
                     follow_subrepos=False,
                 )

dvc/repo/imp_url.py

@@ -74,9 +74,8 @@ def imp_url(
     if no_exec:
         stage.ignore_outs()
     elif to_remote:
-        stage.outs[0].hash_info = self.cloud.transfer(
-            url, jobs=jobs, remote=remote, command="import-url"
-        )
+        remote = self.cloud.get_remote(remote, "import-url")
+        stage.outs[0].transfer(url, odb=remote.odb, jobs=jobs)
     else:
         stage.run(jobs=jobs)
 

dvc/repo/live.py

@@ -14,14 +14,13 @@ if TYPE_CHECKING:
 
 
 def create_summary(out):
-    from dvc.utils.html import write
-
     assert out.live and out.live["html"]
 
     metrics, plots = out.repo.live.show(str(out.path_info))
 
     html_path = out.path_info.with_suffix(".html")
-    write(html_path, plots, metrics)
+
+    out.repo.plots.write_html(html_path, plots, metrics)
     logger.info(f"\nfile://{os.path.abspath(html_path)}")
 
 

dvc/repo/metrics/diff.py

@@ -1,4 +1,5 @@
 from dvc.exceptions import MetricsError
+from dvc.repo.experiments.utils import fix_exp_head
 from dvc.utils.diff import diff as _diff
 from dvc.utils.diff import format_dict
 
@@ -18,7 +19,8 @@ def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
     with_unchanged = kwargs.pop("all", False)
 
     a_rev = a_rev or "HEAD"
-    b_rev = b_rev or "workspace"
+    a_rev = fix_exp_head(repo.scm, a_rev)
+    b_rev = fix_exp_head(repo.scm, b_rev) or "workspace"
 
     metrics = _get_metrics(repo, *args, **kwargs, revs=[a_rev, b_rev])
     old = metrics.get(a_rev, {})

dvc/repo/params/diff.py

@@ -1,3 +1,4 @@
+from dvc.repo.experiments.utils import fix_exp_head
 from dvc.utils.diff import diff as _diff
 from dvc.utils.diff import format_dict
 
@@ -19,7 +20,8 @@ def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
     with_unchanged = kwargs.pop("all", False)
 
     a_rev = a_rev or "HEAD"
-    b_rev = b_rev or "workspace"
+    a_rev = fix_exp_head(repo.scm, a_rev)
+    b_rev = fix_exp_head(repo.scm, b_rev) or "workspace"
 
     params = _get_params(repo, *args, **kwargs, revs=[a_rev, b_rev])
     old = params.get(a_rev, {})

dvc/repo/params/show.py

@@ -1,6 +1,7 @@
 import logging
 from collections import defaultdict
-from typing import TYPE_CHECKING, List
+from copy import copy
+from typing import TYPE_CHECKING, List, Tuple
 
 from dvc.dependency.param import ParamsDependency
 from dvc.exceptions import DvcException
@@ -27,7 +28,9 @@ def _is_params(dep: "BaseOutput"):
     return isinstance(dep, ParamsDependency)
 
 
-def _collect_configs(repo: "Repo", rev, targets=None) -> List["DvcPath"]:
+def _collect_configs(
+    repo: "Repo", rev, targets=None
+) -> Tuple[List["BaseOutput"], List["DvcPath"]]:
     params, path_infos = collect(
         repo,
         targets=targets or [],
@@ -35,31 +38,45 @@ def _collect_configs(repo: "Repo", rev, targets=None) -> List["DvcPath"]:
         output_filter=_is_params,
         rev=rev,
     )
-    path_infos.extend([p.path_info for p in params])
+    all_path_infos = path_infos + [p.path_info for p in params]
     if not targets:
         default_params = (
             PathInfo(repo.root_dir) / ParamsDependency.DEFAULT_PARAMS_FILE
         )
-        if default_params not in path_infos:
+        if default_params not in all_path_infos:
             path_infos.append(default_params)
-    return list(path_infos)
+    return params, path_infos
 
 
-def _read_params(repo, configs, rev):
-    res = {}
-    for config in configs:
-        if not repo.fs.exists(config):
-            continue
+def _read_path_info(fs, path_info, rev):
+    if not fs.exists(path_info):
+        return None
 
-        suffix = config.suffix.lower()
-        loader = LOADERS[suffix]
-        try:
-            res[str(config)] = loader(config, fs=repo.fs)
-        except ParseError:
-            logger.debug(
-                "failed to read '%s' on '%s'", config, rev, exc_info=True
-            )
-            continue
+    suffix = path_info.suffix.lower()
+    loader = LOADERS[suffix]
+    try:
+        return loader(path_info, fs=fs)
+    except ParseError:
+        logger.debug(
+            "failed to read '%s' on '%s'", path_info, rev, exc_info=True
+        )
+        return None
+
+
+def _read_params(repo, params, params_path_infos, rev, deps=False):
+    res = defaultdict(dict)
+    path_infos = copy(params_path_infos)
+
+    if deps:
+        for param in params:
+            res[str(param.path_info)].update(param.read_params_d())
+    else:
+        path_infos += [param.path_info for param in params]
+
+    for path_info in path_infos:
+        from_path = _read_path_info(repo.fs, path_info, rev)
+        if from_path:
+            res[str(path_info)] = from_path
 
     return res
 
@@ -79,12 +96,12 @@ def _collect_vars(repo, params):
 
 
 @locked
-def show(repo, revs=None, targets=None):
+def show(repo, revs=None, targets=None, deps=False):
     res = {}
 
     for branch in repo.brancher(revs=revs):
-        configs = _collect_configs(repo, branch, targets)
-        params = _read_params(repo, configs, branch)
+        params, params_path_infos = _collect_configs(repo, branch, targets)
+        params = _read_params(repo, params, params_path_infos, branch, deps)
         vars_params = _collect_vars(repo, params)
 
         # NOTE: only those that are not added as a ParamDependency are included

dvc/repo/plots/__init__.py

@@ -1,5 +1,6 @@
 import logging
-from typing import TYPE_CHECKING, Dict, List
+import os
+from typing import TYPE_CHECKING, Dict, List, Optional
 
 from funcy import cached_property, first, project
 
@@ -9,6 +10,7 @@ from dvc.exceptions import (
     NoMetricsFoundError,
     NoMetricsParsedError,
 )
+from dvc.types import StrPath
 from dvc.utils import relpath
 
 if TYPE_CHECKING:
@@ -219,6 +221,26 @@ class Plots:
 
         return PlotTemplates(self.repo.dvc_dir)
 
+    def write_html(
+        self,
+        path: StrPath,
+        plots: Dict[str, Dict],
+        metrics: Optional[Dict[str, Dict]] = None,
+        html_template_path: Optional[StrPath] = None,
+    ):
+        if not html_template_path:
+            html_template_path = self.repo.config.get("plots", {}).get(
+                "html_template", None
+            )
+            if html_template_path and not os.path.isabs(html_template_path):
+                html_template_path = os.path.join(
+                    self.repo.dvc_dir, html_template_path
+                )
+
+        from dvc.utils.html import write
+
+        write(path, plots, metrics, html_template_path)
+
 
 def _is_plot(out: "BaseOutput") -> bool:
     return bool(out.plot) or bool(out.live)

dvc/repo/plots/diff.py

@@ -1,3 +1,6 @@
+from dvc.repo.experiments.utils import fix_exp_head
+
+
 def _revisions(repo, revs, experiment):
     revisions = revs or []
     if experiment and len(revisions) == 1:
@@ -6,7 +9,7 @@ def _revisions(repo, revs, experiment):
             revisions.append(baseline[:7])
     if len(revisions) <= 1:
         if len(revisions) == 0 and repo.scm.is_dirty():
-            revisions.append("HEAD")
+            revisions.append(fix_exp_head(repo.scm, "HEAD"))
         revisions.append("workspace")
     return revisions
 

dvc/repo/reproduce.py

@@ -61,7 +61,11 @@ def _track_stage(stage):
 
     stage.repo.scm.track_file(stage.dvcfile.relpath)
     for dep in stage.deps:
-        if not dep.use_scm_ignore and dep.is_in_repo:
+        if (
+            not dep.use_scm_ignore
+            and dep.is_in_repo
+            and not stage.repo.repo_fs.isdvc(dep.path_info)
+        ):
             stage.repo.scm.track_file(relpath(dep.path_info))
     for out in stage.outs:
         if not out.use_scm_ignore and out.is_in_repo:

dvc/scm/git/__init__.py

@@ -7,7 +7,7 @@ import shlex
 from collections.abc import Mapping
 from contextlib import contextmanager
 from functools import partialmethod
-from typing import Dict, Iterable, List, Optional, Set, Type
+from typing import Dict, Iterable, List, Optional, Set, Tuple, Type
 
 from funcy import cached_property, first
 from pathspec.patterns import GitWildMatchPattern
@@ -77,6 +77,7 @@ class Git(Base):
     GIT_DIR = ".git"
     LOCAL_BRANCH_PREFIX = "refs/heads/"
     RE_HEXSHA = re.compile(r"^[0-9A-Fa-f]{4,40}$")
+    BAD_REF_CHARS_RE = re.compile("[\177\\s~^:?*\\[]")
 
     def __init__(
         self, *args, backends: Optional[Iterable[str]] = None, **kwargs
@@ -123,6 +124,11 @@ class Git(Base):
     def is_sha(cls, rev):
         return rev and cls.RE_HEXSHA.search(rev)
 
+    @classmethod
+    def split_ref_pattern(cls, ref: str) -> Tuple[str, str]:
+        name = cls.BAD_REF_CHARS_RE.split(ref, maxsplit=1)[0]
+        return name, ref[len(name) :]
+
     @staticmethod
     def _get_git_dir(root_dir):
         return os.path.join(root_dir, Git.GIT_DIR)
@@ -145,7 +151,9 @@ class Git(Base):
         gitignore = os.path.join(ignore_file_dir, self.GITIGNORE)
 
         if not path_isin(os.path.realpath(gitignore), self.root_dir):
-            raise FileNotInRepoError(path)
+            raise FileNotInRepoError(
+                f"'{path}' is outside of git repository '{self.root_dir}'"
+            )
 
         return entry, gitignore
 
@@ -343,7 +351,7 @@ class Git(Base):
         from .objects import GitTrie
 
         resolved = self.resolve_rev(rev)
-        tree_obj = self._backend_func("get_tree_obj", rev=resolved)
+        tree_obj = self.pygit2.get_tree_obj(rev=resolved)
         trie = GitTrie(tree_obj, resolved)
         return GitFileSystem(self.root_dir, trie, **kwargs)
 
@@ -377,7 +385,7 @@ class Git(Base):
     _stash_iter = partialmethod(_backend_func, "_stash_iter")
     _stash_push = partialmethod(_backend_func, "_stash_push")
     _stash_apply = partialmethod(_backend_func, "_stash_apply")
-    reflog_delete = partialmethod(_backend_func, "reflog_delete")
+    _stash_drop = partialmethod(_backend_func, "_stash_drop")
     describe = partialmethod(_backend_func, "describe")
     diff = partialmethod(_backend_func, "diff")
     reset = partialmethod(_backend_func, "reset")
@@ -418,7 +426,7 @@ class Git(Base):
             commit = self.resolve_commit(parent)
 
     @contextmanager
-    def detach_head(self, rev: Optional[str] = None):
+    def detach_head(self, rev: Optional[str] = None, force: bool = False):
         """Context manager for performing detached HEAD SCM operations.
 
         Detaches and restores HEAD similar to interactive git rebase.
@@ -432,7 +440,7 @@ class Git(Base):
             rev = "HEAD"
         orig_head = self.get_ref("HEAD", follow=False)
         logger.debug("Detaching HEAD at '%s'", rev)
-        self.checkout(rev, detach=True)
+        self.checkout(rev, detach=True, force=force)
         try:
             yield self.get_ref("HEAD")
         finally:

dvc/scm/git/backend/base.py

@@ -67,7 +67,11 @@ class BaseGitBackend(ABC):
 
     @abstractmethod
     def checkout(
-        self, branch: str, create_new: Optional[bool] = False, **kwargs,
+        self,
+        branch: str,
+        create_new: Optional[bool] = False,
+        force: bool = False,
+        **kwargs,
     ):
         pass
 
@@ -96,7 +100,7 @@ class BaseGitBackend(ABC):
         pass
 
     @abstractmethod
-    def is_dirty(self, **kwargs) -> bool:
+    def is_dirty(self, untracked_files: bool = False) -> bool:
         pass
 
     @abstractmethod
@@ -245,17 +249,21 @@ class BaseGitBackend(ABC):
         message: Optional[str] = None,
         include_untracked: Optional[bool] = False,
     ) -> Tuple[Optional[str], bool]:
-        """Push a commit onto the specified stash."""
+        """Push a commit onto the specified stash.
+
+        Returns a tuple of the form (rev, need_reset) where need_reset
+        indicates whether or not the workspace should be `reset --hard`
+        (some backends will not clean the workspace after creating a stash
+        commit).
+        """
 
     @abstractmethod
     def _stash_apply(self, rev: str):
         """Apply the specified stash revision."""
 
     @abstractmethod
-    def reflog_delete(
-        self, ref: str, updateref: bool = False, rewrite: bool = False
-    ):
-        """Delete the specified reflog entry."""
+    def _stash_drop(self, ref: str, index: int):
+        """Drop the specified stash revision."""
 
     @abstractmethod
     def describe(

dvc/scm/git/backend/dulwich.py

@@ -182,14 +182,23 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
     def commit(self, msg: str, no_verify: bool = False):
         from dulwich.errors import CommitError
         from dulwich.porcelain import commit
+        from dulwich.repo import InvalidUserIdentity
 
         try:
             commit(self.root_dir, message=msg, no_verify=no_verify)
         except CommitError as exc:
             raise SCMError("Git commit failed") from exc
+        except InvalidUserIdentity as exc:
+            raise SCMError(
+                "Git username and email must be configured"
+            ) from exc
 
     def checkout(
-        self, branch: str, create_new: Optional[bool] = False, **kwargs,
+        self,
+        branch: str,
+        create_new: Optional[bool] = False,
+        force: bool = False,
+        **kwargs,
     ):
         raise NotImplementedError
 
@@ -222,8 +231,9 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
                 return True
         return False
 
-    def is_dirty(self, **kwargs) -> bool:
-        raise NotImplementedError
+    def is_dirty(self, untracked_files: bool = False) -> bool:
+        staged, unstaged, untracked = self.status()
+        return bool(staged or unstaged or (untracked_files and untracked))
 
     def active_branch(self) -> str:
         raise NotImplementedError
@@ -244,7 +254,10 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
         return DulwichObject(self.repo, ".", stat.S_IFDIR, tree.id)
 
     def get_rev(self) -> str:
-        raise NotImplementedError
+        rev = self.get_ref("HEAD")
+        if rev:
+            return rev
+        raise SCMError("Empty git repo")
 
     def resolve_rev(self, rev: str) -> str:
         raise NotImplementedError
@@ -520,6 +533,8 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
         message: Optional[str] = None,
         include_untracked: Optional[bool] = False,
     ) -> Tuple[Optional[str], bool]:
+        from dulwich.repo import InvalidUserIdentity
+
         from dvc.scm.git import Stash
 
         if include_untracked or ref == Stash.DEFAULT_STASH:
@@ -529,16 +544,28 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
 
         stash = self._get_stash(ref)
         message_b = message.encode("utf-8") if message else None
-        rev = stash.push(message=message_b)
+        try:
+            rev = stash.push(message=message_b)
+        except InvalidUserIdentity as exc:
+            raise SCMError(
+                "Git username and email must be configured"
+            ) from exc
         return os.fsdecode(rev), True
 
     def _stash_apply(self, rev: str):
         raise NotImplementedError
 
-    def reflog_delete(
-        self, ref: str, updateref: bool = False, rewrite: bool = False
-    ):
-        raise NotImplementedError
+    def _stash_drop(self, ref: str, index: int):
+        from dvc.scm.git import Stash
+
+        if ref == Stash.DEFAULT_STASH:
+            raise NotImplementedError
+
+        stash = self._get_stash(ref)
+        try:
+            stash.drop(index)
+        except ValueError as exc:
+            raise SCMError("Failed to drop stash entry") from exc
 
     def describe(
         self,

dvc/scm/git/backend/gitpython.py

@@ -217,12 +217,16 @@ class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
             raise SCMError("Git pre-commit hook failed") from exc
 
     def checkout(
-        self, branch: str, create_new: Optional[bool] = False, **kwargs
+        self,
+        branch: str,
+        create_new: Optional[bool] = False,
+        force: bool = False,
+        **kwargs,
     ):
         if create_new:
-            self.repo.git.checkout("HEAD", b=branch, **kwargs)
+            self.repo.git.checkout("HEAD", b=branch, force=force, **kwargs)
         else:
-            self.repo.git.checkout(branch, **kwargs)
+            self.repo.git.checkout(branch, force=force, **kwargs)
 
     def pull(self, **kwargs):
         infos = self.repo.remote().pull(**kwargs)
@@ -249,8 +253,8 @@ class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
     def is_tracked(self, path):
         return bool(self.repo.git.ls_files(path))
 
-    def is_dirty(self, **kwargs):
-        return self.repo.is_dirty(**kwargs)
+    def is_dirty(self, untracked_files: bool = False) -> bool:
+        return self.repo.is_dirty(untracked_files=untracked_files)
 
     def active_branch(self):
         return self.repo.active_branch.name
@@ -495,16 +499,22 @@ class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
                 ) from exc
             raise SCMError("Could not apply stash") from exc
 
-    def reflog_delete(
-        self, ref: str, updateref: bool = False, rewrite: bool = False
-    ):
-        args = ["delete"]
-        if updateref:
-            args.append("--updateref")
-        if rewrite:
-            args.append("--rewrite")
-        args.append(ref)
-        self.git.reflog(*args)
+    def _stash_drop(self, ref: str, index: int):
+        from git.exc import GitCommandError
+
+        from dvc.scm.git import Stash
+
+        if ref == Stash.DEFAULT_STASH:
+            self.git.stash("drop", index)
+            return
+
+        self.git.reflog(
+            "delete", "--updateref", "--rewrite", f"{ref}@{{{index}}}"
+        )
+        try:
+            self.git.reflog("exists", ref)
+        except GitCommandError:
+            self.remove_ref(ref)
 
     def describe(
         self,

dvc/scm/git/backend/pygit2.py

@@ -1,9 +1,13 @@
 import locale
 import logging
 import os
+import stat
+from contextlib import contextmanager
 from io import BytesIO, StringIO
 from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union
 
+from funcy import cached_property
+
 from dvc.scm.base import MergeConflictError, RevError, SCMError
 from dvc.utils import relpath
 
@@ -31,6 +35,8 @@ class Pygit2Object(GitObject):
 
     @property
     def mode(self):
+        if not self.obj.filemode and self.obj.type_str == "tree":
+            return stat.S_IFDIR
         return self.obj.filemode
 
     def scandir(self) -> Iterable["Pygit2Object"]:
@@ -61,12 +67,40 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         self._stashes: dict = {}
 
     def close(self):
+        if hasattr(self, "_refdb"):
+            del self._refdb
         self.repo.free()
 
     @property
     def root_dir(self) -> str:
         return self.repo.workdir
 
+    @cached_property
+    def _refdb(self):
+        from pygit2 import RefdbFsBackend
+
+        return RefdbFsBackend(self.repo)
+
+    @property
+    def default_signature(self):
+        try:
+            return self.repo.default_signature
+        except KeyError as exc:
+            raise SCMError(
+                "Git username and email must be configured"
+            ) from exc
+
+    # Workaround to force git_backend_odb_pack to release open file handles
+    # in DVC's mixed git-backend environment.
+    # See https://github.com/iterative/dvc/issues/5641
+    @contextmanager
+    def release_odb_handles(self):
+        yield
+        # It is safe to free the libgit2 repo context multiple times - free
+        # just forces libgit/pygit to release git ODB related contexts which
+        # can be reacquired later as needed.
+        self.repo.free()
+
     @staticmethod
     def clone(
         url: str,
@@ -87,27 +121,34 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         raise NotImplementedError
 
     def checkout(
-        self, branch: str, create_new: Optional[bool] = False, **kwargs,
+        self,
+        branch: str,
+        create_new: Optional[bool] = False,
+        force: bool = False,
+        **kwargs,
     ):
-        from pygit2 import GitError
+        from pygit2 import GIT_CHECKOUT_FORCE, GitError
 
-        if create_new:
-            commit = self.repo.revparse_single("HEAD")
-            new_branch = self.repo.branches.local.create(branch, commit)
-            self.repo.checkout(new_branch)
-        else:
-            if branch == "-":
-                branch = "@{-1}"
-            try:
-                commit, ref = self.repo.resolve_refish(branch)
-            except (KeyError, GitError):
-                raise RevError(f"unknown Git revision '{branch}'")
-            self.repo.checkout_tree(commit)
-            detach = kwargs.get("detach", False)
-            if ref and not detach:
-                self.repo.set_head(ref.name)
+        checkout_strategy = GIT_CHECKOUT_FORCE if force else None
+
+        with self.release_odb_handles():
+            if create_new:
+                commit = self.repo.revparse_single("HEAD")
+                new_branch = self.repo.branches.local.create(branch, commit)
+                self.repo.checkout(new_branch, strategy=checkout_strategy)
             else:
-                self.repo.set_head(commit.id)
+                if branch == "-":
+                    branch = "@{-1}"
+                try:
+                    commit, ref = self.repo.resolve_refish(branch)
+                except (KeyError, GitError):
+                    raise RevError(f"unknown Git revision '{branch}'")
+                self.repo.checkout_tree(commit, strategy=checkout_strategy)
+                detach = kwargs.get("detach", False)
+                if ref and not detach:
+                    self.repo.set_head(ref.name)
+                else:
+                    self.repo.set_head(commit.id)
 
     def pull(self, **kwargs):
         raise NotImplementedError
@@ -133,7 +174,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
     def is_tracked(self, path: str) -> bool:
         raise NotImplementedError
 
-    def is_dirty(self, **kwargs) -> bool:
+    def is_dirty(self, untracked_files: bool = False) -> bool:
         raise NotImplementedError
 
     def active_branch(self) -> str:
@@ -210,12 +251,16 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         if old_ref and old_ref != self.get_ref(name, follow=False):
             raise SCMError(f"Failed to set '{name}'")
 
-        if symbolic:
-            ref = self.repo.create_reference_symbolic(name, new_ref, True)
-        else:
-            ref = self.repo.create_reference_direct(name, new_ref, True)
         if message:
-            ref.set_target(new_ref, message)
+            self._refdb.ensure_log(name)
+        if symbolic:
+            self.repo.create_reference_symbolic(
+                name, new_ref, True, message=message
+            )
+        else:
+            self.repo.create_reference_direct(
+                name, new_ref, True, message=message
+            )
 
     def get_ref(self, name, follow: bool = True) -> Optional[str]:
         from pygit2 import GIT_REF_SYMBOLIC
@@ -302,15 +347,64 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         message: Optional[str] = None,
         include_untracked: Optional[bool] = False,
     ) -> Tuple[Optional[str], bool]:
-        raise NotImplementedError
+        from dvc.scm.git import Stash
+
+        oid = self.repo.stash(
+            self.default_signature,
+            message=message,
+            include_untracked=include_untracked,
+        )
+        commit = self.repo[oid]
+
+        if ref != Stash.DEFAULT_STASH:
+            self.set_ref(ref, commit.id, message=commit.message)
+            self.repo.stash_drop()
+        return str(oid), False
 
     def _stash_apply(self, rev: str):
-        raise NotImplementedError
+        from pygit2 import (
+            GIT_CHECKOUT_RECREATE_MISSING,
+            GIT_CHECKOUT_SAFE,
+            GitError,
+        )
 
-    def reflog_delete(
-        self, ref: str, updateref: bool = False, rewrite: bool = False
-    ):
-        raise NotImplementedError
+        from dvc.scm.git import Stash
+
+        def _apply(index):
+            try:
+                self.repo.index.read(False)
+                self.repo.stash_apply(
+                    index,
+                    strategy=GIT_CHECKOUT_SAFE | GIT_CHECKOUT_RECREATE_MISSING,
+                )
+            except GitError as exc:
+                raise MergeConflictError(
+                    "Stash apply resulted in merge conflicts"
+                ) from exc
+
+        # libgit2 stash apply only accepts refs/stash items by index. If rev is
+        # not in refs/stash, we will push it onto the stash, and then pop it
+        commit, _ref = self.repo.resolve_refish(rev)
+        stash = self.repo.references.get(Stash.DEFAULT_STASH)
+        if stash:
+            for i, entry in enumerate(stash.log()):
+                if entry.oid_new == commit.id:
+                    _apply(i)
+                    return
+
+        self.set_ref(Stash.DEFAULT_STASH, commit.id, message=commit.message)
+        try:
+            _apply(0)
+        finally:
+            self.repo.stash_drop()
+
+    def _stash_drop(self, ref: str, index: int):
+        from dvc.scm.git import Stash
+
+        if ref != Stash.DEFAULT_STASH:
+            raise NotImplementedError
+
+        self.repo.stash_drop(index)
 
     def describe(
         self,
@@ -378,24 +472,26 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
                 ]
         else:
             path_list = None
-        self.repo.checkout_index(
-            index=index, paths=path_list, strategy=strategy,
-        )
 
-        if index.conflicts and (ours or theirs):
-            for ancestor, ours_entry, theirs_entry in index.conflicts:
-                if not ancestor:
-                    continue
-                if ours:
-                    entry = ours_entry
-                    index.add(ours_entry)
-                else:
-                    entry = theirs_entry
-                path = os.path.join(self.root_dir, entry.path)
-                with open(path, "wb") as fobj:
-                    fobj.write(self.repo.get(entry.id).read_raw())
-                index.add(entry.path)
-            index.write()
+        with self.release_odb_handles():
+            self.repo.checkout_index(
+                index=index, paths=path_list, strategy=strategy,
+            )
+
+            if index.conflicts and (ours or theirs):
+                for ancestor, ours_entry, theirs_entry in index.conflicts:
+                    if not ancestor:
+                        continue
+                    if ours:
+                        entry = ours_entry
+                        index.add(ours_entry)
+                    else:
+                        entry = theirs_entry
+                    path = os.path.join(self.root_dir, entry.path)
+                    with open(path, "wb") as fobj:
+                        fobj.write(self.repo.get(entry.id).read_raw())
+                    index.add(entry.path)
+                index.write()
 
     def iter_remote_refs(self, url: str, base: Optional[str] = None):
         raise NotImplementedError
@@ -420,25 +516,26 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         if commit and not msg:
             raise SCMError("Merge commit message is required")
 
-        try:
-            self.repo.index.read(False)
-            self.repo.merge(rev)
-            self.repo.index.write()
-        except GitError as exc:
-            raise SCMError("Merge failed") from exc
+        with self.release_odb_handles():
+            try:
+                self.repo.index.read(False)
+                self.repo.merge(rev)
+                self.repo.index.write()
+            except GitError as exc:
+                raise SCMError("Merge failed") from exc
 
-        if self.repo.index.conflicts:
-            raise MergeConflictError("Merge contained conflicts")
+            if self.repo.index.conflicts:
+                raise MergeConflictError("Merge contained conflicts")
 
-        if commit:
-            user = self.repo.default_signature
-            tree = self.repo.index.write_tree()
-            merge_commit = self.repo.create_commit(
-                "HEAD", user, user, msg, tree, [self.repo.head.target, rev]
-            )
-            return str(merge_commit)
-        if squash:
-            self.repo.reset(self.repo.head.target, GIT_RESET_MIXED)
-            self.repo.state_cleanup()
-            self.repo.index.write()
+            if commit:
+                user = self.default_signature
+                tree = self.repo.index.write_tree()
+                merge_commit = self.repo.create_commit(
+                    "HEAD", user, user, msg, tree, [self.repo.head.target, rev]
+                )
+                return str(merge_commit)
+            if squash:
+                self.repo.reset(self.repo.head.target, GIT_RESET_MIXED)
+                self.repo.state_cleanup()
+                self.repo.index.write()
         return None

dvc/scm/git/stash.py

@@ -1,11 +1,9 @@
 """git stash convenience wrapper."""
 
 import logging
-import os
 from typing import Optional
 
 from dvc.scm.base import SCMError
-from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
@@ -62,19 +60,12 @@ class Stash:
         self.scm._stash_apply(rev)  # pylint: disable=protected-access
 
     def drop(self, index: int = 0):
-        ref = "{0}@{{{1}}}".format(self.ref, index)
         if index < 0 or index >= len(self):
-            raise SCMError(f"Invalid stash ref '{ref}'")
-        logger.debug("Dropping '%s'", ref)
-        self.scm.reflog_delete(ref, updateref=True, rewrite=True)
-
-        # if we removed the last reflog entry, delete the ref and reflog
-        if len(self) == 0:
-            self.scm.remove_ref(self.ref)
-            parts = self.ref.split("/")
-            reflog = os.path.join(self.scm.root_dir, ".git", "logs", *parts)
-            if os.path.exists(reflog):
-                remove(reflog)
+            raise SCMError(f"Invalid stash ref '{self.ref}@{{{index}}}'")
+        logger.debug("Dropping '%s@{%d}'", self.ref, index)
+        self.scm._stash_drop(  # pylint: disable=protected-access
+            self.ref, index
+        )
 
     def clear(self):
         logger.debug("Clear stash '%s'", self.ref)

dvc/stage/__init__.py

@@ -433,15 +433,21 @@ class Stage(params.StageParams):
         return m
 
     def save(self, allow_missing=False):
-        self.save_deps()
+        self.save_deps(allow_missing=allow_missing)
         self.save_outs(allow_missing=allow_missing)
         self.md5 = self.compute_md5()
 
         self.repo.stage_cache.save(self)
 
-    def save_deps(self):
+    def save_deps(self, allow_missing=False):
+        from dvc.dependency.base import DependencyDoesNotExistError
+
         for dep in self.deps:
-            dep.save()
+            try:
+                dep.save()
+            except DependencyDoesNotExistError:
+                if not allow_missing:
+                    raise
 
     def save_outs(self, allow_missing=False):
         from dvc.output.base import OutputDoesNotExistError

dvc/stage/imports.py

@@ -12,10 +12,9 @@ def _update_import_on_remote(stage, remote, jobs):
             "be updated with --to-remote"
         )
 
-    url = stage.deps[0].path_info.url
-    stage.outs[0].hash_info = stage.repo.cloud.transfer(
-        url, jobs=jobs, remote=remote, command="update"
-    )
+    url = stage.deps[0].def_path
+    remote = stage.repo.cloud.get_remote(remote, "update")
+    stage.outs[0].transfer(url, odb=remote.odb, jobs=jobs, update=True)
 
 
 def update_import(stage, rev=None, to_remote=False, remote=None, jobs=None):

dvc/state.py

@@ -76,7 +76,10 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
         if not tmp_dir:
             return
 
-        config = {"eviction_policy": "least-recently-used"}
+        config = {
+            "eviction_policy": "least-recently-used",
+            "disk_pickle_protocol": 4,
+        }
         self.links = Cache(directory=os.path.join(tmp_dir, "links"), **config)
         self.md5s = Cache(directory=os.path.join(tmp_dir, "md5s"), **config)