dvc: bump to 2.0.18

This part of logic used to be only used by external repo for cloned
repos, but since we now handle local repos without cloning, we don't
always pass scm instance. This PR automatically creates scm instance
for such cases.

Fixes #5590

.github/ISSUE_TEMPLATE/bug_report.md

@@ -57,5 +57,8 @@ $ dvc doctor
 **Additional Information (if any):**
 
 <!--
+Please check https://github.com/iterative/dvc/wiki/Debugging-DVC on ways to gather more information regarding the issue.
+
 If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
+If the issue is regarding the performance, please attach the profiling information and the benchmark comparisons.
 -->

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+
+updates:
+  - directory: "/"
+    package-ecosystem: "pip"
+    schedule:
+      interval: "daily"
+
+  - directory: "/"
+    package-ecosystem: "github-actions"
+    schedule:
+      interval: "daily"

.github/workflows/packages.yaml

@@ -6,91 +6,130 @@ on:
   release:
     types: [released, prereleased]
 jobs:
-  linux:
+  deb:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+    - uses: actions/[email protected].4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2.2.2
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Set up Ruby 2.4
       uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.4
-    - name: Build
-      run: ./scripts/build_posix.sh
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build deb
+      run: python scripts/build.py deb
     - name: Publish deb
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_path: scripts/fpm/dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_content_type: binary/octet-stream
-    - name: Publish rpm
-      if: github.event_name == 'release'
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
-        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
-        asset_content_type: binary/octet-stream
-  mac:
-    runs-on: macos-10.15
+  rpm:
+    runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+    - uses: actions/[email protected].4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2.2.2
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Set up Ruby 2.4
       uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.4
-    - name: Build
-      run: ./scripts/build_posix.sh
-    - name: Publish
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build rpm
+      run: python scripts/build.py rpm
+    - name: Publish rpm
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_content_type: binary/octet-stream
+  osxpkg:
+    runs-on: macos-10.15
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Install deps
+      run: |
+        # available lxml (webdav dependency) wheels are built with an old
+        # SDK, which breaks osxpkg notarization. Building from scratch
+        # instead.
+        pip install .[all] --no-binary lxml
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build
+      run: python scripts/build.py osxpkg
+    - name: Publish
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}.pkg
         asset_name: dvc-${{ github.event.release.tag_name }}.pkg
         asset_content_type: binary/octet-stream
-  windows:
+  exe:
     runs-on: windows-2019
     steps:
-    - uses: actions/[email protected].2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+    - uses: actions/[email protected].4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2.2.2
       with:
-        python-version: 3.7
+        python-version: 3.8
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+    - name: Pull images
+      run: |
+        dvc pull
     - name: Build
-      run: scripts\build_windows.cmd
+      run: python scripts\build.py exe
       shell: cmd
     - name: Publish
       if: github.event_name == 'release'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}.exe
+        asset_path: scripts/innosetup/dvc-${{ github.event.release.tag_name }}.exe
         asset_name: dvc-${{ github.event.release.tag_name }}.exe
         asset_content_type: binary/octet-stream
-  python:
+  pip:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.7
     - name: Install
@@ -103,29 +142,3 @@ jobs:
       with:
         user: __token__
         password: ${{ secrets.PYPI_TOKEN }}
-  snap:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/[email protected]
-    - name: Build the package
-      uses: snapcore/action-build@v1
-      id: snapcraft
-#    - name: Test the package
-#      run: |
-#        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
-#        dvc version
-#        sudo snap remove dvc
-    - name: Publish to the edge channel
-      uses: snapcore/action-publish@v1
-      if: github.event_name == 'push'
-      with:
-        store_login: ${{ secrets.STORE_LOGIN }}
-        snap: ${{ steps.snapcraft.outputs.snap }}
-        release: edge,v2/edge
-    - name: Publish to the stable channel
-      uses: snapcore/action-publish@v1
-      if: ${{ github.event_name == 'release' && github.event.release.released }}
-      with:
-        store_login: ${{ secrets.STORE_LOGIN }}
-        snap: ${{ steps.snapcraft.outputs.snap }}
-        release: stable,v2/stable

.github/workflows/tests.yaml

@@ -21,20 +21,22 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.8
     - name: Install requirements
-      run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
+      run: |
+        pip install wheel
+        pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
     - name: install temporary dependencies
       run: |
         pip install git+https://github.com/isidentical/gdrivefs@service-account
         pip install git+https://github.com/isidentical/pydata-google-auth@service-account
     - name: Check README
       run: python setup.py checkdocs
-    - uses: pre-commit/[email protected].0
+    - uses: pre-commit/[email protected].2
   tests:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -51,9 +53,9 @@ jobs:
             pyv: "3.9"
 
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: ${{ matrix.pyv }}
     - name: get pip cache dir
@@ -62,7 +64,7 @@ jobs:
         echo "::set-output name=dir::$(pip cache dir)"
     - name: set pip cache
       id: pip-cache
-      uses: actions/cache@v2
+      uses: actions/cache@v2.1.5
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
@@ -88,20 +90,9 @@ jobs:
         mkdir -p scripts/ci
         echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
     - name: run tests
-      run: python -m tests --all --cov-report=xml --cov-report=term --tap-combined -n=4
+      run: python -m tests --all --cov-report=xml --cov-report=term -n=4
     - name: upload coverage report
-      uses: codecov/codecov-action@v1.0.13
+      uses: codecov/codecov-action@v1.3.2
       with:
         file: ./coverage.xml
         fail_ci_if_error: false
-    - name: Post TAP
-      # Running custom fork with an option, that does not fail CI on error
-      uses: skshetry/flaky-service/packages/action@master
-      if: always() && github.ref == 'refs/heads/master'
-      with:
-        file-path: ${{github.workspace}}/testresults.tap
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        os: ${{runner.os}}
-        matrix: ${{toJson(matrix)}}
-        repo-description: ${{github.event.repository.description}}
-        fail_ci_if_error: false

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 5.6.4
+    rev: 5.8.0
   - hooks:
       - id: flake8
         language_version: python3
@@ -18,13 +18,13 @@ repos:
           - flake8-debugger
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 3.9.0
   - hooks:
       -  id: mypy
          language_version: python3
          files: ^dvc/
     repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.790'
+    rev: 'v0.812'
   - repo: local
     hooks:
     - id: pylint

.pylintrc

@@ -15,10 +15,13 @@ ignore-patterns=
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-init-hook="import sys; \
-           from pathlib import Path; \
-           from pylint.config import find_pylintrc; \
-           sys.path.append(Path(find_pylintrc()).parent / 'tests');"
+init-hook="import sys;
+           from pathlib import Path;
+           from pylint.config import find_pylintrc;
+
+           tests_folder = Path(find_pylintrc()).parent / 'tests'
+           sys.path.append(str(tests_folder));
+           "
 
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
@@ -178,7 +181,10 @@ disable=format,
         # we use it to speedup/optimize
         import-outside-toplevel,
         fixme,
-
+        # we use cause of the exception for error message output
+        # so some of the times we don't use `raise from` to not show the cause
+        # of the error
+        raise-missing-from,
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

MANIFEST.in

@@ -1,2 +1,3 @@
 include fastentrypoints.py
 include LICENSE
+include test_requirements.txt

README.rst

@@ -129,7 +129,11 @@ Conda (Anaconda)
 
 .. code-block:: bash
 
-   conda install -c conda-forge dvc
+   conda install -c conda-forge mamba # installs much faster than conda
+   mamba install -c conda-forge dvc
+
+Depending on the remote storage type you plan to use to keep and share your data, you might need to
+install optional dependencies: `dvc-s3`, `dvc-azure`, `dvc-gdrive`, `dvc-gs`, `dvc-oss`, `dvc-ssh`.
 
 pip (PyPI)
 ----------
@@ -222,7 +226,7 @@ Citation
 Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
 `DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
 
-Barrak, A., Eghan, E.E. and Adams, B. On the Co-evolution of ML Pipelines and Source Code - Empirical Study of DVC Projects, in Proceedings of the 28th IEEE International Conference on Software Analysis, Evolution, and Reengineering, SANER 2021. Hawaii, USA, to appear.
+Barrak, A., Eghan, E.E. and Adams, B. `On the Co-evolution of ML Pipelines and Source Code - Empirical Study of DVC Projects <https://mcis.cs.queensu.ca/publications/2021/saner.pdf>`_ , in Proceedings of the 28th IEEE International Conference on Software Analysis, Evolution, and Reengineering, SANER 2021. Hawaii, USA.
 
 
 .. |Banner| image:: https://dvc.org/img/logo-github-readme.png

debian/changelog

@@ -1,5 +0,0 @@
-dvc (0.75.b1) experimental; urgency=low
-
-  * First native Debian package version.
-
- -- Dmitry Petrov <[email protected]>  Sat, 07 Dec 2019 09:49:28 +0300

debian/compat

@@ -1 +0,0 @@
-10

debian/control

@@ -1,33 +0,0 @@
-Source: dvc
-Maintainer: Dmitry Petrov <[email protected]>
-Section: python
-Priority: optional
-Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)
-Standards-Version: 4.0.0
-Homepage: http://dvc.org
-Vcs-Git: https://github.com/iterative/dvc.git
-Vcs-browser: https://github.com/iterative/dvc
-
-Package: dvc
-Architecture: all
-Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2)
-Recommends: python3-arrow (=0.14.0),
-            python3-paramiko (>=2.5.0),
-            python3-gssapi (>= 2.5.0),
-            python3-boto3 (>=1.9.86),
-            python3-pydrive (=1.3.1),
-            python3-azure-storage-blob (=12.1.0),
-            python3-oss2 (=2.6.1),
-            python3-google-cloud-storage (=1.19.0)
-Description: Git for data scientists - manage your code and data together
- Data Version Control or DVC is an open-source tool for data science and machine learning projects.
- .
- Key features:
- .
- 1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services.
- .
- 2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
- .
- 3. Makes projects reproducible and shareable; helping to answer questions about how a model was built.
- .
- 4. Helps manage experiments with Git tags/branches and metrics tracking.

debian/copyright

@@ -1,11 +0,0 @@
-Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: DVC
-Upstream-Contact: Dmitry Petrov <[email protected]>
-Source: https://github.com/iterative/dvc
-
-Files: *
-Copyright: Dmitry Petrov <[email protected]>
-License: Apache 2.0
- On Debian systems, the full text of the Apache 2.0
- License can be found in the file
- `/usr/share/common-licenses/Apache-2.0`.

debian/rules

@@ -1,6 +0,0 @@
-#!/usr/bin/make -f
-
-export PYBUILD_NAME=dvc
-%:
-	dh $@ --with python3 --buildsystem=pybuild
-

debian/source/format

@@ -1 +0,0 @@
-3.0 (native)

debian/watch

@@ -1,4 +0,0 @@
-# please also check http://pypi.debian.net/dvc/watch
-version=3
-opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
-http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))

dvc/api.py

@@ -4,7 +4,6 @@ from contextlib import _GeneratorContextManager as GCM
 from funcy import reraise
 
 from dvc.exceptions import OutputNotFoundError, PathMissingError
-from dvc.objects.stage import get_hash
 from dvc.path_info import PathInfo
 from dvc.repo import Repo
 
@@ -29,8 +28,8 @@ def get_url(path, repo=None, rev=None, remote=None):
             raise OutputNotFoundError(path, repo)
 
         cloud = metadata.repo.cloud
-        hash_info = get_hash(path_info, _repo.repo_fs, "md5", _repo.odb)
-        return cloud.get_url_for(remote, checksum=hash_info.value)
+        md5 = metadata.repo.dvcfs.info(path_info)["md5"]
+        return cloud.get_url_for(remote, checksum=md5)
 
 
 def open(  # noqa, pylint: disable=redefined-builtin

dvc/checkout.py

@@ -9,8 +9,9 @@ from dvc.exceptions import (
     ConfirmRemoveError,
     DvcException,
 )
-from dvc.objects import ObjectFormatError, check, load
-from dvc.objects.stage import get_hash
+from dvc.objects import check, load
+from dvc.objects.errors import ObjectFormatError
+from dvc.objects.stage import stage
 from dvc.remote.slow_link_detection import (  # type: ignore[attr-defined]
     slow_link_guard,
 )
@@ -30,7 +31,7 @@ def _changed(path_info, fs, obj, cache):
         return True
 
     try:
-        actual = get_hash(path_info, fs, obj.hash_info.name, cache)
+        actual = stage(cache, path_info, fs, obj.hash_info.name).hash_info
     except FileNotFoundError:
         logger.debug("'%s' doesn't exist.", path_info)
         return True
@@ -56,7 +57,7 @@ def _remove(path_info, fs, cache, force=False):
         fs.remove(path_info)
         return
 
-    current = get_hash(path_info, fs, fs.PARAM_CHECKSUM, cache)
+    current = stage(cache, path_info, fs, fs.PARAM_CHECKSUM).hash_info
     try:
         obj = load(cache, current)
         check(cache, obj)
@@ -174,10 +175,10 @@ def _checkout_file(
     return modified
 
 
-def _remove_redundant_files(path_info, fs, dir_info, cache, force):
+def _remove_redundant_files(path_info, fs, obj, cache, force):
     existing_files = set(fs.walk_files(path_info))
 
-    needed_files = {path_info.joinpath(*key) for key, _ in dir_info.items()}
+    needed_files = {path_info.joinpath(*key) for key, _ in obj}
     redundant_files = existing_files - needed_files
     for path in redundant_files:
         _remove(path, fs, cache, force)
@@ -197,11 +198,11 @@ def _checkout_dir(
 
     logger.debug("Linking directory '%s'.", path_info)
 
-    for entry_key, entry_hash_info in obj.hash_info.dir_info.items():
+    for entry_key, entry_obj in obj:
         entry_modified = _checkout_file(
             path_info.joinpath(*entry_key),
             fs,
-            cache.get(entry_hash_info),
+            entry_obj,
             cache,
             force,
             progress_callback,
@@ -211,10 +212,7 @@ def _checkout_dir(
             modified = True
 
     modified = (
-        _remove_redundant_files(
-            path_info, fs, obj.hash_info.dir_info, cache, force
-        )
-        or modified
+        _remove_redundant_files(path_info, fs, obj, cache, force) or modified
     )
 
     fs.repo.state.save(path_info, fs, obj.hash_info)

dvc/command/experiments.py

@@ -3,7 +3,7 @@ import logging
 from collections import Counter, OrderedDict, defaultdict
 from collections.abc import Mapping
 from datetime import date, datetime
-from itertools import groupby
+from fnmatch import fnmatch
 from typing import Dict, Iterable, Optional
 
 import dvc.prompt as prompt
@@ -27,32 +27,23 @@ def _filter_name(names, label, filter_strs):
 
     for filter_s in filter_strs:
         path, _, name = filter_s.rpartition(":")
-        path_filters[path].append(tuple(name.split(".")))
+        path_filters[path].append(name)
 
     for path, filters in path_filters.items():
         if path:
             match_paths = [path]
         else:
             match_paths = names.keys()
-        for length, groups in groupby(filters, len):
-            for group in groups:
-                for match_path in match_paths:
-                    possible_names = [
-                        tuple(name.split(".")) for name in names[match_path]
-                    ]
-                    matches = [
-                        name
-                        for name in possible_names
-                        if name[:length] == group
-                    ]
-                    if not matches:
-                        name = ".".join(group)
-                        raise InvalidArgumentError(
-                            f"'{name}' does not match any known {label}"
-                        )
-                    ret[match_path].update(
-                        {".".join(match): None for match in matches}
+        for match_path in match_paths:
+            for f in filters:
+                matches = [
+                    name for name in names[match_path] if fnmatch(name, f)
+                ]
+                if not matches:
+                    raise InvalidArgumentError(
+                        f"'{f}' does not match any known {label}"
                     )
+                ret[match_path].update({match: None for match in matches})
 
     return ret
 
@@ -103,7 +94,6 @@ def _collect_names(all_experiments, **kwargs):
         for exp in experiments.values():
             _update_names(metric_names, exp.get("metrics", {}).items())
             _update_names(param_names, exp.get("params", {}).items())
-
     metric_names = _filter_names(
         metric_names,
         "metrics",
@@ -268,7 +258,8 @@ def _extend_row(row, names, items, precision):
     from rich.text import Text
 
     if not items:
-        row.extend(["-"] * len(names))
+        for keys in names.values():
+            row.extend(["-"] * len(keys))
         return
 
     for fname, item in items:
@@ -374,6 +365,7 @@ class CmdExperimentsShow(CmdBase):
                 all_commits=self.args.all_commits,
                 sha_only=self.args.sha,
                 num=self.args.num,
+                param_deps=self.args.param_deps,
             )
 
             if self.args.show_json:
@@ -408,7 +400,9 @@ class CmdExperimentsShow(CmdBase):
                 # NOTE: rich does not have native support for unlimited width
                 # via pager. we override rich table compression by setting
                 # console width to the full width of the table
-                measurement = table.__rich_measure__(console, SHOW_MAX_WIDTH)
+                console_options = console.options
+                console_options.max_width = SHOW_MAX_WIDTH
+                measurement = table.__rich_measure__(console, console_options)
                 console._width = (  # pylint: disable=protected-access
                     measurement.maximum
                 )
@@ -477,6 +471,7 @@ class CmdExperimentsDiff(CmdBase):
                 a_rev=self.args.a_rev,
                 b_rev=self.args.b_rev,
                 all=self.args.all,
+                param_deps=self.args.param_deps,
             )
 
             if self.args.show_json:
@@ -514,6 +509,11 @@ class CmdExperimentsRun(CmdRepro):
     def run(self):
         from dvc.command.metrics import _show_metrics
 
+        if self.args.reset and self.args.checkpoint_resume:
+            raise InvalidArgumentError(
+                "--reset and --rev are mutually exclusive."
+            )
+
         if self.args.reset:
             logger.info("Any existing checkpoints will be reset and re-run.")
 
@@ -783,6 +783,12 @@ def add_parser(subparsers, parent_parser):
         help="Exclude the specified params from output table.",
         metavar="<params_list>",
     )
+    experiments_show_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     experiments_show_parser.add_argument(
         "--sort-by",
         help="Sort related experiments by the specified metric or param.",
@@ -868,6 +874,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show unchanged metrics/params as well.",
     )
+    experiments_diff_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     experiments_diff_parser.add_argument(
         "--show-json",
         action="store_true",
@@ -1215,6 +1227,7 @@ def _add_run_common(parser):
         "-j",
         "--jobs",
         type=int,
+        default=1,
         help="Run the specified number of experiments at a time in parallel.",
         metavar="<number>",
     )

dvc/command/params.py

@@ -40,6 +40,7 @@ class CmdParamsDiff(CmdBase):
                 b_rev=self.args.b_rev,
                 targets=self.args.targets,
                 all=self.args.all,
+                deps=self.args.deps,
             )
 
             if self.args.show_json:
@@ -111,6 +112,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show unchanged params as well.",
     )
+    params_diff_parser.add_argument(
+        "--deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     params_diff_parser.add_argument(
         "--show-json",
         action="store_true",

dvc/command/plots.py

@@ -1,6 +1,5 @@
 import argparse
 import logging
-import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
@@ -23,6 +22,8 @@ class CmdPlots(CmdBase):
         return {k: v for k, v in props.items() if v is not None}
 
     def run(self):
+        from pathlib import Path
+
         if self.args.show_vega:
             if not self.args.targets:
                 logger.error("please specify a target for `--show-vega`")
@@ -41,17 +42,24 @@ class CmdPlots(CmdBase):
                 logger.info(plots[target])
                 return 0
 
-            path = self.args.out or "plots.html"
-            path = os.path.join(os.getcwd(), path)
-
+            rel: str = self.args.out or "plots.html"
+            path = (Path.cwd() / rel).resolve()
             write(path, plots)
-
-            logger.info(f"file://{path}")
-
         except DvcException:
             logger.exception("")
             return 1
 
+        assert path.is_absolute()  # as_uri throws ValueError if not absolute
+        url = path.as_uri()
+        logger.info(url)
+        if self.args.open:
+            import webbrowser
+
+            opened = webbrowser.open(rel)
+            if not opened:
+                logger.error("Failed to open. Please try opening it manually.")
+                return 1
+
         return 0
 
 
@@ -229,3 +237,9 @@ def _add_output_arguments(parser):
         default=False,
         help="Show output in Vega format.",
     )
+    parser.add_argument(
+        "--open",
+        action="store_true",
+        default=False,
+        help="Open plot file directly in the browser.",
+    )

dvc/command/stage.py

@@ -255,7 +255,8 @@ def _add_common_args(parser):
         "--checkpoints",
         action="append",
         default=[],
-        help=argparse.SUPPRESS,
+        help="Declare checkpoint output file or directory for 'dvc exp run'. "
+        "Not compatible with 'dvc repro'.",
         metavar="<filename>",
     ).complete = completion.FILE
     parser.add_argument(

dvc/command/status.py

@@ -44,19 +44,27 @@ class CmdDataStatus(CmdDataBase):
                 self._show(value, indent + 1)
 
     def run(self):
+        from dvc.repo import lock_repo
+        from dvc.ui import ui
+
         indent = 1 if self.args.cloud else 0
-        try:
-            st = self.repo.status(
-                targets=self.args.targets,
-                jobs=self.args.jobs,
-                cloud=self.args.cloud,
-                remote=self.args.remote,
-                all_branches=self.args.all_branches,
-                all_tags=self.args.all_tags,
-                all_commits=self.args.all_commits,
-                with_deps=self.args.with_deps,
-                recursive=self.args.recursive,
-            )
+
+        with lock_repo(self.repo):
+            try:
+                st = self.repo.status(
+                    targets=self.args.targets,
+                    jobs=self.args.jobs,
+                    cloud=self.args.cloud,
+                    remote=self.args.remote,
+                    all_branches=self.args.all_branches,
+                    all_tags=self.args.all_tags,
+                    all_commits=self.args.all_commits,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("")
+                return 1
 
             if self.args.quiet:
                 return bool(st)
@@ -65,19 +73,21 @@ class CmdDataStatus(CmdDataBase):
                 import json
 
                 logger.info(json.dumps(st))
-            elif st:
+                return 0
+
+            if st:
                 self._show(st, indent)
-            elif not self.repo.stages:
-                logger.info(self.EMPTY_PROJECT_MSG)
+                return 0
+
+            # additional hints for the user
+            if not self.repo.stages:
+                ui.write(self.EMPTY_PROJECT_MSG)
             elif self.args.cloud or self.args.remote:
                 remote = self.args.remote or self.repo.config["core"].get(
                     "remote"
                 )
-                logger.info(self.IN_SYNC_MSG.format(remote=remote))
+                ui.write(self.IN_SYNC_MSG.format(remote=remote))
             else:
-                logger.info(self.UP_TO_DATE_MSG)
+                ui.write(self.UP_TO_DATE_MSG)
 
-        except DvcException:
-            logger.exception("")
-            return 1
         return 0

dvc/config_schema.py

@@ -145,6 +145,7 @@ SCHEMA = {
                     "session_token": str,
                     Optional("listobjects", default=False): Bool,  # obsoleted
                     Optional("use_ssl", default=True): Bool,
+                    Optional("ssl_verify", default=True): Bool,
                     "sse": str,
                     "sse_kms_key_id": str,
                     "acl": str,

dvc/dependency/param.py

@@ -39,6 +39,7 @@ class ParamsDependency(LocalDependency):
             path
             or os.path.join(stage.repo.root_dir, self.DEFAULT_PARAMS_FILE),
             info=info,
+            fs=stage.repo.fs,
         )
 
     def dumpd(self):
@@ -84,19 +85,34 @@ class ParamsDependency(LocalDependency):
     def status(self):
         return self.workspace_status()
 
-    def read_params(self):
+    def _read(self):
         if not self.exists:
             return {}
 
         suffix = self.path_info.suffix.lower()
         loader = LOADERS[suffix]
         try:
-            config = loader(self.path_info, fs=self.repo.fs)
+            return loader(self.path_info, fs=self.repo.fs)
         except ParseError as exc:
             raise BadParamFileError(
                 f"Unable to read parameters from '{self}'"
             ) from exc
 
+    def read_params_d(self):
+        config = self._read()
+
+        ret = {}
+        for param in self.params:
+            dpath.util.merge(
+                ret,
+                dpath.util.search(config, param, separator="."),
+                separator=".",
+            )
+        return ret
+
+    def read_params(self):
+        config = self._read()
+
         ret = {}
         for param in self.params:
             try:

dvc/dependency/repo.py

@@ -55,6 +55,7 @@ class RepoDependency(LocalDependency):
                 self.repo.odb.local,
                 path_info,
                 repo.repo_fs,
+                self.repo.odb.local.fs.PARAM_CHECKSUM,
                 follow_subrepos=False,
             ).hash_info
 
@@ -94,7 +95,12 @@ class RepoDependency(LocalDependency):
             except (NoOutputOrStageError, NoRemoteError):
                 pass
             obj = stage(
-                odb, path_info, repo.repo_fs, jobs=jobs, follow_subrepos=False,
+                odb,
+                path_info,
+                repo.repo_fs,
+                odb.fs.PARAM_CHECKSUM,
+                jobs=jobs,
+                follow_subrepos=False,
             )
             save(odb, obj, jobs=jobs)
 

dvc/dir_info.py

@@ -1,117 +0,0 @@
-import posixpath
-from operator import itemgetter
-
-from funcy import cached_property
-from pygtrie import Trie
-
-from .hash_info import HashInfo
-
-
-def _diff(ancestor, other, allow_removed=False):
-    from dictdiffer import diff
-
-    from dvc.exceptions import MergeError
-
-    allowed = ["add"]
-    if allow_removed:
-        allowed.append("remove")
-
-    result = list(diff(ancestor, other))
-    for typ, _, _ in result:
-        if typ not in allowed:
-            raise MergeError(
-                "unable to auto-merge directories with diff that contains "
-                f"'{typ}'ed files"
-            )
-    return result
-
-
-def _merge(ancestor, our, their):
-    import copy
-
-    from dictdiffer import patch
-
-    our_diff = _diff(ancestor, our)
-    if not our_diff:
-        return copy.deepcopy(their)
-
-    their_diff = _diff(ancestor, their)
-    if not their_diff:
-        return copy.deepcopy(our)
-
-    # make sure there are no conflicting files
-    _diff(our, their, allow_removed=True)
-
-    return patch(our_diff + their_diff, ancestor)
-
-
-class DirInfo:
-    PARAM_RELPATH = "relpath"
-
-    def __init__(self):
-        self._dict = {}
-
-    def as_dict(self):
-        return self._dict.copy()
-
-    def add(self, key, hash_info):
-        self.__dict__.pop("trie", None)
-        self._dict[key] = hash_info
-
-    def get(self, key, default=None):
-        return self._dict.get(key, default)
-
-    @cached_property
-    def trie(self):
-        return Trie(self._dict)
-
-    @property
-    def size(self):
-        try:
-            return sum(hash_info.size for _, hash_info in self._dict.items())
-        except TypeError:
-            return None
-
-    @property
-    def nfiles(self):
-        return len(self._dict)
-
-    def items(self):
-        for key, hash_info in self._dict.items():
-            yield key, hash_info
-
-    @classmethod
-    def from_list(cls, lst):
-        ret = DirInfo()
-        for _entry in lst:
-            entry = _entry.copy()
-            relpath = entry.pop(cls.PARAM_RELPATH)
-            parts = tuple(relpath.split(posixpath.sep))
-            hash_info = HashInfo.from_dict(entry)
-            ret.add(parts, hash_info)
-        return ret
-
-    def to_list(self):
-        # Sorting the list by path to ensure reproducibility
-        return sorted(
-            (
-                {
-                    # NOTE: not using hash_info.to_dict() because we don't want
-                    # size/nfiles fields at this point.
-                    hash_info.name: hash_info.value,
-                    self.PARAM_RELPATH: posixpath.sep.join(parts),
-                }
-                for parts, hash_info in self._dict.items()  # noqa: B301
-            ),
-            key=itemgetter(self.PARAM_RELPATH),
-        )
-
-    def merge(self, ancestor, their):
-        merged_dict = _merge(
-            ancestor.as_dict(), self.as_dict(), their.as_dict(),
-        )
-
-        merged = DirInfo()
-        for key, hi in merged_dict.items():
-            merged.add(key, hi)
-        return merged

dvc/fs/azure.py

@@ -48,11 +48,12 @@ class AzureFileSystem(FSSpecWrapper):
         self.path_info = self.PATH_CLS(url)
         self.bucket = self.path_info.bucket
 
-        self.login_method, self.login_info = self._prepare_credentials(config)
-
     def _prepare_credentials(self, config):
         from azure.identity.aio import DefaultAzureCredential
 
+        # Disable spam from failed cred types for DefaultAzureCredential
+        logging.getLogger("azure.identity.aio").setLevel(logging.ERROR)
+
         login_info = {}
         login_info["connection_string"] = config.get(
             "connection_string",
@@ -71,8 +72,12 @@ class AzureFileSystem(FSSpecWrapper):
         login_info["client_id"] = config.get("client_id")
         login_info["client_secret"] = config.get("client_secret")
 
-        if not any(login_info.values()):
-            login_info["credentials"] = DefaultAzureCredential()
+        if not any(
+            value for key, value in login_info.items() if key != "account_name"
+        ):
+            login_info["credential"] = DefaultAzureCredential(
+                exclude_interactive_browser_credential=False
+            )
 
         for login_method, required_keys in [  # noqa
             ("connection string", ["connection_string"]),
@@ -82,15 +87,19 @@ class AzureFileSystem(FSSpecWrapper):
             ),
             ("account key", ["account_name", "account_key"]),
             ("SAS token", ["account_name", "sas_token"]),
+            (
+                f"default credentials ({_DEFAULT_CREDS_STEPS})",
+                ["account_name", "credential"],
+            ),
             ("anonymous login", ["account_name"]),
-            (f"default credentials ({_DEFAULT_CREDS_STEPS})", ["credentials"]),
         ]:
             if all(login_info.get(key) is not None for key in required_keys):
                 break
         else:
             login_method = None
 
-        return login_method, login_info
+        self.login_method = login_method
+        return login_info
 
     @cached_property
     def _az_config(self):
@@ -111,7 +120,7 @@ class AzureFileSystem(FSSpecWrapper):
         from azure.core.exceptions import AzureError
 
         try:
-            file_system = AzureBlobFileSystem(**self.login_info)
+            file_system = AzureBlobFileSystem(**self.fs_args)
             if self.bucket not in [
                 container.rstrip("/") for container in file_system.ls("/")
             ]:

dvc/fs/base.py

@@ -1,6 +1,6 @@
 import logging
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from functools import partial
+from functools import partial, partialmethod
 from multiprocessing import cpu_count
 from typing import Any, ClassVar, Dict, FrozenSet, Optional
 
@@ -38,7 +38,7 @@ class BaseFileSystem:
     scheme = "base"
     REQUIRES: ClassVar[Dict[str, str]] = {}
     PATH_CLS = URLInfo  # type: Any
-    JOBS = 4 * cpu_count()
+    _JOBS = 4 * cpu_count()
 
     CHECKSUM_DIR_SUFFIX = ".dir"
     HASH_JOBS = max(1, min(4, cpu_count() // 2))
@@ -67,7 +67,7 @@ class BaseFileSystem:
         return (
             self.config.get("jobs")
             or (self.repo and self.repo.config["core"].get("jobs"))
-            or self.JOBS
+            or self._JOBS
         )
 
     @cached_property
@@ -258,6 +258,7 @@ class BaseFileSystem:
         name=None,
         no_progress_bar=False,
         jobs=None,
+        _only_file=False,
         **kwargs,
     ):
         if not hasattr(self, "_download"):
@@ -273,16 +274,21 @@ class BaseFileSystem:
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        if self.isdir(from_info):
+        if not _only_file and self.isdir(from_info):
             return self._download_dir(
                 from_info, to_info, name, no_progress_bar, jobs, **kwargs,
             )
         return self._download_file(from_info, to_info, name, no_progress_bar,)
 
+    download_file = partialmethod(download, _only_file=True)
+
     def _download_dir(
         self, from_info, to_info, name, no_progress_bar, jobs, **kwargs,
     ):
         from_infos = list(self.walk_files(from_info, **kwargs))
+        if not from_infos:
+            makedirs(to_info, exist_ok=True)
+            return None
         to_infos = (
             to_info / info.relative_to(from_info) for info in from_infos
         )

dvc/fs/dvc.py

@@ -48,9 +48,9 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         # NOTE: use string paths here for performance reasons
         key = tuple(relpath(path_info, out.path_info).split(os.sep))
         out.get_dir_cache(remote=remote)
-        file_hash = out.hash_info.dir_info.get(key)
-        if file_hash:
-            return file_hash
+        obj = out.obj.trie.get(key)
+        if obj:
+            return obj.hash_info
         raise FileNotFoundError
 
     def open(  # type: ignore
@@ -131,21 +131,14 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         # pull dir cache if needed
         out.get_dir_cache(**kwargs)
 
-        dir_cache = out.dir_cache
-        if not dir_cache:
-            raise FileNotFoundError
-
-        from dvc.objects import Tree
-
-        hash_info = Tree.save_dir_info(out.odb, dir_cache)
-        if hash_info != out.hash_info:
+        if not out.obj:
             raise FileNotFoundError
 
     def _add_dir(self, trie, out, **kwargs):
         self._fetch_dir(out, **kwargs)
 
         base = out.path_info.parts
-        for key, _ in out.dir_cache.items():  # noqa: B301
+        for key, _ in out.obj:  # noqa: B301
             trie[base + key] = None
 
     def _walk(self, root, trie, topdown=True, **kwargs):
@@ -241,9 +234,9 @@ class DvcFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         elif meta.part_of_output:
             (out,) = meta.outs
             key = path_info.relative_to(out.path_info).parts
-            hash_info = out.hash_info.dir_info.trie.get(key)
-            if hash_info:
-                ret["size"] = hash_info.size
-                ret[hash_info.name] = hash_info.value
+            obj = out.obj.trie.get(key)
+            if obj:
+                ret["size"] = obj.size
+                ret[obj.hash_info.name] = obj.hash_info.value
 
         return ret

dvc/fs/fsspec_wrapper.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+from functools import lru_cache
 
 from dvc.progress import Tqdm
 
@@ -8,17 +9,29 @@ from .base import BaseFileSystem
 
 # pylint: disable=no-member
 class FSSpecWrapper(BaseFileSystem):
+    def __init__(self, repo, config):
+        super().__init__(repo, config)
+        self.fs_args = {"skip_instance_cache": True}
+        self.fs_args.update(self._prepare_credentials(config))
+
     def fs(self):
         raise NotImplementedError
 
+    @lru_cache(512)
     def _with_bucket(self, path):
         if isinstance(path, self.PATH_CLS):
             return f"{path.bucket}/{path.path}"
         return path
 
     def _strip_bucket(self, entry):
-        _, entry = entry.split("/", 1)
-        return entry
+        try:
+            bucket, path = entry.split("/", 1)
+        except ValueError:
+            # If there is no path attached, only returns
+            # the bucket (top-level).
+            bucket, path = entry, None
+
+        return path or bucket
 
     def _strip_buckets(self, entries, detail, prefix=None):
         for entry in entries:
@@ -36,11 +49,38 @@ class FSSpecWrapper(BaseFileSystem):
         entries within info() and ls(detail=True) calls"""
         return entry
 
+    def _prepare_credentials(self, config):  # pylint: disable=unused-argument
+        """Prepare the arguments for authentication to the
+        host filesystem"""
+        return {}
+
     def isdir(self, path_info):
-        return self.fs.isdir(self._with_bucket(path_info))
+        if not self.exists(path_info):
+            return False
+
+        # Directory in object storages are interpreted differently
+        # among different fsspec providers, so this logic is a temporary
+        # measure for us to adapt as of now. It checks whether it is a
+        # directory (as in a prefix with contents) or whether it is an empty
+        # file where it's name ends with a forward slash
+        entry = self.info(path_info)
+        return entry["type"] == "directory" or (
+            entry["size"] == 0
+            and entry["type"] == "file"
+            and entry["name"].endswith("/")
+        )
 
     def isfile(self, path_info):
-        return self.fs.isfile(self._with_bucket(path_info))
+        if not self.exists(path_info):
+            return False
+
+        return not self.isdir(path_info)
+
+    def is_empty(self, path_info):
+        entry = self.info(path_info)
+        if entry["type"] == "directory":
+            return not self.fs.ls(self._with_bucket(path_info))
+        return entry["size"] == 0
 
     def open(
         self, path_info, mode="r", **kwargs
@@ -56,6 +96,9 @@ class FSSpecWrapper(BaseFileSystem):
     def ls(
         self, path_info, detail=False, recursive=False
     ):  # pylint: disable=arguments-differ
+        if self.isdir(path_info) and self.is_empty(path_info):
+            return None
+
         path = self._with_bucket(path_info)
         if recursive:
             for root, _, files in self.fs.walk(path, detail=detail):

dvc/fs/gs.py

@@ -22,16 +22,15 @@ class GSFileSystem(FSSpecWrapper):
         url = config.get("url", "gs://")
         self.path_info = self.PATH_CLS(url)
 
-        self.login_info = self._prepare_credentials(config)
-
     def _prepare_credentials(self, config):
-        login_info = {}
+        login_info = {"consistency": None}
         login_info["project"] = config.get("projectname")
         login_info["token"] = config.get("credentialpath")
         return login_info
 
     def _entry_hook(self, entry):
-        entry["etag"] = base64.b64decode(entry["etag"]).hex()
+        if "etag" in entry:
+            entry["etag"] = base64.b64decode(entry["etag"]).hex()
         return entry
 
     @wrap_prop(threading.Lock())
@@ -39,4 +38,4 @@ class GSFileSystem(FSSpecWrapper):
     def fs(self):
         from gcsfs import GCSFileSystem
 
-        return GCSFileSystem(**self.login_info)
+        return GCSFileSystem(**self.fs_args)

dvc/fs/http.py

@@ -1,6 +1,7 @@
 import logging
 import os.path
 import threading
+from typing import Optional
 
 from funcy import cached_property, memoize, wrap_prop, wrap_with
 
@@ -146,10 +147,9 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         raise HTTPError(res.status_code, res.reason)
 
     def info(self, path_info):
-        url = path_info.url
-        headers = self._head(url).headers
-        etag = headers.get("ETag") or headers.get("Content-MD5")
-        size = int(headers.get("Content-Length"))
+        resp = self._head(path_info.url)
+        etag = resp.headers.get("ETag") or resp.headers.get("Content-MD5")
+        size = self._content_length(resp)
         return {"etag": etag, "size": size}
 
     def _upload_fobj(self, fobj, to_info):
@@ -195,6 +195,6 @@ class HTTPFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             )
 
     @staticmethod
-    def _content_length(response):
+    def _content_length(response) -> Optional[int]:
         res = response.headers.get("Content-Length")
         return int(res) if res else None

dvc/fs/memory.py

@@ -13,13 +13,16 @@ class MemoryFileSystem(BaseFileSystem):
         self.fs = MemFS()
 
     def exists(self, path_info, use_dvcignore=True):
-        return self.fs.exists(path_info.path)
+        return self.fs.exists(path_info.fspath)
 
     def open(self, path_info, mode="r", encoding=None, **kwargs):
         return self.fs.open(
             path_info.fspath, mode=mode, encoding=encoding, **kwargs
         )
 
+    def info(self, path_info):
+        return self.fs.info(path_info.fspath)
+
     def stat(self, path_info):
         import os
 

dvc/fs/s3.py

@@ -38,6 +38,7 @@ class S3FileSystem(BaseFileSystem):
         self.endpoint_url = config.get("endpointurl")
 
         self.use_ssl = config.get("use_ssl", True)
+        self.ssl_verify = config.get("ssl_verify", True)
 
         self.extra_args = {}
 
@@ -136,6 +137,7 @@ class S3FileSystem(BaseFileSystem):
         return session.resource(
             "s3",
             endpoint_url=self.endpoint_url,
+            verify=self.ssl_verify,
             use_ssl=self.use_ssl,
             config=boto3.session.Config(
                 signature_version="s3v4", s3=s3_config

dvc/fs/ssh/__init__.py

@@ -33,7 +33,7 @@ def ask_password(host, user, port):
 class SSHFileSystem(BaseFileSystem):
     scheme = Schemes.SSH
     REQUIRES = {"paramiko": "paramiko"}
-    JOBS = 4
+    _JOBS = 4
 
     PARAM_CHECKSUM = "md5"
     # At any given time some of the connections will go over network and

dvc/fs/ssh/connection.py

@@ -17,7 +17,7 @@ from ..base import RemoteCmdError
 try:
     import paramiko
 except ImportError:
-    paramiko = None
+    paramiko = None  # type: ignore
 
 
 logger = logging.getLogger(__name__)

dvc/fs/webdav.py

@@ -117,7 +117,7 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             )
 
         # Check whether connection is valid (root should always exist)
-        if not client.check("/"):
+        if not client.check(self.path_info.path):
             raise WebDAVConnectionError(self.hostname)
 
         return client
@@ -213,7 +213,7 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     # Creates directories
     def makedirs(self, path_info):
         # Terminate recursion
-        if path_info.path == "/" or self.exists(path_info):
+        if path_info.path == self.path_info.path or self.exists(path_info):
             return
 
         # Recursively descent to root

dvc/hash_info.py

@@ -1,11 +1,9 @@
 from collections import OrderedDict
 from dataclasses import dataclass, field
-from typing import Dict, Optional
+from typing import Optional
 
 HASH_DIR_SUFFIX = ".dir"
 
-DirInfo = Dict[str, str]
-
 
 @dataclass
 class HashInfo:
@@ -14,7 +12,6 @@ class HashInfo:
 
     name: Optional[str]
     value: Optional[str]
-    dir_info: Optional[DirInfo] = field(default=None, compare=False)
     size: Optional[int] = field(default=None, compare=False)
     nfiles: Optional[int] = field(default=None, compare=False)
 

dvc/ignore.py

@@ -183,7 +183,12 @@ class DvcIgnoreFilter:
     def __init__(self, fs, root_dir):
         from dvc.repo import Repo
 
-        default_ignore_patterns = [".hg/", ".git/", "{}/".format(Repo.DVC_DIR)]
+        default_ignore_patterns = [
+            ".hg/",
+            ".git/",
+            ".git",
+            "{}/".format(Repo.DVC_DIR),
+        ]
 
         self.fs = fs
         self.root_dir = root_dir

dvc/logger.py

@@ -23,14 +23,19 @@ def addLoggingLevel(levelName, levelNum, methodName=None):
     Adds a new logging level to the `logging` module and the
     currently configured logging class.
 
+    Uses the existing numeric levelNum if already defined.
+
     Based on https://stackoverflow.com/questions/2183233
     """
     if methodName is None:
         methodName = levelName.lower()
 
-    assert not hasattr(logging, levelName)
-    assert not hasattr(logging, methodName)
-    assert not hasattr(logging.getLoggerClass(), methodName)
+    # If the level name is already defined as a top-level `logging`
+    # constant, then adopt the existing numeric level.
+    if hasattr(logging, levelName):
+        existingLevelNum = getattr(logging, levelName)
+        assert isinstance(existingLevelNum, int)
+        levelNum = existingLevelNum
 
     def logForLevel(self, message, *args, **kwargs):
         if self.isEnabledFor(levelNum):
@@ -40,10 +45,19 @@ def addLoggingLevel(levelName, levelNum, methodName=None):
     def logToRoot(message, *args, **kwargs):
         logging.log(levelNum, message, *args, **kwargs)
 
-    logging.addLevelName(levelNum, levelName)
-    setattr(logging, levelName, levelNum)
-    setattr(logging.getLoggerClass(), methodName, logForLevel)
-    setattr(logging, methodName, logToRoot)
+    # getLevelName resolves the numeric log level if already defined,
+    # otherwise returns a string
+    if not isinstance(logging.getLevelName(levelName), int):
+        logging.addLevelName(levelNum, levelName)
+
+    if not hasattr(logging, levelName):
+        setattr(logging, levelName, levelNum)
+
+    if not hasattr(logging.getLoggerClass(), methodName):
+        setattr(logging.getLoggerClass(), methodName, logForLevel)
+
+    if not hasattr(logging, methodName):
+        setattr(logging, methodName, logToRoot)
 
 
 class LoggingException(Exception):

dvc/main.py

@@ -45,6 +45,11 @@ def main(argv=None):  # noqa: C901
 
         logger.trace(args)
 
+        if not args.quiet:
+            from dvc.ui import ui
+
+            ui.enable()
+
         with debugtools(args):
             cmd = args.func(args)
             ret = cmd.run()

dvc/objects/__init__.py

@@ -1,315 +1,28 @@
-import itertools
-import json
 import logging
-from concurrent import futures
 
-from dvc.dir_info import DirInfo
-from dvc.exceptions import DvcException
 from dvc.progress import Tqdm
 
-from .stage import get_hash
+from .tree import Tree
 
 logger = logging.getLogger(__name__)
 
 
-class ObjectError(DvcException):
-    pass
-
-
-class ObjectFormatError(ObjectError):
-    pass
-
-
-class HashFile:
-    def __init__(self, path_info, fs, hash_info):
-        self.path_info = path_info
-        self.fs = fs
-        self.hash_info = hash_info
-
-    def __len__(self):
-        return 1
-
-    def __str__(self):
-        return f"object {self.hash_info}"
-
-    def __bool__(self):
-        return bool(self.hash_info)
-
-    def check(self, odb):
-        actual = get_hash(self.path_info, self.fs, odb.fs.PARAM_CHECKSUM, odb)
-
-        logger.trace(
-            "cache '%s' expected '%s' actual '%s'",
-            self.path_info,
-            self.hash_info,
-            actual,
-        )
-
-        assert actual.name == self.hash_info.name
-        if actual.value.split(".")[0] != self.hash_info.value.split(".")[0]:
-            raise ObjectFormatError(f"{self} is corrupted")
-
-    @classmethod
-    def load(cls, odb, hash_info):
-        return odb.get(hash_info)
-
-    def save(self, odb, **kwargs):
-        odb.add(self.path_info, self.fs, self.hash_info, **kwargs)
-
-
-class File(HashFile):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.src = None
-
-    @classmethod
-    def stage(cls, odb, path_info, fs, **kwargs):
-        hash_info = get_hash(
-            path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs
-        )
-        raw = odb.get(hash_info)
-        obj = cls(raw.path_info, raw.fs, hash_info)
-        obj.src = HashFile(path_info, fs, hash_info)
-        return obj
-
-    def save(self, odb, **kwargs):
-        self.src.save(odb, **kwargs)
-
-
-class Tree(HashFile):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.src_hash_info = None
-        self.src_path_info = None
-        self.src_fs = None
-
-    def __len__(self):
-        return self.hash_info.dir_info.nfiles
-
-    def __iter__(self):
-        yield from self.hash_info.dir_info.items()
-
-    @classmethod
-    def save_dir_info(cls, odb, dir_info, hash_info=None):
-        if hash_info and hash_info.name == odb.fs.PARAM_CHECKSUM:
-            try:
-                odb.check(hash_info)
-                assert hash_info.dir_info == dir_info
-                return hash_info
-            except (FileNotFoundError, ObjectFormatError):
-                pass
-
-        from dvc.fs.memory import MemoryFileSystem
-        from dvc.path_info import PathInfo
-        from dvc.utils import tmp_fname
-
-        fs = MemoryFileSystem(None, {})
-        path_info = PathInfo(tmp_fname(""))
-        with fs.open(path_info, "w") as fobj:
-            json.dump(dir_info.to_list(), fobj, sort_keys=True)
-
-        tmp_info = odb.fs.path_info / tmp_fname("")
-        with fs.open(path_info, "rb") as fobj:
-            odb.fs.upload_fobj(fobj, tmp_info)
-
-        hash_info = get_hash(tmp_info, odb.fs, odb.fs.PARAM_CHECKSUM, odb)
-        hash_info.value += odb.fs.CHECKSUM_DIR_SUFFIX
-        hash_info.dir_info = dir_info
-        hash_info.nfiles = dir_info.nfiles
-
-        odb.add(tmp_info, odb.fs, hash_info)
-
-        return hash_info
-
-    @classmethod
-    def stage(cls, odb, path_info, fs, **kwargs):
-        hash_info = get_hash(
-            path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs
-        )
-        hi = cls.save_dir_info(odb, hash_info.dir_info, hash_info)
-        hi.size = hash_info.size
-        raw = odb.get(hi)
-        obj = cls(raw.path_info, raw.fs, hi)
-        obj.src_hash_info = hash_info
-        obj.src_path_info = path_info
-        obj.src_fs = fs
-        return obj
-
-    @classmethod
-    def load(cls, odb, hash_info):
-
-        obj = odb.get(hash_info)
-
-        try:
-            with obj.fs.open(obj.path_info, "r") as fobj:
-                raw = json.load(fobj)
-        except ValueError as exc:
-            raise ObjectFormatError(f"{obj} is corrupted") from exc
-
-        if not isinstance(raw, list):
-            logger.error(
-                "dir cache file format error '%s' [skipping the file]",
-                obj.path_info,
-            )
-            raise ObjectFormatError(f"{obj} is corrupted")
-
-        dir_info = DirInfo.from_list(raw)
-        hash_info.dir_info = dir_info
-        hash_info.nfiles = dir_info.nfiles
-
-        return cls(obj.path_info, obj.fs, hash_info)
-
-    def save(self, odb, **kwargs):
-        assert self.src_hash_info.dir_info
-        hi = self.save_dir_info(
-            odb, self.src_hash_info.dir_info, self.hash_info
-        )
-        for entry_key, entry_hash in Tqdm(
-            hi.dir_info.items(),
-            desc="Saving " + self.src_path_info.name,
-            unit="file",
-        ):
-            entry_info = self.src_path_info.joinpath(*entry_key)
-            entry_obj = HashFile(entry_info, self.src_fs, entry_hash)
-            entry_obj.save(odb, **kwargs)
-        self.src_fs.repo.state.save(self.src_path_info, self.src_fs, hi)
-
-    def filter(self, odb, prefix):
-        hash_info = self.hash_info.dir_info.get(prefix)
-        if hash_info:
-            return load(odb, hash_info)
-
-        depth = len(prefix)
-        dir_info = DirInfo()
-        try:
-            for key, value in self.hash_info.dir_info.trie.items(prefix):
-                dir_info.add(key[depth:], value)
-        except KeyError:
-            return None
-
-        return load(odb, self.save_dir_info(odb, dir_info))
-
-
 def save(odb, obj, **kwargs):
-    obj.save(odb, **kwargs)
+    if isinstance(obj, Tree):
+        for _, entry in Tqdm(obj):
+            odb.add(entry.path_info, entry.fs, entry.hash_info, **kwargs)
+    odb.add(obj.path_info, obj.fs, obj.hash_info, **kwargs)
 
 
 def check(odb, obj):
     odb.check(obj.hash_info)
 
     if isinstance(obj, Tree):
-        for _, hash_info in obj:
-            odb.check(hash_info)
+        for _, entry in obj:
+            odb.check(entry.hash_info)
 
 
 def load(odb, hash_info):
     if hash_info.isdir:
         return Tree.load(odb, hash_info)
-    return File.load(odb, hash_info)
-
-
-def _get_dir_size(odb, dir_info):
-    try:
-        return sum(
-            odb.fs.getsize(odb.hash_to_path_info(hi.value))
-            for _, hi in dir_info.items()
-        )
-    except FileNotFoundError:
-        return None
-
-
-def merge(odb, ancestor_info, our_info, their_info):
-    assert our_info
-    assert their_info
-
-    if ancestor_info:
-        ancestor = load(odb, ancestor_info).hash_info.dir_info
-    else:
-        ancestor = DirInfo()
-
-    our = load(odb, our_info).hash_info.dir_info
-    their = load(odb, their_info).hash_info.dir_info
-
-    merged = our.merge(ancestor, their)
-    hash_info = Tree.save_dir_info(odb, merged)
-    hash_info.size = _get_dir_size(odb, merged)
-    return hash_info
-
-
-def _transfer_file(odb, from_fs, from_info):
-    from dvc.utils import tmp_fname
-    from dvc.utils.stream import HashedStreamReader
-
-    tmp_info = odb.fs.path_info / tmp_fname()
-    with from_fs.open(
-        from_info, mode="rb", chunk_size=from_fs.CHUNK_SIZE
-    ) as stream:
-        stream_reader = HashedStreamReader(stream)
-        # Since we don't know the hash beforehand, we'll
-        # upload it to a temporary location and then move
-        # it.
-        odb.fs.upload_fobj(
-            stream_reader,
-            tmp_info,
-            total=from_fs.getsize(from_info),
-            desc=from_info.name,
-        )
-
-    hash_info = stream_reader.hash_info
-    return tmp_info, hash_info
-
-
-def _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
-    rel_path_infos = {}
-    from_infos = from_fs.walk_files(from_info)
-
-    def create_tasks(executor, amount):
-        for entry_info in itertools.islice(from_infos, amount):
-            pbar.total += 1
-            task = executor.submit(
-                pbar.wrap_fn(_transfer_file), odb, from_fs, entry_info
-            )
-            rel_path_infos[task] = entry_info.relative_to(from_info)
-            yield task
-
-    pbar.total = 0
-    with futures.ThreadPoolExecutor(max_workers=jobs) as executor:
-        tasks = set(create_tasks(executor, jobs * 5))
-
-        while tasks:
-            done, tasks = futures.wait(
-                tasks, return_when=futures.FIRST_COMPLETED
-            )
-            tasks.update(create_tasks(executor, len(done)))
-            for task in done:
-                yield rel_path_infos.pop(task), task.result()
-
-
-def _transfer_directory(odb, from_fs, from_info, jobs, no_progress_bar=False):
-    dir_info = DirInfo()
-
-    with Tqdm(total=1, unit="Files", disable=no_progress_bar) as pbar:
-        for (
-            entry_info,
-            (entry_tmp_info, entry_hash),
-        ) in _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
-            odb.add(entry_tmp_info, odb.fs, entry_hash)
-            dir_info.add(entry_info.parts, entry_hash)
-
-    return Tree.save_dir_info(odb, dir_info)
-
-
-def transfer(odb, from_fs, from_info, jobs=None, no_progress_bar=False):
-    jobs = jobs or min((from_fs.jobs, odb.fs.jobs))
-
-    if from_fs.isdir(from_info):
-        return _transfer_directory(
-            odb,
-            from_fs,
-            from_info,
-            jobs=jobs,
-            no_progress_bar=no_progress_bar,
-        )
-    tmp_info, hash_info = _transfer_file(odb, from_fs, from_info)
-    odb.add(tmp_info, odb.fs, hash_info)
-    return hash_info
+    return odb.get(hash_info)

dvc/objects/db/base.py

@@ -4,10 +4,10 @@ from concurrent.futures import ThreadPoolExecutor
 from copy import copy
 from typing import Optional
 
+from dvc.objects.errors import ObjectFormatError
+from dvc.objects.file import HashFile
 from dvc.progress import Tqdm
 
-from .. import HashFile, ObjectFormatError
-
 logger = logging.getLogger(__name__)
 
 
@@ -229,7 +229,7 @@ class ObjectDB:
         keeping all of it in memory.
         """
         num_pages = remote_size / self.fs.LIST_OBJECT_PAGE_SIZE
-        if num_pages < 256 / self.fs.JOBS:
+        if num_pages < 256 / self.fs.jobs:
             # Fetching prefixes in parallel requires at least 255 more
             # requests, for small enough remotes it will be faster to fetch
             # entire cache without splitting it into prefixes.
@@ -263,7 +263,7 @@ class ObjectDB:
                 )
 
             with ThreadPoolExecutor(
-                max_workers=jobs or self.fs.JOBS
+                max_workers=jobs or self.fs.jobs
             ) as executor:
                 in_remote = executor.map(list_with_update, traverse_prefixes,)
                 yield from itertools.chain.from_iterable(in_remote)
@@ -331,7 +331,7 @@ class ObjectDB:
                 return ret
 
             with ThreadPoolExecutor(
-                max_workers=jobs or self.fs.JOBS
+                max_workers=jobs or self.fs.jobs
             ) as executor:
                 path_infos = map(self.hash_to_path_info, hashes)
                 in_remote = executor.map(exists_with_progress, path_infos)

dvc/objects/db/local.py

@@ -6,7 +6,7 @@ from funcy import cached_property
 from shortuuid import uuid
 
 from dvc.hash_info import HashInfo
-from dvc.objects import ObjectFormatError
+from dvc.objects.errors import ObjectFormatError
 from dvc.path_info import PathInfo
 from dvc.progress import Tqdm
 from dvc.utils import relpath

dvc/objects/db/ssh.py

@@ -64,10 +64,10 @@ class SSHObjectDB(ObjectDB):
                 return self.batch_exists(chunks, callback=pbar.update_msg)
 
             with ThreadPoolExecutor(
-                max_workers=jobs or self.fs.JOBS
+                max_workers=jobs or self.fs.jobs
             ) as executor:
                 path_infos = [self.hash_to_path_info(x) for x in hashes]
-                chunks = to_chunks(path_infos, num_chunks=self.fs.JOBS)
+                chunks = to_chunks(path_infos, num_chunks=self.fs.jobs)
                 results = executor.map(exists_with_progress, chunks)
                 in_remote = itertools.chain.from_iterable(results)
                 ret = list(itertools.compress(hashes, in_remote))

dvc/objects/errors.py

@@ -0,0 +1,9 @@
+from dvc.exceptions import DvcException
+
+
+class ObjectError(DvcException):
+    pass
+
+
+class ObjectFormatError(ObjectError):
+    pass

dvc/objects/file.py

@@ -0,0 +1,53 @@
+import logging
+
+from .errors import ObjectFormatError
+from .stage import get_file_hash
+
+logger = logging.getLogger(__name__)
+
+
+class HashFile:
+    def __init__(self, path_info, fs, hash_info):
+        self.path_info = path_info
+        self.fs = fs
+        self.hash_info = hash_info
+
+    @property
+    def size(self):
+        if not (self.path_info and self.fs):
+            return None
+        return self.fs.getsize(self.path_info)
+
+    def __len__(self):
+        return 1
+
+    def __str__(self):
+        return f"object {self.hash_info}"
+
+    def __bool__(self):
+        return bool(self.hash_info)
+
+    def __eq__(self, other):
+        if not isinstance(other, HashFile):
+            return False
+        return (
+            self.path_info == other.path_info
+            and self.fs == other.fs
+            and self.hash_info == other.hash_info
+        )
+
+    def check(self, odb):
+        actual = get_file_hash(
+            self.path_info, self.fs, self.hash_info.name, odb.repo.state
+        )
+
+        logger.trace(
+            "cache '%s' expected '%s' actual '%s'",
+            self.path_info,
+            self.hash_info,
+            actual,
+        )
+
+        assert actual.name == self.hash_info.name
+        if actual.value.split(".")[0] != self.hash_info.value.split(".")[0]:
+            raise ObjectFormatError(f"{self} is corrupted")

dvc/objects/stage.py

@@ -9,7 +9,7 @@ from dvc.progress import Tqdm
 from dvc.utils import file_md5
 
 
-def get_file_hash(path_info, fs, name):
+def _get_file_hash(path_info, fs, name):
     info = fs.info(path_info)
     if name in info:
         assert not info[name].endswith(".dir")
@@ -27,20 +27,36 @@ def get_file_hash(path_info, fs, name):
     raise NotImplementedError
 
 
-def _calculate_hashes(path_info, fs, name, state, **kwargs):
-    def _get_file_hash(file_info):
+def get_file_hash(path_info, fs, name, state=None):
+    if state:
         hash_info = state.get(  # pylint: disable=assignment-from-none
-            file_info, fs,
+            path_info, fs,
         )
-        if not hash_info:
-            hash_info = get_file_hash(file_info, fs, name)
-            state.save(file_info, fs, hash_info)
-        return file_info, hash_info
+        if hash_info:
+            return hash_info
+
+    if not fs.exists(path_info):
+        raise FileNotFoundError(
+            errno.ENOENT, os.strerror(errno.ENOENT), path_info
+        )
+
+    hash_info = _get_file_hash(path_info, fs, name)
+
+    if state:
+        assert ".dir" not in hash_info.value
+        state.save(path_info, fs, hash_info)
+
+    return hash_info
+
+
+def _calculate_hashes(path_info, fs, name, state, **kwargs):
+    def _hash(file_info):
+        return file_info, get_file_hash(file_info, fs, name, state)
 
     with Tqdm(
         unit="md5", desc="Computing file/dir hashes (only done once)",
     ) as pbar:
-        worker = pbar.wrap_fn(_get_file_hash)
+        worker = pbar.wrap_fn(_hash)
         with ThreadPoolExecutor(max_workers=fs.hash_jobs) as executor:
             pairs = executor.map(worker, fs.walk_files(path_info, **kwargs))
             return dict(pairs)
@@ -60,14 +76,17 @@ def _iter_hashes(path_info, fs, name, state, **kwargs):
     yield from _calculate_hashes(path_info, fs, name, state, **kwargs).items()
 
 
-def _collect_dir(path_info, fs, name, state, **kwargs):
-    from dvc.dir_info import DirInfo
+def _build_tree(path_info, fs, name, state, **kwargs):
+    from .file import HashFile
+    from .tree import Tree
 
-    dir_info = DirInfo()
+    tree = Tree(None, None, None)
     for fi, hi in _iter_hashes(path_info, fs, name, state, **kwargs):
         if DvcIgnore.DVCIGNORE_FILE == fi.name:
             raise DvcIgnoreInCollectedDirError(fi.parent)
 
+        obj = HashFile(fi, fs, hi)
+
         # NOTE: this is lossy transformation:
         #   "hey\there" -> "hey/there"
         #   "hey/there" -> "hey/there"
@@ -76,31 +95,44 @@ def _collect_dir(path_info, fs, name, state, **kwargs):
         #
         # Yes, this is a BUG, as long as we permit "/" in
         # filenames on Windows and "\" on Unix
-        dir_info.add(fi.relative_to(path_info).parts, hi)
+        tree.add(fi.relative_to(path_info).parts, obj)
 
-    return dir_info
+    tree.digest()
+
+    return tree
 
 
-def get_dir_hash(path_info, fs, name, odb, state, **kwargs):
-    from . import Tree
+def _get_tree_obj(path_info, fs, name, odb, state, **kwargs):
+    from .tree import Tree
 
     value = fs.info(path_info).get(name)
     if value:
         hash_info = HashInfo(name, value)
         try:
-            Tree.load(odb, hash_info)
-            return hash_info
+            return Tree.load(odb, hash_info)
         except FileNotFoundError:
             pass
 
-    dir_info = _collect_dir(path_info, fs, name, state, **kwargs)
-    hash_info = Tree.save_dir_info(fs.repo.odb.local, dir_info)
-    hash_info.size = dir_info.size
-    hash_info.dir_info = dir_info
-    return hash_info
+    tree = _build_tree(path_info, fs, name, state, **kwargs)
+
+    odb.add(tree.path_info, tree.fs, tree.hash_info)
+    if name != "md5":
+        # NOTE: used only for external outputs. Initial reasoning was to be
+        # able to validate .dir files right in the workspace (e.g. check s3
+        # etag), but could be dropped for manual validation with regular md5,
+        # that would be universal for all clouds.
+        raw = odb.get(tree.hash_info)
+        hash_info = get_file_hash(raw.path_info, raw.fs, name, state)
+        tree.hash_info.name = hash_info.name
+        tree.hash_info.value = hash_info.value
+        if not tree.hash_info.value.endswith(".dir"):
+            tree.hash_info.value += ".dir"
+        odb.add(tree.path_info, tree.fs, tree.hash_info)
+
+    return tree
 
 
-def get_hash(path_info, fs, name, odb, **kwargs):
+def stage(odb, path_info, fs, name, **kwargs):
     assert path_info and (
         isinstance(path_info, str) or path_info.scheme == fs.scheme
     )
@@ -125,28 +157,31 @@ def get_hash(path_info, fs, name, odb, **kwargs):
         hash_info = None
 
     if hash_info:
-        if hash_info.isdir:
-            from . import Tree
+        from . import load
+        from .tree import Tree
 
-            # NOTE: loading the fs will restore hash_info.dir_info
-            Tree.load(odb, hash_info)
-        assert hash_info.name == name
-        return hash_info
+        obj = load(odb, hash_info)
+        if isinstance(obj, Tree):
+            obj.hash_info.nfiles = len(obj)
+            for key, entry in obj:
+                entry.fs = fs
+                entry.path_info = path_info.joinpath(*key)
+        else:
+            obj.fs = fs
+            obj.path_info = path_info
+        assert obj.hash_info.name == name
+        obj.hash_info.size = hash_info.size
+        return obj
 
     if fs.isdir(path_info):
-        hash_info = get_dir_hash(path_info, fs, name, odb, state, **kwargs)
+        obj = _get_tree_obj(path_info, fs, name, odb, state, **kwargs)
     else:
-        hash_info = get_file_hash(path_info, fs, name)
+        from .file import HashFile
 
-    if hash_info and fs.exists(path_info):
-        state.save(path_info, fs, hash_info)
+        hash_info = get_file_hash(path_info, fs, name, state)
+        obj = HashFile(path_info, fs, hash_info)
 
-    return hash_info
+    if obj.hash_info and fs.exists(path_info):
+        state.save(path_info, fs, obj.hash_info)
 
-
-def stage(odb, path_info, fs, **kwargs):
-    from . import File, Tree
-
-    if fs.isdir(path_info):
-        return Tree.stage(odb, path_info, fs, **kwargs)
-    return File.stage(odb, path_info, fs, **kwargs)
+    return obj

dvc/objects/transfer.py

@@ -0,0 +1,92 @@
+import itertools
+import logging
+from concurrent import futures
+
+from dvc.progress import Tqdm
+
+from . import save
+from .file import HashFile
+from .tree import Tree
+
+logger = logging.getLogger(__name__)
+
+
+def _transfer_file(odb, from_fs, from_info):
+    from dvc.utils import tmp_fname
+    from dvc.utils.stream import HashedStreamReader
+
+    tmp_info = odb.fs.path_info / tmp_fname()
+    with from_fs.open(
+        from_info, mode="rb", chunk_size=from_fs.CHUNK_SIZE
+    ) as stream:
+        stream_reader = HashedStreamReader(stream)
+        # Since we don't know the hash beforehand, we'll
+        # upload it to a temporary location and then move
+        # it.
+        odb.fs.upload_fobj(
+            stream_reader,
+            tmp_info,
+            total=from_fs.getsize(from_info),
+            desc=from_info.name,
+        )
+
+    hash_info = stream_reader.hash_info
+    return tmp_info, hash_info
+
+
+def _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
+    rel_path_infos = {}
+    from_infos = from_fs.walk_files(from_info)
+
+    def create_tasks(executor, amount):
+        for entry_info in itertools.islice(from_infos, amount):
+            pbar.total += 1
+            task = executor.submit(
+                pbar.wrap_fn(_transfer_file), odb, from_fs, entry_info
+            )
+            rel_path_infos[task] = entry_info.relative_to(from_info)
+            yield task
+
+    pbar.total = 0
+    with futures.ThreadPoolExecutor(max_workers=jobs) as executor:
+        tasks = set(create_tasks(executor, jobs * 5))
+
+        while tasks:
+            done, tasks = futures.wait(
+                tasks, return_when=futures.FIRST_COMPLETED
+            )
+            tasks.update(create_tasks(executor, len(done)))
+            for task in done:
+                yield rel_path_infos.pop(task), task.result()
+
+
+def _transfer_directory(odb, from_fs, from_info, jobs, no_progress_bar=False):
+    tree = Tree(None, None, None)
+
+    with Tqdm(total=1, unit="Files", disable=no_progress_bar) as pbar:
+        for (
+            entry_info,
+            (entry_tmp_info, entry_hash),
+        ) in _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
+            obj = HashFile(entry_tmp_info, odb.fs, entry_hash)
+            tree.add(entry_info.parts, obj)
+
+    tree.digest()
+    save(odb, tree)
+    return tree.hash_info
+
+
+def transfer(odb, from_fs, from_info, jobs=None, no_progress_bar=False):
+    jobs = jobs or min((from_fs.jobs, odb.fs.jobs))
+
+    if from_fs.isdir(from_info):
+        return _transfer_directory(
+            odb,
+            from_fs,
+            from_info,
+            jobs=jobs,
+            no_progress_bar=no_progress_bar,
+        )
+    tmp_info, hash_info = _transfer_file(odb, from_fs, from_info)
+    odb.add(tmp_info, odb.fs, hash_info)
+    return hash_info

dvc/objects/tree.py

@@ -0,0 +1,211 @@
+import json
+import logging
+import posixpath
+
+from funcy import cached_property
+
+from .errors import ObjectFormatError
+from .file import HashFile
+from .stage import get_file_hash
+
+logger = logging.getLogger(__name__)
+
+
+class Tree(HashFile):
+    PARAM_RELPATH = "relpath"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._dict = {}
+
+    @cached_property
+    def trie(self):
+        from pygtrie import Trie
+
+        return Trie(self._dict)
+
+    @property
+    def size(self):
+        try:
+            return sum(obj.size for _, obj in self)
+        except TypeError:
+            return None
+
+    def add(self, key, obj):
+        self.__dict__.pop("trie", None)
+        self._dict[key] = obj
+
+    def digest(self):
+        from dvc.fs.memory import MemoryFileSystem
+        from dvc.path_info import PathInfo
+        from dvc.utils import tmp_fname
+
+        memfs = MemoryFileSystem(None, {})
+        path_info = PathInfo(tmp_fname(""))
+        with memfs.open(path_info, "wb") as fobj:
+            fobj.write(self.as_bytes())
+        self.fs = memfs
+        self.path_info = path_info
+        self.hash_info = get_file_hash(path_info, memfs, "md5")
+        self.hash_info.value += ".dir"
+        self.hash_info.size = self.size
+        self.hash_info.nfiles = len(self)
+
+    def __len__(self):
+        return len(self._dict)
+
+    def __iter__(self):
+        yield from self._dict.items()
+
+    def as_dict(self):
+        return self._dict.copy()
+
+    def as_list(self):
+        from operator import itemgetter
+
+        # Sorting the list by path to ensure reproducibility
+        return sorted(
+            (
+                {
+                    # NOTE: not using hash_info.to_dict() because we don't want
+                    # size/nfiles fields at this point.
+                    obj.hash_info.name: obj.hash_info.value,
+                    self.PARAM_RELPATH: posixpath.sep.join(parts),
+                }
+                for parts, obj in self._dict.items()  # noqa: B301
+            ),
+            key=itemgetter(self.PARAM_RELPATH),
+        )
+
+    def as_bytes(self):
+        return json.dumps(self.as_list(), sort_keys=True).encode("utf-8")
+
+    @classmethod
+    def from_list(cls, lst):
+        from dvc.hash_info import HashInfo
+
+        tree = cls(None, None, None)
+        for _entry in lst:
+            entry = _entry.copy()
+            relpath = entry.pop(cls.PARAM_RELPATH)
+            parts = tuple(relpath.split(posixpath.sep))
+            hash_info = HashInfo.from_dict(entry)
+            obj = HashFile(None, None, hash_info)
+            tree.add(parts, obj)
+        return tree
+
+    @classmethod
+    def load(cls, odb, hash_info):
+
+        obj = odb.get(hash_info)
+
+        try:
+            with obj.fs.open(obj.path_info, "r") as fobj:
+                raw = json.load(fobj)
+        except ValueError as exc:
+            raise ObjectFormatError(f"{obj} is corrupted") from exc
+
+        if not isinstance(raw, list):
+            logger.error(
+                "dir cache file format error '%s' [skipping the file]",
+                obj.path_info,
+            )
+            raise ObjectFormatError(f"{obj} is corrupted")
+
+        tree = cls.from_list(raw)
+        tree.path_info = obj.path_info
+        tree.fs = obj.fs
+        tree.hash_info = hash_info
+
+        return tree
+
+    def filter(self, odb, prefix):
+        obj = self._dict.get(prefix)
+        if obj:
+            return obj
+
+        depth = len(prefix)
+        tree = Tree(None, None, None)
+        try:
+            for key, obj in self.trie.items(prefix):
+                tree.add(key[depth:], obj)
+        except KeyError:
+            return None
+        tree.digest()
+        odb.add(tree.path_info, tree.fs, tree.hash_info)
+        return tree
+
+
+def _get_dir_size(odb, tree):
+    try:
+        return sum(
+            odb.fs.getsize(odb.hash_to_path_info(obj.hash_info.value))
+            for _, obj in tree
+        )
+    except FileNotFoundError:
+        return None
+
+
+def _diff(ancestor, other, allow_removed=False):
+    from dictdiffer import diff
+
+    from dvc.exceptions import MergeError
+
+    allowed = ["add"]
+    if allow_removed:
+        allowed.append("remove")
+
+    result = list(diff(ancestor, other))
+    for typ, _, _ in result:
+        if typ not in allowed:
+            raise MergeError(
+                "unable to auto-merge directories with diff that contains "
+                f"'{typ}'ed files"
+            )
+    return result
+
+
+def _merge(ancestor, our, their):
+    import copy
+
+    from dictdiffer import patch
+
+    our_diff = _diff(ancestor, our)
+    if not our_diff:
+        return copy.deepcopy(their)
+
+    their_diff = _diff(ancestor, their)
+    if not their_diff:
+        return copy.deepcopy(our)
+
+    # make sure there are no conflicting files
+    _diff(our, their, allow_removed=True)
+
+    return patch(our_diff + their_diff, ancestor)
+
+
+def merge(odb, ancestor_info, our_info, their_info):
+    from . import load
+
+    assert our_info
+    assert their_info
+
+    if ancestor_info:
+        ancestor = load(odb, ancestor_info)
+    else:
+        ancestor = Tree(None, None, None)
+
+    our = load(odb, our_info)
+    their = load(odb, their_info)
+
+    merged_dict = _merge(ancestor.as_dict(), our.as_dict(), their.as_dict(),)
+
+    merged = Tree(None, None, None)
+    for key, hi in merged_dict.items():
+        merged.add(key, hi)
+    merged.digest()
+
+    odb.add(merged.path_info, merged.fs, merged.hash_info)
+    hash_info = merged.hash_info
+    hash_info.size = _get_dir_size(odb, merged)
+    return hash_info

dvc/output/base.py

@@ -18,7 +18,7 @@ from dvc.exceptions import (
 )
 from dvc.hash_info import HashInfo
 from dvc.objects.db import NamedCache
-from dvc.objects.stage import get_hash
+from dvc.objects.errors import ObjectFormatError
 from dvc.objects.stage import stage as ostage
 
 from ..fs.base import BaseFileSystem
@@ -183,23 +183,21 @@ class BaseOutput:
     def odb(self):
         return getattr(self.repo.odb, self.scheme)
 
-    @property
-    def dir_cache(self):
-        return self.hash_info.dir_info
-
     @property
     def cache_path(self):
         return self.odb.hash_to_path_info(self.hash_info.value).url
 
     def get_hash(self):
         if not self.use_cache:
-            return get_hash(
+            return ostage(
+                self.repo.odb.local,
                 self.path_info,
                 self.fs,
                 self.fs.PARAM_CHECKSUM,
-                self.repo.odb.local,
-            )
-        return ostage(self.odb, self.path_info, self.fs).hash_info
+            ).hash_info
+        return ostage(
+            self.odb, self.path_info, self.fs, self.odb.fs.PARAM_CHECKSUM
+        ).hash_info
 
     @property
     def is_dir_checksum(self):
@@ -223,7 +221,7 @@ class BaseOutput:
         try:
             objects.check(self.odb, obj)
             return False
-        except (FileNotFoundError, objects.ObjectFormatError):
+        except (FileNotFoundError, ObjectFormatError):
             return True
 
     def workspace_status(self):
@@ -307,7 +305,9 @@ class BaseOutput:
             logger.debug("Output '%s' didn't change. Skipping saving.", self)
             return
 
-        self.obj = ostage(self.odb, self.path_info, self.fs)
+        self.obj = ostage(
+            self.odb, self.path_info, self.fs, self.odb.fs.PARAM_CHECKSUM
+        )
         self.hash_info = self.obj.hash_info
         self.isexec = self.isfile() and self.fs.isexec(self.path_info)
 
@@ -322,7 +322,12 @@ class BaseOutput:
         assert self.hash_info
 
         if self.use_cache:
-            obj = ostage(self.odb, filter_info or self.path_info, self.fs)
+            obj = ostage(
+                self.odb,
+                filter_info or self.path_info,
+                self.fs,
+                self.odb.fs.PARAM_CHECKSUM,
+            )
             objects.save(self.odb, obj)
             checkout(
                 filter_info or self.path_info,
@@ -403,6 +408,7 @@ class BaseOutput:
         relink=False,
         filter_info=None,
         allow_missing=False,
+        checkpoint_reset=False,
         **kwargs,
     ):
         if not self.use_cache:
@@ -417,6 +423,11 @@ class BaseOutput:
             # backward compatibility
             return None
 
+        if self.checkpoint and checkpoint_reset:
+            if self.exists:
+                self.remove()
+            return None
+
         added = not self.exists
 
         try:
@@ -460,8 +471,6 @@ class BaseOutput:
             self.repo.scm.ignore(self.fspath)
 
     def get_files_number(self, filter_info=None):
-        from funcy import ilen
-
         if not self.use_cache or not self.hash_info:
             return 0
 
@@ -471,13 +480,8 @@ class BaseOutput:
         if not filter_info or filter_info == self.path_info:
             return self.hash_info.nfiles or 0
 
-        if not self.hash_info.dir_info:
-            return 0
-
-        return ilen(
-            filter_info.isin_or_eq(self.path_info.joinpath(*relpath))
-            for relpath, _ in self.hash_info.dir_info.items()
-        )
+        obj = self.get_obj(filter_info=filter_info)
+        return len(obj) if obj else 0
 
     def unprotect(self):
         if self.exists:
@@ -490,7 +494,7 @@ class BaseOutput:
 
         try:
             objects.check(self.odb, self.odb.get(self.hash_info))
-        except (FileNotFoundError, objects.ObjectFormatError):
+        except (FileNotFoundError, ObjectFormatError):
             self.repo.cloud.pull(
                 NamedCache.make("local", self.hash_info.value, str(self)),
                 show_checksums=False,
@@ -498,12 +502,11 @@ class BaseOutput:
             )
 
         try:
-            objects.load(self.odb, self.hash_info)
-            assert self.hash_info.dir_info
-        except (objects.ObjectFormatError, FileNotFoundError):
-            self.hash_info.dir_info = None
+            self.obj = objects.load(self.odb, self.hash_info)
+        except (FileNotFoundError, ObjectFormatError):
+            self.obj = None
 
-        return self.dir_cache
+        return self.obj
 
     def collect_used_dir_cache(
         self, remote=None, force=False, jobs=None, filter_info=None
@@ -536,7 +539,7 @@ class BaseOutput:
 
         try:
             objects.check(self.odb, self.odb.get(self.hash_info))
-        except (FileNotFoundError, objects.ObjectFormatError):
+        except (FileNotFoundError, ObjectFormatError):
             msg = (
                 "Missing cache for directory '{}'. "
                 "Cache for files inside will be lost. "
@@ -551,14 +554,14 @@ class BaseOutput:
 
         path = str(self.path_info)
         filter_path = str(filter_info) if filter_info else None
-        for entry_key, entry_hash_info in self.dir_cache.items():
+        for entry_key, entry_obj in self.obj:
             entry_path = os.path.join(path, *entry_key)
             if (
                 not filter_path
                 or entry_path == filter_path
                 or entry_path.startswith(filter_path + os.sep)
             ):
-                cache.add(self.scheme, entry_hash_info.value, entry_path)
+                cache.add(self.scheme, entry_obj.hash_info.value, entry_path)
 
         return cache
 
@@ -651,6 +654,8 @@ class BaseOutput:
             )
 
     def merge(self, ancestor, other):
+        from dvc.objects.tree import merge
+
         assert other
 
         if ancestor:
@@ -662,6 +667,6 @@ class BaseOutput:
         self._check_can_merge(self)
         self._check_can_merge(other)
 
-        self.hash_info = objects.merge(
+        self.hash_info = merge(
             self.odb, ancestor_info, self.hash_info, other.hash_info
         )

dvc/parsing/context.py

@@ -401,9 +401,9 @@ class Context(CtxDict):
     ):
         path, _, keys_str = item.partition(":")
         select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
-        path_info = wdir / path
 
-        abspath = os.path.abspath(path_info)
+        abspath = os.path.abspath(wdir / path)
+        path_info = PathInfo(abspath)
         if abspath in self.imports:
             if not select_keys and self.imports[abspath] is None:
                 return  # allow specifying complete filepath multiple times

dvc/remote/base.py

@@ -309,7 +309,7 @@ class Remote:
         )
 
         if download:
-            func = _log_exceptions(self.fs.download, "download")
+            func = _log_exceptions(self.fs.download_file, "download")
             status = STATUS_DELETED
             desc = "Downloading"
         else:
@@ -318,7 +318,7 @@ class Remote:
             desc = "Uploading"
 
         if jobs is None:
-            jobs = self.fs.JOBS
+            jobs = self.fs.jobs
 
         dir_status, file_status, dir_contents = self._status(
             cache,
@@ -506,7 +506,7 @@ class Remote:
         return ret
 
     def transfer(self, from_fs, from_info, jobs=None, no_progress_bar=False):
-        from dvc.objects import transfer
+        from dvc.objects.transfer import transfer
 
         return transfer(
             self.odb,

dvc/repo/__init__.py

@@ -141,6 +141,7 @@ class Repo:
         from dvc.repo.params import Params
         from dvc.repo.plots import Plots
         from dvc.repo.stage import StageLoad
+        from dvc.scm import SCM
         from dvc.stage.cache import StageCache
         from dvc.state import State, StateNoop
 
@@ -149,6 +150,9 @@ class Repo:
             "repo_factory": repo_factory,
         }
 
+        if rev and not scm:
+            scm = SCM(root_dir or os.curdir)
+
         self.root_dir, self.dvc_dir, self.tmp_dir = self._get_repo_dirs(
             root_dir=root_dir, scm=scm, rev=rev, uninitialized=uninitialized
         )
@@ -463,6 +467,12 @@ class Repo:
         path_parts = os.path.normpath(path).split(os.path.sep)
         return self.DVC_DIR in path_parts
 
+    @cached_property
+    def dvcfs(self):
+        from dvc.fs.dvc import DvcFileSystem
+
+        return DvcFileSystem(self)
+
     @cached_property
     def repo_fs(self):
         from dvc.fs.repo import RepoFileSystem

dvc/repo/add.py

@@ -173,7 +173,7 @@ def _process_stages(
             )
         else:
             from dvc.fs import get_cloud_fs
-            from dvc.objects import transfer
+            from dvc.objects.transfer import transfer
 
             from_fs = get_cloud_fs(repo, url=target)
             out.hash_info = transfer(

dvc/repo/diff.py

@@ -1,5 +1,7 @@
 import logging
 import os
+from collections import defaultdict
+from typing import Dict, List
 
 from dvc.exceptions import PathMissingError
 from dvc.objects.stage import get_file_hash
@@ -130,7 +132,7 @@ def _output_paths(repo, repo_fs, targets):
     def _to_checksum(output):
         if on_working_fs:
             return ostage(
-                repo.odb.local, output.path_info, repo.odb.local.fs,
+                repo.odb.local, output.path_info, repo.odb.local.fs, "md5",
             ).hash_info.value
         return output.hash_info.value
 
@@ -190,29 +192,23 @@ def _targets_to_path_infos(repo_fs, targets):
 
 
 def _calculate_renamed(new, old, added, deleted):
-    old_inverted = {}
+    old_inverted: Dict[str, List[str]] = defaultdict(list)
     # It is needed to be dict of lists to cover cases
     # when repo has paths with same hash
     for path, path_hash in old.items():
-        bucket = old_inverted.get(path_hash, None)
-        if bucket is None:
-            old_inverted[path_hash] = [path]
-        else:
-            bucket.append(path)
+        old_inverted[path_hash].append(path)
 
     renamed = []
     for path in added:
         path_hash = new[path]
-        old_paths = old_inverted.get(path_hash, None)
-        if not old_paths:
-            continue
-        old_path = None
-        for tmp_old_path in old_paths:
-            if tmp_old_path in deleted:
-                old_path = tmp_old_path
-                break
-        if not old_path:
+        old_paths = old_inverted[path_hash]
+        try:
+            iterator = enumerate(old_paths)
+            index = next(idx for idx, path in iterator if path in deleted)
+        except StopIteration:
             continue
+
+        old_path = old_paths.pop(index)
         renamed.append(
             {"path": {"old": old_path, "new": path}, "hash": path_hash}
         )

dvc/repo/experiments/__init__.py

@@ -182,8 +182,6 @@ class Experiments:
                         )
                         self.scm.reset()
 
-                    self._prune_lockfiles()
-
                     # update experiment params from command line
                     if params:
                         self._update_params(params)
@@ -244,34 +242,6 @@ class Experiments:
 
         return stash_rev
 
-    def _prune_lockfiles(self):
-        from dvc.dvcfile import is_lock_file
-
-        # NOTE: dirty DVC lock files must be restored to index state to
-        # avoid checking out incorrect persist or checkpoint outs
-        fs = self.scm.get_fs("HEAD")
-        lock_files = [
-            str(fname)
-            for fname in fs.walk_files(self.scm.root_dir)
-            if is_lock_file(fname)
-        ]
-        if lock_files:
-
-            self.scm.reset(paths=lock_files)
-            self.scm.checkout_index(paths=lock_files, force=True)
-
-    def _prune_untracked_lockfiles(self):
-        from dvc.dvcfile import is_lock_file
-        from dvc.utils.fs import remove
-
-        untracked = [
-            fname
-            for fname in self.scm.untracked_files()
-            if is_lock_file(fname)
-        ]
-        for fname in untracked:
-            remove(fname)
-
     def _stash_msg(
         self,
         rev: str,
@@ -345,9 +315,16 @@ class Experiments:
         queue: bool = False,
         tmp_dir: bool = False,
         checkpoint_resume: Optional[str] = None,
+        reset: bool = False,
         **kwargs,
     ):
         """Reproduce and checkout a single experiment."""
+        if queue and not checkpoint_resume:
+            reset = True
+
+        if reset:
+            self.reset_checkpoints()
+
         if not (queue or tmp_dir):
             staged, _, _ = self.scm.status()
             if staged:
@@ -370,7 +347,9 @@ class Experiments:
         else:
             checkpoint_resume = self._workspace_resume_rev()
 
-        stash_rev = self.new(checkpoint_resume=checkpoint_resume, **kwargs)
+        stash_rev = self.new(
+            checkpoint_resume=checkpoint_resume, reset=reset, **kwargs
+        )
         if queue:
             logger.info(
                 "Queued experiment '%s' for future execution.", stash_rev[:7],
@@ -709,7 +688,6 @@ class Experiments:
         # result in conflict between workspace params and stashed CLI params).
         self.scm.reset(hard=True)
         with self.scm.detach_head(entry.rev):
-            self._prune_untracked_lockfiles()
             rev = self.stash.pop()
             self.scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
             if entry.branch:
@@ -723,12 +701,17 @@ class Experiments:
                     rev,
                     name=entry.name,
                     rel_cwd=relpath(os.getcwd(), self.scm.root_dir),
+                    log_errors=False,
                 )
 
-                if not exec_result.exp_hash or not exec_result.ref_info:
+                if not exec_result.exp_hash:
                     raise DvcException(
                         f"Failed to reproduce experiment '{rev[:7]}'"
                     )
+                if not exec_result.ref_info:
+                    # repro succeeded but result matches baseline
+                    # (no experiment generated or applied)
+                    return {}
                 exp_rev = self.scm.get_ref(str(exec_result.ref_info))
                 self.scm.set_ref(EXEC_APPLY, exp_rev)
                 return {exp_rev: exec_result.exp_hash}

dvc/repo/experiments/diff.py

@@ -6,7 +6,7 @@ from dvc.utils.diff import format_dict
 logger = logging.getLogger(__name__)
 
 
-def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
+def diff(repo, *args, a_rev=None, b_rev=None, param_deps=False, **kwargs):
     from dvc.repo.experiments.show import _collect_experiment_commit
 
     if repo.scm.no_commits:
@@ -14,15 +14,17 @@ def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
 
     if a_rev:
         rev = repo.scm.resolve_rev(a_rev)
-        old = _collect_experiment_commit(repo, rev)
+        old = _collect_experiment_commit(repo, rev, param_deps=param_deps)
     else:
-        old = _collect_experiment_commit(repo, "HEAD")
+        old = _collect_experiment_commit(repo, "HEAD", param_deps=param_deps)
 
     if b_rev:
         rev = repo.scm.resolve_rev(b_rev)
-        new = _collect_experiment_commit(repo, rev)
+        new = _collect_experiment_commit(repo, rev, param_deps=param_deps)
     else:
-        new = _collect_experiment_commit(repo, "workspace")
+        new = _collect_experiment_commit(
+            repo, "workspace", param_deps=param_deps
+        )
 
     with_unchanged = kwargs.pop("all", False)
 

dvc/repo/experiments/executor/base.py

@@ -181,16 +181,16 @@ class BaseExecutor(ABC):
                 refs.append(ref)
 
         def on_diverged_ref(orig_ref: str, new_rev: str):
-            orig_rev = dest_scm.get_ref(orig_ref)
-            if dest_scm.diff(orig_rev, new_rev):
-                if force:
-                    logger.debug(
-                        "Replacing existing experiment '%s'", orig_ref,
-                    )
-                    return True
-                if on_diverged:
-                    checkpoint = self.scm.get_ref(EXEC_CHECKPOINT) is not None
-                    on_diverged(orig_ref, checkpoint)
+            if force:
+                logger.debug(
+                    "Replacing existing experiment '%s'", orig_ref,
+                )
+                return True
+
+            checkpoint = self.scm.get_ref(EXEC_CHECKPOINT) is not None
+            self._raise_ref_conflict(dest_scm, orig_ref, new_rev, checkpoint)
+            if on_diverged:
+                on_diverged(orig_ref, checkpoint)
             logger.debug("Reproduced existing experiment '%s'", orig_ref)
             return False
 
@@ -218,6 +218,7 @@ class BaseExecutor(ABC):
         queue: Optional["Queue"] = None,
         rel_cwd: Optional[str] = None,
         name: Optional[str] = None,
+        log_errors: bool = True,
         log_level: Optional[int] = None,
     ) -> "ExecutorResult":
         """Run dvc repro and return the result.
@@ -234,7 +235,7 @@ class BaseExecutor(ABC):
 
         if queue is not None:
             queue.put((rev, os.getpid()))
-        if log_level is not None:
+        if log_errors and log_level is not None:
             cls._set_log_level(log_level)
 
         def filter_pipeline(stages):
@@ -246,7 +247,7 @@ class BaseExecutor(ABC):
         exp_ref: Optional["ExpRefInfo"] = None
         repro_force: bool = False
 
-        with cls._repro_dvc(dvc_dir, rel_cwd) as dvc:
+        with cls._repro_dvc(dvc_dir, rel_cwd, log_errors) as dvc:
             args, kwargs = cls._repro_args(dvc)
             if args:
                 targets: Optional[Union[list, str]] = args[0]
@@ -258,17 +259,19 @@ class BaseExecutor(ABC):
                 "Executor repro with force = '%s'", str(repro_force)
             )
 
-            # NOTE: for checkpoint experiments we handle persist outs slightly
-            # differently than normal:
+            # NOTE: checkpoint outs are handled as a special type of persist
+            # out:
             #
             # - checkpoint out may not yet exist if this is the first time this
             #   experiment has been run, this is not an error condition for
             #   experiments
-            # - at the start of a repro run, we need to remove the persist out
-            #   and restore it to its last known (committed) state (which may
-            #   be removed/does not yet exist) so that our executor workspace
-            #   is not polluted with the (persistent) out from an unrelated
-            #   experiment run
+            # - if experiment was run with --reset, the checkpoint out will be
+            #   removed at the start of the experiment (regardless of any
+            #   dvc.lock entry for the checkpoint out)
+            # - if run without --reset, the checkpoint out will be checked out
+            #   using any hash present in dvc.lock (or removed if no entry
+            #   exists in dvc.lock)
+            checkpoint_reset: bool = kwargs.pop("reset", False)
             dvc_checkout(
                 dvc,
                 targets=targets,
@@ -276,10 +279,14 @@ class BaseExecutor(ABC):
                 force=True,
                 quiet=True,
                 allow_missing=True,
+                checkpoint_reset=checkpoint_reset,
             )
 
             checkpoint_func = partial(
-                cls.checkpoint_callback, dvc.scm, name, repro_force
+                cls.checkpoint_callback,
+                dvc.scm,
+                name,
+                repro_force or checkpoint_reset,
             )
             stages = dvc_reproduce(
                 dvc,
@@ -291,12 +298,18 @@ class BaseExecutor(ABC):
 
             exp_hash = cls.hash_exp(stages)
             try:
+                is_checkpoint = any(stage.is_checkpoint for stage in stages)
+                if is_checkpoint and checkpoint_reset:
+                    # For reset checkpoint stages, we need to force overwriting
+                    # existing checkpoint refs even though repro may not have
+                    # actually been run with --force
+                    repro_force = True
                 cls.commit(
                     dvc.scm,
                     exp_hash,
                     exp_name=name,
                     force=repro_force,
-                    checkpoint=any(stage.is_checkpoint for stage in stages),
+                    checkpoint=is_checkpoint,
                 )
             except UnchangedExperimentError:
                 pass
@@ -321,7 +334,9 @@ class BaseExecutor(ABC):
 
     @classmethod
     @contextmanager
-    def _repro_dvc(cls, dvc_dir: Optional[str], rel_cwd: Optional[str]):
+    def _repro_dvc(
+        cls, dvc_dir: Optional[str], rel_cwd: Optional[str], log_errors: bool
+    ):
         from dvc.repo import Repo
 
         dvc = Repo(dvc_dir)
@@ -342,10 +357,12 @@ class BaseExecutor(ABC):
         except CheckpointKilledError:
             raise
         except DvcException:
-            logger.exception("")
+            if log_errors:
+                logger.exception("")
             raise
         except Exception:
-            logger.exception("unexpected error")
+            if log_errors:
+                logger.exception("unexpected error")
             raise
         finally:
             dvc.close()
@@ -358,6 +375,8 @@ class BaseExecutor(ABC):
         if os.path.exists(args_path):
             args, kwargs = cls.unpack_repro_args(args_path)
             remove(args_path)
+            # explicitly git rm/unstage the args file
+            dvc.scm.add([args_path])
         else:
             args = []
             kwargs = {}
@@ -396,6 +415,7 @@ class BaseExecutor(ABC):
             logger.debug("No changes to commit")
             raise UnchangedExperimentError(rev)
 
+        check_conflict = False
         branch = scm.get_ref(EXEC_BRANCH, follow=False)
         if branch:
             old_ref = rev
@@ -406,21 +426,41 @@ class BaseExecutor(ABC):
             ref_info = ExpRefInfo(baseline_rev, name)
             branch = str(ref_info)
             old_ref = None
-            if not force and scm.get_ref(branch):
-                if checkpoint:
-                    raise CheckpointExistsError(ref_info.name)
-                raise ExperimentExistsError(ref_info.name)
-            logger.debug("Commit to new experiment branch '%s'", branch)
+            if scm.get_ref(branch):
+                if not force:
+                    check_conflict = True
+                logger.debug(
+                    "%s existing experiment branch '%s'",
+                    "Replace" if force else "Reuse",
+                    branch,
+                )
+            else:
+                logger.debug("Commit to new experiment branch '%s'", branch)
 
         scm.add([], update=True)
         scm.commit(f"dvc: commit experiment {exp_hash}", no_verify=True)
         new_rev = scm.get_rev()
-        scm.set_ref(branch, new_rev, old_ref=old_ref)
+        if check_conflict:
+            new_rev = cls._raise_ref_conflict(scm, branch, new_rev, checkpoint)
+        else:
+            scm.set_ref(branch, new_rev, old_ref=old_ref)
         scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
         if checkpoint:
             scm.set_ref(EXEC_CHECKPOINT, new_rev)
         return new_rev
 
+    @staticmethod
+    def _raise_ref_conflict(scm, ref, new_rev, checkpoint):
+        # If this commit is a duplicate of the existing commit at 'ref', return
+        # the existing commit. Otherwise, error out and require user to re-run
+        # with --force as needed
+        orig_rev = scm.get_ref(ref)
+        if scm.diff(orig_rev, new_rev):
+            if checkpoint:
+                raise CheckpointExistsError(ref)
+            raise ExperimentExistsError(ref)
+        return orig_rev
+
     @staticmethod
     def _set_log_level(level):
         from dvc.logger import disable_other_loggers

dvc/repo/experiments/executor/local.py

@@ -3,13 +3,29 @@ import os
 from tempfile import TemporaryDirectory
 from typing import Optional
 
-from dvc.utils.fs import remove
-
 from .base import BaseExecutor
 
 logger = logging.getLogger(__name__)
 
 
+class ExpTemporaryDirectory(TemporaryDirectory):
+    # Python's TemporaryDirectory cleanup shutil.rmtree wrapper does not handle
+    # git read-only dirs cleanly in Windows on Python <3.8, so we use our own
+    # remove(). See:
+    # https://github.com/iterative/dvc/pull/5425
+    # https://bugs.python.org/issue26660
+
+    @classmethod
+    def _rmtree(cls, name):
+        from dvc.utils.fs import remove
+
+        remove(name)
+
+    def cleanup(self):
+        if self._finalizer.detach():
+            self._rmtree(self.name)
+
+
 class BaseLocalExecutor(BaseExecutor):
     """Base local machine executor."""
 
@@ -37,7 +53,7 @@ class TempDirExecutor(BaseLocalExecutor):
         cache_dir: Optional[str] = None,
         **kwargs,
     ):
-        self._tmp_dir = TemporaryDirectory(dir=tmp_dir)
+        self._tmp_dir = ExpTemporaryDirectory(dir=tmp_dir)
         kwargs["root_dir"] = self._tmp_dir.name
         super().__init__(*args, **kwargs)
         if cache_dir:
@@ -55,4 +71,4 @@ class TempDirExecutor(BaseLocalExecutor):
     def cleanup(self):
         super().cleanup()
         logger.debug("Removing tmpdir '%s'", self._tmp_dir)
-        remove(self._tmp_dir.name)
+        self._tmp_dir.cleanup()

dvc/repo/experiments/run.py

@@ -15,7 +15,6 @@ def run(
     run_all: bool = False,
     jobs: int = 1,
     tmp_dir: bool = False,
-    reset: bool = False,
     **kwargs,
 ) -> dict:
     """Reproduce the specified targets as an experiment.
@@ -25,10 +24,6 @@ def run(
     Returns a dict mapping new experiment SHAs to the results
     of `repro` for that experiment.
     """
-    if reset:
-        repo.experiments.reset_checkpoints()
-        kwargs["force"] = True
-
     if run_all:
         return repo.experiments.reproduce_queued(jobs=jobs)
 

dvc/repo/experiments/show.py

@@ -12,17 +12,23 @@ from dvc.scm.base import SCMError
 logger = logging.getLogger(__name__)
 
 
-def _collect_experiment_commit(repo, rev, stash=False, sha_only=True):
+def _collect_experiment_commit(
+    repo, exp_rev, stash=False, sha_only=True, param_deps=False
+):
     res = defaultdict(dict)
-    for rev in repo.brancher(revs=[rev]):
+    for rev in repo.brancher(revs=[exp_rev]):
         if rev == "workspace":
+            if exp_rev != "workspace":
+                continue
             res["timestamp"] = None
         else:
             commit = repo.scm.resolve_commit(rev)
             res["timestamp"] = datetime.fromtimestamp(commit.commit_time)
 
-        configs = _collect_configs(repo, rev=rev)
-        params = _read_params(repo, configs, rev)
+        params, params_path_infos = _collect_configs(repo, rev=rev)
+        params = _read_params(
+            repo, params, params_path_infos, rev, deps=param_deps
+        )
         if params:
             res["params"] = params
 
@@ -83,6 +89,7 @@ def show(
     all_commits=False,
     sha_only=False,
     num=1,
+    param_deps=False,
 ):
     res = defaultdict(OrderedDict)
 
@@ -110,7 +117,7 @@ def show(
 
     for rev in revs:
         res[rev]["baseline"] = _collect_experiment_commit(
-            repo, rev, sha_only=sha_only
+            repo, rev, sha_only=sha_only, param_deps=param_deps
         )
 
         if rev == "workspace":
@@ -127,14 +134,19 @@ def show(
             ref_info = ExpRefInfo.from_ref(exp_ref)
             assert ref_info.baseline_sha == rev
             _collect_experiment_branch(
-                res[rev], repo, exp_ref, rev, sha_only=sha_only
+                res[rev],
+                repo,
+                exp_ref,
+                rev,
+                sha_only=sha_only,
+                param_deps=param_deps,
             )
 
     # collect queued (not yet reproduced) experiments
     for stash_rev, entry in repo.experiments.stash_revs.items():
         if entry.baseline_rev in revs:
             experiment = _collect_experiment_commit(
-                repo, stash_rev, stash=True
+                repo, stash_rev, stash=True, param_deps=param_deps
             )
             res[entry.baseline_rev][stash_rev] = experiment
 

dvc/repo/fetch.py

@@ -115,6 +115,7 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs):
                     odb,
                     path_info,
                     repo.repo_fs,
+                    "md5",
                     jobs=jobs,
                     follow_subrepos=False,
                 )

dvc/repo/params/show.py

@@ -1,6 +1,7 @@
 import logging
 from collections import defaultdict
-from typing import TYPE_CHECKING, List
+from copy import copy
+from typing import TYPE_CHECKING, List, Tuple
 
 from dvc.dependency.param import ParamsDependency
 from dvc.exceptions import DvcException
@@ -27,7 +28,9 @@ def _is_params(dep: "BaseOutput"):
     return isinstance(dep, ParamsDependency)
 
 
-def _collect_configs(repo: "Repo", rev, targets=None) -> List["DvcPath"]:
+def _collect_configs(
+    repo: "Repo", rev, targets=None
+) -> Tuple[List["BaseOutput"], List["DvcPath"]]:
     params, path_infos = collect(
         repo,
         targets=targets or [],
@@ -35,31 +38,45 @@ def _collect_configs(repo: "Repo", rev, targets=None) -> List["DvcPath"]:
         output_filter=_is_params,
         rev=rev,
     )
-    path_infos.extend([p.path_info for p in params])
+    all_path_infos = path_infos + [p.path_info for p in params]
     if not targets:
         default_params = (
             PathInfo(repo.root_dir) / ParamsDependency.DEFAULT_PARAMS_FILE
         )
-        if default_params not in path_infos:
+        if default_params not in all_path_infos:
             path_infos.append(default_params)
-    return list(path_infos)
+    return params, path_infos
 
 
-def _read_params(repo, configs, rev):
-    res = {}
-    for config in configs:
-        if not repo.fs.exists(config):
-            continue
+def _read_path_info(fs, path_info, rev):
+    if not fs.exists(path_info):
+        return None
 
-        suffix = config.suffix.lower()
-        loader = LOADERS[suffix]
-        try:
-            res[str(config)] = loader(config, fs=repo.fs)
-        except ParseError:
-            logger.debug(
-                "failed to read '%s' on '%s'", config, rev, exc_info=True
-            )
-            continue
+    suffix = path_info.suffix.lower()
+    loader = LOADERS[suffix]
+    try:
+        return loader(path_info, fs=fs)
+    except ParseError:
+        logger.debug(
+            "failed to read '%s' on '%s'", path_info, rev, exc_info=True
+        )
+        return None
+
+
+def _read_params(repo, params, params_path_infos, rev, deps=False):
+    res = defaultdict(dict)
+    path_infos = copy(params_path_infos)
+
+    if deps:
+        for param in params:
+            res[str(param.path_info)].update(param.read_params_d())
+    else:
+        path_infos += [param.path_info for param in params]
+
+    for path_info in path_infos:
+        from_path = _read_path_info(repo.fs, path_info, rev)
+        if from_path:
+            res[str(path_info)] = from_path
 
     return res
 
@@ -79,12 +96,12 @@ def _collect_vars(repo, params):
 
 
 @locked
-def show(repo, revs=None, targets=None):
+def show(repo, revs=None, targets=None, deps=False):
     res = {}
 
     for branch in repo.brancher(revs=revs):
-        configs = _collect_configs(repo, branch, targets)
-        params = _read_params(repo, configs, branch)
+        params, params_path_infos = _collect_configs(repo, branch, targets)
+        params = _read_params(repo, params, params_path_infos, branch, deps)
         vars_params = _collect_vars(repo, params)
 
         # NOTE: only those that are not added as a ParamDependency are included

dvc/scm/git/__init__.py

@@ -272,19 +272,10 @@ class Git(Base):
         self.files_to_track = set()
 
     def remind_to_track(self):
-        def _filter(files):
-            if not files:
-                return set()
-            staged, unstaged, untracked = self.status()
-            return files.intersection(
-                staged.get("modify", []) + unstaged + untracked
-            )
-
-        files_to_track = _filter(self.files_to_track)
-        if self.quiet or not files_to_track:
+        if self.quiet or not self.files_to_track:
             return
 
-        files = " ".join(shlex.quote(path) for path in sorted(files_to_track))
+        files = " ".join(shlex.quote(path) for path in self.files_to_track)
 
         logger.info(
             "\n"

dvc/scm/git/backend/base.py

@@ -67,7 +67,11 @@ class BaseGitBackend(ABC):
 
     @abstractmethod
     def checkout(
-        self, branch: str, create_new: Optional[bool] = False, **kwargs,
+        self,
+        branch: str,
+        create_new: Optional[bool] = False,
+        force: bool = False,
+        **kwargs,
     ):
         pass
 

dvc/scm/git/backend/dulwich.py

@@ -182,14 +182,23 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
     def commit(self, msg: str, no_verify: bool = False):
         from dulwich.errors import CommitError
         from dulwich.porcelain import commit
+        from dulwich.repo import InvalidUserIdentity
 
         try:
             commit(self.root_dir, message=msg, no_verify=no_verify)
         except CommitError as exc:
             raise SCMError("Git commit failed") from exc
+        except InvalidUserIdentity as exc:
+            raise SCMError(
+                "Git username and email must be configured"
+            ) from exc
 
     def checkout(
-        self, branch: str, create_new: Optional[bool] = False, **kwargs,
+        self,
+        branch: str,
+        create_new: Optional[bool] = False,
+        force: bool = False,
+        **kwargs,
     ):
         raise NotImplementedError
 
@@ -524,6 +533,8 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
         message: Optional[str] = None,
         include_untracked: Optional[bool] = False,
     ) -> Tuple[Optional[str], bool]:
+        from dulwich.repo import InvalidUserIdentity
+
         from dvc.scm.git import Stash
 
         if include_untracked or ref == Stash.DEFAULT_STASH:
@@ -533,7 +544,12 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
 
         stash = self._get_stash(ref)
         message_b = message.encode("utf-8") if message else None
-        rev = stash.push(message=message_b)
+        try:
+            rev = stash.push(message=message_b)
+        except InvalidUserIdentity as exc:
+            raise SCMError(
+                "Git username and email must be configured"
+            ) from exc
         return os.fsdecode(rev), True
 
     def _stash_apply(self, rev: str):

dvc/scm/git/backend/gitpython.py

@@ -217,12 +217,16 @@ class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
             raise SCMError("Git pre-commit hook failed") from exc
 
     def checkout(
-        self, branch: str, create_new: Optional[bool] = False, **kwargs
+        self,
+        branch: str,
+        create_new: Optional[bool] = False,
+        force: bool = False,
+        **kwargs,
     ):
         if create_new:
-            self.repo.git.checkout("HEAD", b=branch, **kwargs)
+            self.repo.git.checkout("HEAD", b=branch, force=force, **kwargs)
         else:
-            self.repo.git.checkout(branch, **kwargs)
+            self.repo.git.checkout(branch, force=force, **kwargs)
 
     def pull(self, **kwargs):
         infos = self.repo.remote().pull(**kwargs)

dvc/scm/git/backend/pygit2.py

@@ -80,6 +80,15 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
 
         return RefdbFsBackend(self.repo)
 
+    @property
+    def default_signature(self):
+        try:
+            return self.repo.default_signature
+        except KeyError as exc:
+            raise SCMError(
+                "Git username and email must be configured"
+            ) from exc
+
     @staticmethod
     def clone(
         url: str,
@@ -100,14 +109,20 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         raise NotImplementedError
 
     def checkout(
-        self, branch: str, create_new: Optional[bool] = False, **kwargs,
+        self,
+        branch: str,
+        create_new: Optional[bool] = False,
+        force: bool = False,
+        **kwargs,
     ):
-        from pygit2 import GitError
+        from pygit2 import GIT_CHECKOUT_FORCE, GitError
+
+        checkout_strategy = GIT_CHECKOUT_FORCE if force else None
 
         if create_new:
             commit = self.repo.revparse_single("HEAD")
             new_branch = self.repo.branches.local.create(branch, commit)
-            self.repo.checkout(new_branch)
+            self.repo.checkout(new_branch, strategy=checkout_strategy)
         else:
             if branch == "-":
                 branch = "@{-1}"
@@ -115,7 +130,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
                 commit, ref = self.repo.resolve_refish(branch)
             except (KeyError, GitError):
                 raise RevError(f"unknown Git revision '{branch}'")
-            self.repo.checkout_tree(commit)
+            self.repo.checkout_tree(commit, strategy=checkout_strategy)
             detach = kwargs.get("detach", False)
             if ref and not detach:
                 self.repo.set_head(ref.name)
@@ -322,7 +337,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         from dvc.scm.git import Stash
 
         oid = self.repo.stash(
-            self.repo.default_signature,
+            self.default_signature,
             message=message,
             include_untracked=include_untracked,
         )
@@ -497,7 +512,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
             raise MergeConflictError("Merge contained conflicts")
 
         if commit:
-            user = self.repo.default_signature
+            user = self.default_signature
             tree = self.repo.index.write_tree()
             merge_commit = self.repo.create_commit(
                 "HEAD", user, user, msg, tree, [self.repo.head.target, rev]

dvc/stage/imports.py

@@ -12,7 +12,7 @@ def _update_import_on_remote(stage, remote, jobs):
             "be updated with --to-remote"
         )
 
-    url = stage.deps[0].path_info.url
+    url = stage.deps[0].def_path
     stage.outs[0].hash_info = stage.repo.cloud.transfer(
         url, jobs=jobs, remote=remote, command="update"
     )

dvc/types.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 from dvc.path_info import PathInfo, URLInfo
 
@@ -11,3 +11,5 @@ AnyPath = Union[str, DvcPath, StrPath]
 
 OptStr = Optional[str]
 TargetType = Union[List[str], str]
+DictStrAny = Dict[str, Any]
+DictAny = Dict[Any, Any]

dvc/ui/__init__.py

@@ -1,14 +1,21 @@
 import sys
 from collections import defaultdict
-from typing import Any, Dict, Iterable, Optional, TextIO
-
-from funcy import cached_property
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterable,
+    Optional,
+    Sequence,
+    TextIO,
+    Union,
+)
 
 from dvc.progress import Tqdm
 from dvc.utils import colorize
 
-NEWLINE = "\n"
-SEP = " "
+if TYPE_CHECKING:
+    from dvc.ui.table import Headers, Styles, TableData
 
 
 class Formatter:
@@ -30,14 +37,23 @@ class Console:
         formatter: Formatter = None,
         output: TextIO = None,
         error: TextIO = None,
-        disable: bool = False,
+        enable: bool = False,
     ) -> None:
-        self._input: TextIO = sys.stdin
-        self._output: TextIO = output or sys.stdout
-        self._error: TextIO = error or sys.stderr
-
+        self._output: Optional[TextIO] = output
+        self._error: Optional[TextIO] = error
         self.formatter: Formatter = formatter or Formatter()
-        self.disabled: bool = disable
+        self._enabled: bool = enable
+
+    @property
+    def output(self) -> TextIO:
+        return self._output or sys.stdout
+
+    @property
+    def error_output(self) -> TextIO:
+        return self._error or sys.stderr
+
+    def enable(self) -> None:
+        self._enabled = True
 
     def success(self, message: str) -> None:
         self.write(message, style="success")
@@ -52,8 +68,8 @@ class Console:
         self,
         *objects: Any,
         style: str = None,
-        sep: str = SEP,
-        end: str = NEWLINE,
+        sep: str = None,
+        end: str = None,
         flush: bool = False,
     ) -> None:
         return self.write(
@@ -61,7 +77,7 @@ class Console:
             style=style,
             sep=sep,
             end=end,
-            file=self._error,
+            file=self.error_output,
             flush=flush,
         )
 
@@ -69,20 +85,21 @@ class Console:
         self,
         *objects: Any,
         style: str = None,
-        sep: str = SEP,
-        end: str = NEWLINE,
+        sep: str = None,
+        end: str = None,
         file: TextIO = None,
         flush: bool = False,
+        force: bool = False,
     ) -> None:
-        if self.disabled:
+        if not self._enabled and not force:
             return
 
-        file = file or self._output
+        file = file or self.output
         values = (self.formatter.format(obj, style=style) for obj in objects)
         return print(*values, sep=sep, end=end, file=file, flush=flush)
 
     def progress(self, *args, **kwargs) -> Tqdm:
-        kwargs.setdefault("file", self._error)
+        kwargs.setdefault("file", self.error_output)
         return Tqdm(*args, **kwargs)
 
     def prompt(
@@ -120,47 +137,64 @@ class Console:
             return False
         return answer.startswith("y")
 
-    @cached_property
+    @property
     def rich_console(self):
         """rich_console is only set to stdout for now."""
         from rich import console
 
-        return console.Console(file=self._output)
+        # FIXME: Getting IO Operation on closed file error
+        #  when testing with capsys, therefore we are creating
+        #  one instance each time as a temporary workaround.
+        return console.Console(file=self.output)
 
-    def rich_table(self, pager: bool = True):
-        pass
+    def table(
+        self,
+        data: "TableData",
+        headers: "Headers" = None,
+        markdown: bool = False,
+        rich_table: bool = False,
+        force: bool = True,
+        pager: bool = False,
+        header_styles: Sequence["Styles"] = None,
+        row_styles: Sequence["Styles"] = None,
+        borders: Union[bool, str] = False,
+    ) -> None:
+        from dvc.ui import table as t
 
-    def table(self, header, rows, markdown: bool = False):
-        from tabulate import tabulate
+        if not data and not markdown:
+            return
 
-        if not rows and not markdown:
-            return ""
+        if not markdown and rich_table:
+            if force or self._enabled:
+                return t.rich_table(
+                    self,
+                    data,
+                    headers,
+                    pager=pager,
+                    header_styles=header_styles,
+                    row_styles=row_styles,
+                    borders=borders,
+                )
 
-        ret = tabulate(
-            rows,
-            header,
-            tablefmt="github" if markdown else "plain",
-            disable_numparse=True,
-            # None will be shown as "" by default, overriding
-            missingval="—",
+            return
+
+        return t.plain_table(
+            self, data, headers, markdown=markdown, pager=pager, force=force,
         )
 
-        if markdown:
-            # NOTE: md table is incomplete without the trailing newline
-            ret += "\n"
 
-        self.write(ret)
+ui = Console()
 
 
 if __name__ == "__main__":
-    ui = Console()
+    ui.enable()
 
     ui.write("No default remote set")
     ui.success("Everything is up to date.")
     ui.warn("Run queued experiments will be removed.")
     ui.error("too few arguments.")
 
-    ui.table("keys", {"Path": ["scores.json"], "auc": ["0.5674"]})
+    ui.table([("scores.json", "0.5674")], headers=["Path", "auc"])
     ui.table(
-        "keys", {"Path": ["scores.json"], "auc": ["0.5674"]}, markdown=True
+        [("scores.json", "0.5674")], headers=["Path", "auc"], markdown=True
     )

dvc/ui/table.py

@@ -0,0 +1,119 @@
+from contextlib import contextmanager
+from itertools import zip_longest
+from typing import TYPE_CHECKING, Iterator, List, Sequence, Union
+
+from dvc.types import DictStrAny
+
+if TYPE_CHECKING:
+    from rich.console import Console as RichConsole
+    from rich.table import Table
+    from rich.text import Text
+
+    from dvc.ui import Console
+
+
+SHOW_MAX_WIDTH = 1024
+
+
+CellT = Union[str, "Text"]  # Text is mostly compatible with str
+Row = Sequence[CellT]
+TableData = List[Row]
+Headers = Sequence[str]
+Styles = DictStrAny
+
+
+def plain_table(
+    ui: "Console",
+    data: TableData,
+    headers: Headers = None,
+    markdown: bool = False,
+    pager: bool = False,
+    force: bool = True,
+) -> None:
+    from tabulate import tabulate
+
+    text: str = tabulate(
+        data,
+        headers if headers is not None else (),
+        tablefmt="github" if markdown else "plain",
+        disable_numparse=True,
+        # None will be shown as "" by default, overriding
+        missingval="-",
+    )
+    if markdown:
+        # NOTE: md table is incomplete without the trailing newline
+        text += "\n"
+
+    if pager:
+        from dvc.utils.pager import pager as _pager
+
+        _pager(text)
+    else:
+        ui.write(text, force=force)
+
+
+@contextmanager
+def console_width(
+    table: "Table", console: "RichConsole", val: int
+) -> Iterator[None]:
+    # NOTE: rich does not have native support for unlimited width
+    # via pager. we override rich table compression by setting
+    # console width to the full width of the table
+    # pylint: disable=protected-access
+
+    console_options = console.options
+    original = console_options.max_width
+    con_width = console._width
+
+    try:
+        console_options.max_width = val
+        measurement = table.__rich_measure__(console, console_options)
+        console._width = measurement.maximum
+
+        yield
+    finally:
+        console_options.max_width = original
+        console._width = con_width
+
+
+def rich_table(
+    ui: "Console",
+    data: TableData,
+    headers: Headers = None,
+    pager: bool = False,
+    header_styles: Sequence[Styles] = None,
+    row_styles: Sequence[Styles] = None,
+    borders: Union[bool, str] = False,
+) -> None:
+    from rich import box
+
+    from dvc.utils.table import Table
+
+    border_style = {
+        True: box.HEAVY_HEAD,  # is a default in rich,
+        False: None,
+        "simple": box.SIMPLE,
+        "minimal": box.MINIMAL,
+    }
+
+    table = Table(box=border_style[borders])
+    hs: Sequence[Styles] = header_styles or []
+    rs: Sequence[Styles] = row_styles or []
+
+    for header, style in zip_longest(headers or [], hs):
+        table.add_column(header, **(style or {}))
+
+    for row, style in zip_longest(data, rs):
+        table.add_row(*row, **(style or {}))
+
+    console = ui.rich_console
+
+    if not pager:
+        console.print(table)
+        return
+
+    from dvc.utils.pager import DvcPager
+
+    with console_width(table, console, SHOW_MAX_WIDTH):
+        with console.pager(pager=DvcPager(), styles=True):
+            console.print(table)

dvc/utils/pager.py

@@ -46,7 +46,7 @@ def find_pager():
     return pydoc.plainpager
 
 
-def pager(text):
+def pager(text: str) -> None:
     find_pager()(text)
 
 

dvc/utils/table.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING, List, cast
 
 from rich.style import StyleType
 from rich.table import Column as RichColumn
@@ -8,6 +8,7 @@ from rich.table import Table as RichTable
 if TYPE_CHECKING:
     from rich.console import (
         Console,
+        ConsoleOptions,
         JustifyMethod,
         OverflowMethod,
         RenderableType,
@@ -56,7 +57,7 @@ class Table(RichTable):
         self.columns.append(column)
 
     def _calculate_column_widths(
-        self, console: "Console", max_width: int
+        self, console: "Console", options: "ConsoleOptions"
     ) -> List[int]:
         """Calculate the widths of each column, including padding, not
         including borders.
@@ -64,22 +65,23 @@ class Table(RichTable):
         Adjacent collapsed columns will be removed until there is only a single
         truncated column remaining.
         """
-        widths = super()._calculate_column_widths(console, max_width)
+        widths = super()._calculate_column_widths(console, options)
         last_collapsed = -1
-        for i in range(len(self.columns) - 1, -1, -1):
-            if widths[i] == 1 and self.columns[i].collapse:
+        columns = cast(List[Column], self.columns)
+        for i in range(len(columns) - 1, -1, -1):
+            if widths[i] == 0 and columns[i].collapse:
                 if last_collapsed >= 0:
                     del widths[last_collapsed]
-                    del self.columns[last_collapsed]
+                    del columns[last_collapsed]
                     if self.box:
-                        max_width += 1
-                    for column in self.columns[last_collapsed:]:
+                        options.max_width += 1
+                    for column in columns[last_collapsed:]:
                         column._index -= 1
                 last_collapsed = i
                 padding = self._get_padding_width(i)
                 if (
-                    self.columns[i].overflow == "ellipsis"
-                    and (sum(widths) + padding) <= max_width
+                    columns[i].overflow == "ellipsis"
+                    and (sum(widths) + padding) <= options.max_width
                 ):
                     # Set content width to 1 (plus padding) if we can fit a
                     # single unicode ellipsis in this column
@@ -88,7 +90,7 @@ class Table(RichTable):
                 last_collapsed = -1
         return widths
 
-    def _collapse_widths(
+    def _collapse_widths(  # type: ignore[override]
         self, widths: List[int], wrapable: List[bool], max_width: int,
     ) -> List[int]:
         """Collapse columns right-to-left if possible to fit table into
@@ -97,7 +99,8 @@ class Table(RichTable):
         If table is still too wide after collapsing, rich's automatic overflow
         handling will be used.
         """
-        collapsible = [column.collapse for column in self.columns]
+        columns = cast(List[Column], self.columns)
+        collapsible = [column.collapse for column in columns]
         total_width = sum(widths)
         excess_width = total_width - max_width
         if any(collapsible):

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.0.4"
+_BASE_VERSION = "2.0.18"
 
 
 def _generate_version(base_version):

scripts/build-requirements.txt

@@ -1,2 +1,2 @@
-PyInstaller==3.5
+PyInstaller==4.2
 numpy==1.19.3

scripts/build.py

@@ -0,0 +1,103 @@
+import argparse
+import importlib
+import pathlib
+import textwrap
+from subprocess import STDOUT, check_call
+
+path = pathlib.Path(__file__).parent.absolute()
+dvc = path.parent / "dvc"
+pyinstaller = path / "pyinstaller"
+innosetup = path / "innosetup"
+fpm = path / "fpm"
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "pkg", choices=["deb", "rpm", "osxpkg", "exe"], help="package type"
+)
+parser.add_argument("--sign-application", default=False, action="store_true")
+parser.add_argument("--application-id")
+parser.add_argument("--sign-installer", default=False, action="store_true")
+parser.add_argument("--installer-id")
+parser.add_argument("--notarize", default=False, action="store_true")
+parser.add_argument("--apple-id-username")
+parser.add_argument("--apple-id-password")
+args = parser.parse_args()
+
+(dvc / "utils" / "build.py").write_text(f'PKG = "{args.pkg}"')
+
+# Autogenerate version, similar to what we do in setup.py
+spec = importlib.util.spec_from_file_location(
+    "dvc.version", dvc / "version.py"
+)
+dvc_version = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(dvc_version)
+version = dvc_version.__version__
+
+(dvc / "version.py").write_text(
+    textwrap.dedent(
+        f"""\
+        # AUTOGENERATED at build time by scripts/build.py
+        __version__ = "{version}"
+    """
+    )
+)
+
+check_call(
+    ["python", "build.py"], cwd=pyinstaller, stderr=STDOUT,
+)
+
+if args.sign_application:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not args.application_id:
+        print("--sign-application requires --application-id")
+        exit(1)
+    check_call(
+        ["python", "sign.py", "--application-id", args.application_id],
+        cwd=pyinstaller,
+        stderr=STDOUT,
+    )
+
+if args.pkg == "exe":
+    check_call(
+        ["python", "build.py"], cwd=innosetup, stderr=STDOUT,
+    )
+else:
+    check_call(
+        ["python", "build.py", args.pkg], cwd=fpm, stderr=STDOUT,
+    )
+
+if args.sign_installer:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not all(
+        [args.installer_id, args.apple_id_username, args.apple_id_password]
+    ):
+        print("--sign-installer requires --installer-id")
+        exit(1)
+    check_call(
+        ["python", "sign.py", "--installer-id", args.installer_id],
+        cwd=fpm,
+        stderr=STDOUT,
+    )
+
+if args.notarize:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not all([args.apple_id_username, args.apple_id_password]):
+        print(
+            "--notarize requires --apple-id-username and --apple-id-password"
+        )
+        exit(1)
+    check_call(
+        [
+            "python",
+            "notarize.py",
+            "--apple-id-username",
+            args.apple_id_username,
+            "--apple-id-password",
+            args.apple_id_password,
+        ],
+        cwd=fpm,
+        stderr=STDOUT,
+    )

scripts/build_posix.sh

@@ -1,139 +0,0 @@
-#!/bin/bash
-
-set -e
-set -x
-
-FPM_FLAGS='--description "Data Version Control - datasets, models, and experiments versioning for ML or data science projects"'
-FPM_FLAGS+=' -n dvc -s dir -f'
-FPM_FLAGS+=' --url https://dvc.org'
-FPM_FLAGS+=' --license "Apache License 2.0"'
-
-if [[ "$(uname)" == 'Linux' ]]; then
-  INSTALL_DIR=usr
-  FPM_FLAGS+=' --depends "git >= 1.7.0"'  # needed for gitpython
-else
-  INSTALL_DIR=usr/local
-  FPM_FLAGS+=' --osxpkg-identifier-prefix com.iterative'
-  FPM_FLAGS+=' --after-install scripts/fpm/after-install.sh'
-  FPM_FLAGS+=' --after-remove scripts/fpm/after-remove.sh'
-fi
-
-BUILD_DIR=build
-BIN_DIR=$BUILD_DIR/$INSTALL_DIR/bin
-LIB_DIR=$BUILD_DIR/$INSTALL_DIR/lib
-
-FPM_PACKAGE_DIRS="usr"
-ZSH_CMPLT_DIR=usr/share/zsh/site-functions/_dvc
-if [[ "$(uname)" == 'Linux' ]]; then
-  BASH_CMPLT_DIR=etc/bash_completion.d
-  FPM_PACKAGE_DIRS="$FPM_PACKAGE_DIRS etc"
-else
-  BASH_CMPLT_DIR=usr/local/etc/bash_completion.d
-fi
-
-print_error() {
-  echo -e "\e[31m$1\e[0m" >&2
-}
-
-if [ ! -d "dvc" ]; then
-  print_error "Please run this script from repository root"
-  exit 1
-fi
-
-trap 'print_error "FAIL"; exit 1' ERR
-
-print_info() {
-  echo -e "\e[32m$1\e[0m"
-}
-
-command_exists() {
-  command -v $1 >/dev/null 2>&1
-}
-
-fpm_build() {
-  print_info "Building $1..."
-  VERSION=$(python -c "import dvc; from dvc import __version__; print(str(__version__))")
-  eval "fpm -t $1 $FPM_FLAGS -v $VERSION -C $BUILD_DIR $FPM_PACKAGE_DIRS"
-}
-
-cleanup() {
-  print_info "Cleaning up..."
-  rm -rf build
-}
-
-install_dependencies() {
-  print_info "Installing fpm..."
-  if command_exists dnf; then
-    sudo dnf install ruby-devel gcc make rpm-build
-  elif command_exists yum; then
-    sudo yum install ruby-devel gcc make rpm-build
-  elif command_exists apt-get; then
-    sudo apt-get update -y
-    sudo apt-get install ruby-dev build-essential rpm python-pip python-dev
-  elif command_exists brew; then
-    brew install ruby
-  else
-    echo "Unable to install fpm dependencies" && exit 1
-  fi
-
-  gem install --no-document fpm
-
-  # https://github.com/iterative/dvc/issues/2970
-  gem uninstall -i /Users/travis/.rvm/gems/ruby-2.4.3@global rubygems-bundler
-
-  print_info "Installing requirements..."
-  pip install .[all]
-  pip install -r scripts/build-requirements.txt
-}
-
-build_dvc() {
-  print_info "Building dvc binary..."
-  pyinstaller \
-    --additional-hooks-dir $(pwd)/scripts/hooks dvc/__main__.py \
-    --name dvc \
-    --distpath $LIB_DIR \
-    --specpath $BUILD_DIR
-
-  $LIB_DIR/dvc/dvc --help
-
-  # NOTE: in osxpkg fpm replaces symlinks with actual file that it
-  # points to, so we need to use after-install hook. See FPM_FLAGS
-  # above.
-  if [[ "$(uname)" == 'Linux' ]]; then
-    mkdir -p $BIN_DIR
-    pushd $BIN_DIR
-    ln -s ../lib/dvc/dvc dvc
-    popd
-    $BIN_DIR/dvc --help
-  fi
-
-  # NOTE: temporarily not adding scripts to mac package. See [1]
-  # [1] https://github.com/iterative/dvc/issues/2585
-  if [[ "$(uname)" == 'Linux' ]]; then
-    mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR
-    $LIB_DIR/dvc/dvc completion -s bash > $BUILD_DIR/$BASH_CMPLT_DIR/dvc
-
-    mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR
-    $LIB_DIR/dvc/dvc completion -s zsh > $BUILD_DIR/$ZSH_CMPLT_DIR/_dvc
-  fi
-}
-
-build() {
-  cleanup
-  echo "PKG = \"$1\"" >dvc/utils/build.py
-  build_dvc
-  fpm_build $1
-}
-
-install_dependencies
-
-if [[ "$(uname)" == 'Linux' ]]; then
-  build rpm
-  build deb
-else
-  build osxpkg
-fi
-
-cleanup
-
-print_info "Successfully built dvc packages"

scripts/build_windows.cmd

@@ -1,48 +0,0 @@
-@echo Off
-
-echo ====== Starting to build dvc installer for Windows... ======
-
-if not exist dvc\NUL (echo Error: Please run this script from repository root && goto :error)
-
-rmdir /Q /S build
-rmdir /Q /S dist
-del /Q /S dvc.spec
-del /Q /S "dvc-*.exe"
-
-where pip
-if %errorlevel% neq 0 (echo Error: pip not found && goto :error)
-
-where choco
-if %errorlevel% neq 0 (echo Error: choco not found && goto :error)
-
-choco install InnoSetup
-call refreshenv
-where iscc
-if %errorlevel% neq 0 (echo Error: Couldn't find Inno Setup compiler. && goto :error)
-
-echo ====== Installing requirements... ======
-echo PKG = "exe" > dvc\utils\build.py
-call pip install .[all] || goto :error
-call pip install -r scripts\build-requirements.txt || goto :error
-call dvc pull || goto :error
-
-echo ====== Building dvc binary... ======
-call pyinstaller --additional-hooks-dir scripts\hooks dvc/__main__.py --name dvc --specpath build
-
-echo ====== Testing dvc binary... ======
-call dist\dvc\dvc.exe version || goto :error
-
-echo ====== Copying additional files... ======
-copy scripts\innosetup\addSymLinkPermissions.ps1 dist\ || goto :error
-
-echo ====== Building dvc installer... ======
-set PYTHONPATH=%cd%
-call python scripts\innosetup\config_gen.py || goto :error
-call iscc scripts\innosetup\setup.iss || goto :error
-
-echo ====== DONE ======
-goto :EOF
-
-:error
-echo ====== FAIL ======
-exit /b 1

scripts/fpm/.gitignore

@@ -0,0 +1,4 @@
+/build
+/*.deb
+/*.rpm
+/*.pkg

scripts/fpm/build.py

@@ -0,0 +1,96 @@
+import argparse
+import os
+import pathlib
+import shutil
+from subprocess import STDOUT, check_call, check_output
+
+path = pathlib.Path(__file__).parent.absolute()
+dvc = path.parent.parent / "dvc"
+pyinstaller = path.parent / "pyinstaller"
+
+build = path / "build"
+install = build / "usr"
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "pkg", choices=["deb", "rpm", "osxpkg"], help="package type"
+)
+args = parser.parse_args()
+
+flags = [
+    "--description",
+    '"Data Version Control | Git for Data & Models"',
+    "-n",
+    "dvc",
+    "-s",
+    "dir",
+    "-f",
+    "--license",
+    '"Apache License 2.0"',
+]
+
+if args.pkg == "osxpkg":
+    install /= "local"
+    bash_dir = install / "etc" / "bash_completion.d"
+    dirs = ["usr"]
+    flags.extend(
+        [
+            "--osxpkg-identifier-prefix",
+            "com.iterative",
+            "--after-install",
+            path / "after-install.sh",
+            "--after-remove",
+            path / "after-remove.sh",
+        ]
+    )
+else:
+    bash_dir = build / "etc" / "bash_completion.d"
+    dirs = ["usr", "etc"]
+    flags.extend(["--depends", "git >= 1.7.0"])  # needed for gitpython
+
+try:
+    shutil.rmtree(build)
+except FileNotFoundError:
+    pass
+
+lib = install / "lib"
+lib.mkdir(parents=True)
+shutil.copytree(pyinstaller / "dist" / "dvc", lib / "dvc")
+
+if args.pkg != "osxpkg":
+    # NOTE: in osxpkg fpm replaces symlinks with actual file that it
+    # points to, so we need to use after-install hook.
+    (install / "bin").mkdir()
+    os.symlink("../lib/dvc/dvc", install / "bin" / "dvc")
+
+bash_dir.mkdir(parents=True)
+bash_completion = check_output(
+    [lib / "dvc" / "dvc", "completion", "-s", "bash"], text=True
+)
+(bash_dir / "dvc").write_text(bash_completion)
+
+zsh_dir = install / "share" / "zsh" / "site-functions"
+zsh_dir.mkdir(parents=True)
+zsh_completion = check_output(
+    [lib / "dvc" / "dvc", "completion", "-s", "zsh"], text=True
+)
+(zsh_dir / "_dvc").write_text(zsh_completion)
+
+version = check_output([lib / "dvc" / "dvc", "--version"], text=True).strip()
+
+check_call(
+    [
+        "fpm",
+        "--verbose",
+        "-t",
+        args.pkg,
+        *flags,
+        "-v",
+        version,
+        "-C",
+        build,
+        *dirs,
+    ],
+    cwd=path,
+    stderr=STDOUT,
+)

scripts/fpm/notarize.py

@@ -0,0 +1,64 @@
+import argparse
+import json
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "path",
+    nargs="?",
+    help="Path to the osxpkg to notarize. If not specified - try to find one.",
+)
+parser.add_argument(
+    "--apple-id-username", required=True, help="Apple ID username."
+)
+parser.add_argument(
+    "--apple-id-password",
+    required=True,
+    help=(
+        "Apple ID app-specific password. Note that this is not a regular "
+        "Apple ID password, so you need to generate one at "
+        "https://appleid.apple.com/account/manage"
+    ),
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+
+if args.path:
+    pkg = pathlib.Path(args.path)
+else:
+    pkgs = list(path.glob("*.pkg"))
+    if not pkgs:
+        print("No pkgs found")
+        exit(1)
+
+    if len(pkgs) > 1:
+        print("Too many packages")
+        exit(1)
+
+    (pkg,) = pkgs
+
+
+config = {
+    "notarize": {
+        "path": os.fspath(pkg),
+        "bundle_id": "com.iterative.dvc",
+        "staple": True,
+    },
+    "apple_id": {
+        "username": args.apple_id_username,
+        "password": args.apple_id_password,
+    },
+}
+
+(path / "config.json").write_text(json.dumps(config))
+
+check_call(
+    ["gon", "config.json"], cwd=path, stderr=STDOUT,
+)

scripts/fpm/sign.py

@@ -0,0 +1,54 @@
+import argparse
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "path",
+    nargs="?",
+    help="Path to the osxpkg to sign. If not specified - try to find one.",
+)
+parser.add_argument(
+    "--installer-id",
+    required=True,
+    help="Certificate ID (should be added to the keychain).",
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+
+if args.path:
+    pkg = pathlib.Path(args.path)
+else:
+    pkgs = list(path.glob("*.pkg"))
+    if not pkgs:
+        print("No pkgs found")
+        exit(1)
+
+    if len(pkgs) > 1:
+        print("Too many packages")
+        exit(1)
+
+    (pkg,) = pkgs
+
+unsigned = pkg.with_suffix(".unsigned")
+os.rename(pkg, unsigned)
+check_call(
+    [
+        "productsign",
+        "--sign",
+        args.installer_id,
+        os.fspath(unsigned),
+        os.fspath(pkg),
+    ],
+    stderr=STDOUT,
+)
+
+check_call(
+    ["pkgutil", "--check-signature", os.fspath(pkg)], stderr=STDOUT,
+)

scripts/innosetup/.gitignore

@@ -1,4 +1,6 @@
-dvc.ico
-dvc_left.bmp
-dvc_up.bmp
+/dvc.ico
+/dvc_left.bmp
+/dvc_up.bmp
 /config.ini
+/build
+/dvc*.exe

scripts/innosetup/build.py

@@ -0,0 +1,42 @@
+import argparse
+import configparser
+import os
+import pathlib
+import shutil
+from subprocess import STDOUT, check_call, check_output
+
+path = pathlib.Path(__file__).parent.absolute()
+config = path / "config.ini"
+
+dvc = path.parent.parent / "dvc"
+pyinstaller = path.parent / "pyinstaller"
+
+build = path / "build"
+install = build / "usr"
+
+parser = argparse.ArgumentParser()
+args = parser.parse_args()
+
+try:
+    shutil.rmtree(build)
+except FileNotFoundError:
+    pass
+
+build.mkdir()
+shutil.copytree(pyinstaller / "dist" / "dvc", build / "dvc")
+shutil.copy(path / "addSymLinkPermissions.ps1", build)
+
+version = check_output(
+    [os.fspath(build / "dvc" / "dvc"), "--version"], text=True
+).strip()
+
+cfg = configparser.ConfigParser()
+cfg.add_section("Version")
+cfg.set("Version", "version", version)
+
+with (path / "config.ini").open("w") as fobj:
+    cfg.write(fobj)
+
+check_call(
+    ["iscc", "setup.iss"], cwd=path, stderr=STDOUT,
+)

scripts/innosetup/config_gen.py

@@ -1,11 +0,0 @@
-# This script generates config.ini for setup.iss script
-import configparser
-
-from dvc import __version__
-
-config = configparser.ConfigParser()
-config.add_section("Version")
-config.set("Version", "Version", __version__)
-
-with open("scripts/innosetup/config.ini", "w") as f:
-    config.write(f)

scripts/innosetup/setup.iss

@@ -1,8 +1,8 @@
 #define MyAppName "DVC (Data Version Control)"
-#define MyAppVersion ReadIni(".\scripts\innosetup\config.ini", "Version", "version", "unknown")
+#define MyAppVersion ReadIni(SourcePath + "\config.ini", "Version", "version", "unknown")
 #define MyAppPublisher "Dmitry Petrov"
-#define MyAppURL "https://dataversioncontrol.com/"
-#define MyAppDir "..\..\dist\dvc"
+#define MyAppURL "https://dvc.org"
+#define MyAppDir SourcePath + "\build\dvc"
 
 [Setup]
 AppId={{8258CE8A-110E-4E0D-AE60-FEE00B15F041}
@@ -19,7 +19,7 @@ LicenseFile=..\..\LICENSE
 OutputBaseFilename=dvc-{#MyAppVersion}
 Compression=lzma2/max
 SolidCompression=yes
-OutputDir=..\..\
+OutputDir=.
 ChangesEnvironment=yes
 SetupIconFile=dvc.ico
 WizardSmallImageFile=dvc_up.bmp

scripts/pyinstaller/.gitignore

@@ -0,0 +1,4 @@
+/build
+/dist
+/dvc.spec
+/dvc

scripts/pyinstaller/build.py

@@ -0,0 +1,22 @@
+import os
+import pathlib
+from subprocess import STDOUT, check_call
+
+path = pathlib.Path(__file__).parent.absolute()
+hooks = path / "hooks"
+dvc = path.parent.parent / "dvc"
+entry = dvc / "__main__.py"
+
+check_call(
+    [
+        "pyinstaller",
+        "--additional-hooks-dir",
+        os.fspath(hooks),
+        "--name",
+        "dvc",
+        "-y",
+        os.fspath(entry),
+    ],
+    cwd=path,
+    stderr=STDOUT,
+)

scripts/pyinstaller/entitlements.plist

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<!-- These are required for binaries built by PyInstaller -->
+	<key>com.apple.security.cs.allow-jit</key>
+	<true/>
+	<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+	<true/>
+</dict>
+</plist>

scripts/pyinstaller/hooks/hook-googleapiclient.model.py

@@ -0,0 +1,8 @@
+"""Temporary fix for https://github.com/iterative/dvc/issues/5618."""
+from PyInstaller.utils.hooks import (  # pylint:disable=import-error
+    collect_data_files,
+    copy_metadata,
+)
+
+datas = collect_data_files("googleapiclient.discovery")
+datas += copy_metadata("google_api_python_client")

scripts/pyinstaller/sign.py

@@ -0,0 +1,39 @@
+import argparse
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--application-id",
+    required=True,
+    help="Certificate ID (should be added to the keychain).",
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+dvc = path / "dist" / "dvc"
+for root, _, fnames in os.walk(dvc):
+    for fname in fnames:
+        fpath = os.path.join(root, fname)
+        print(f"signing {fpath}")
+        check_call(
+            [
+                "codesign",
+                "--force",
+                "--verbose",
+                "-s",
+                args.application_id,
+                "-o",
+                "runtime",
+                "--entitlements",
+                "entitlements.plist",
+                fpath,
+            ],
+            stderr=STDOUT,
+            timeout=180,
+        )