dvc: bump to 2.0.18

This part of logic used to be only used by external repo for cloned
repos, but since we now handle local repos without cloning, we don't
always pass scm instance. This PR automatically creates scm instance
for such cases.

Fixes #5590

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+
+updates:
+  - directory: "/"
+    package-ecosystem: "pip"
+    schedule:
+      interval: "daily"
+
+  - directory: "/"
+    package-ecosystem: "github-actions"
+    schedule:
+      interval: "daily"

.github/workflows/packages.yaml

@@ -6,91 +6,130 @@ on:
   release:
     types: [released, prereleased]
 jobs:
-  linux:
+  deb:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+    - uses: actions/[email protected].4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2.2.2
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Set up Ruby 2.4
       uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.4
-    - name: Build
-      run: ./scripts/build_posix.sh
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build deb
+      run: python scripts/build.py deb
     - name: Publish deb
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_path: scripts/fpm/dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_content_type: binary/octet-stream
-    - name: Publish rpm
-      if: github.event_name == 'release'
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
-        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
-        asset_content_type: binary/octet-stream
-  mac:
-    runs-on: macos-10.15
+  rpm:
+    runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+    - uses: actions/[email protected].4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2.2.2
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Set up Ruby 2.4
       uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.4
-    - name: Build
-      run: ./scripts/build_posix.sh
-    - name: Publish
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build rpm
+      run: python scripts/build.py rpm
+    - name: Publish rpm
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_content_type: binary/octet-stream
+  osxpkg:
+    runs-on: macos-10.15
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Install deps
+      run: |
+        # available lxml (webdav dependency) wheels are built with an old
+        # SDK, which breaks osxpkg notarization. Building from scratch
+        # instead.
+        pip install .[all] --no-binary lxml
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build
+      run: python scripts/build.py osxpkg
+    - name: Publish
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}.pkg
         asset_name: dvc-${{ github.event.release.tag_name }}.pkg
         asset_content_type: binary/octet-stream
-  windows:
+  exe:
     runs-on: windows-2019
     steps:
-    - uses: actions/[email protected].2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+    - uses: actions/[email protected].4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2.2.2
       with:
-        python-version: 3.7
+        python-version: 3.8
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+    - name: Pull images
+      run: |
+        dvc pull
     - name: Build
-      run: scripts\build_windows.cmd
+      run: python scripts\build.py exe
       shell: cmd
     - name: Publish
       if: github.event_name == 'release'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}.exe
+        asset_path: scripts/innosetup/dvc-${{ github.event.release.tag_name }}.exe
         asset_name: dvc-${{ github.event.release.tag_name }}.exe
         asset_content_type: binary/octet-stream
-  python:
+  pip:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.7
     - name: Install
@@ -103,29 +142,3 @@ jobs:
       with:
         user: __token__
         password: ${{ secrets.PYPI_TOKEN }}
-  snap:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/[email protected]
-    - name: Build the package
-      uses: snapcore/action-build@v1
-      id: snapcraft
-#    - name: Test the package
-#      run: |
-#        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
-#        dvc version
-#        sudo snap remove dvc
-    - name: Publish to the edge channel
-      uses: snapcore/action-publish@v1
-      if: github.event_name == 'push'
-      with:
-        store_login: ${{ secrets.STORE_LOGIN }}
-        snap: ${{ steps.snapcraft.outputs.snap }}
-        release: edge,v2/edge
-    - name: Publish to the stable channel
-      uses: snapcore/action-publish@v1
-      if: ${{ github.event_name == 'release' && github.event.release.released }}
-      with:
-        store_login: ${{ secrets.STORE_LOGIN }}
-        snap: ${{ steps.snapcraft.outputs.snap }}
-        release: stable,v2/stable

.github/workflows/tests.yaml

@@ -21,20 +21,22 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.8
     - name: Install requirements
-      run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
+      run: |
+        pip install wheel
+        pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
     - name: install temporary dependencies
       run: |
         pip install git+https://github.com/isidentical/gdrivefs@service-account
         pip install git+https://github.com/isidentical/pydata-google-auth@service-account
     - name: Check README
       run: python setup.py checkdocs
-    - uses: pre-commit/[email protected].0
+    - uses: pre-commit/[email protected].2
   tests:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -51,9 +53,9 @@ jobs:
             pyv: "3.9"
 
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: ${{ matrix.pyv }}
     - name: get pip cache dir
@@ -62,7 +64,7 @@ jobs:
         echo "::set-output name=dir::$(pip cache dir)"
     - name: set pip cache
       id: pip-cache
-      uses: actions/cache@v2
+      uses: actions/cache@v2.1.5
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
@@ -88,9 +90,9 @@ jobs:
         mkdir -p scripts/ci
         echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
     - name: run tests
-      run: python -m tests --all --cov-report=xml --cov-report=term --tap-combined -n=4
+      run: python -m tests --all --cov-report=xml --cov-report=term -n=4
     - name: upload coverage report
-      uses: codecov/codecov-action@v1.0.13
+      uses: codecov/codecov-action@v1.3.2
       with:
         file: ./coverage.xml
         fail_ci_if_error: false

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 5.6.4
+    rev: 5.8.0
   - hooks:
       - id: flake8
         language_version: python3
@@ -18,13 +18,13 @@ repos:
           - flake8-debugger
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 3.9.0
   - hooks:
       -  id: mypy
          language_version: python3
          files: ^dvc/
     repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.790'
+    rev: 'v0.812'
   - repo: local
     hooks:
     - id: pylint

MANIFEST.in

@@ -1,2 +1,3 @@
 include fastentrypoints.py
 include LICENSE
+include test_requirements.txt

README.rst

@@ -129,7 +129,11 @@ Conda (Anaconda)
 
 .. code-block:: bash
 
-   conda install -c conda-forge dvc
+   conda install -c conda-forge mamba # installs much faster than conda
+   mamba install -c conda-forge dvc
+
+Depending on the remote storage type you plan to use to keep and share your data, you might need to
+install optional dependencies: `dvc-s3`, `dvc-azure`, `dvc-gdrive`, `dvc-gs`, `dvc-oss`, `dvc-ssh`.
 
 pip (PyPI)
 ----------

debian/changelog

@@ -1,5 +0,0 @@
-dvc (0.75.b1) experimental; urgency=low
-
-  * First native Debian package version.
-
- -- Dmitry Petrov <[email protected]>  Sat, 07 Dec 2019 09:49:28 +0300

debian/compat

@@ -1 +0,0 @@
-10

debian/control

@@ -1,33 +0,0 @@
-Source: dvc
-Maintainer: Dmitry Petrov <[email protected]>
-Section: python
-Priority: optional
-Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)
-Standards-Version: 4.0.0
-Homepage: http://dvc.org
-Vcs-Git: https://github.com/iterative/dvc.git
-Vcs-browser: https://github.com/iterative/dvc
-
-Package: dvc
-Architecture: all
-Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2)
-Recommends: python3-arrow (=0.14.0),
-            python3-paramiko (>=2.5.0),
-            python3-gssapi (>= 2.5.0),
-            python3-boto3 (>=1.9.86),
-            python3-pydrive (=1.3.1),
-            python3-azure-storage-blob (=12.1.0),
-            python3-oss2 (=2.6.1),
-            python3-google-cloud-storage (=1.19.0)
-Description: Git for data scientists - manage your code and data together
- Data Version Control or DVC is an open-source tool for data science and machine learning projects.
- .
- Key features:
- .
- 1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services.
- .
- 2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
- .
- 3. Makes projects reproducible and shareable; helping to answer questions about how a model was built.
- .
- 4. Helps manage experiments with Git tags/branches and metrics tracking.

debian/copyright

@@ -1,11 +0,0 @@
-Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: DVC
-Upstream-Contact: Dmitry Petrov <[email protected]>
-Source: https://github.com/iterative/dvc
-
-Files: *
-Copyright: Dmitry Petrov <[email protected]>
-License: Apache 2.0
- On Debian systems, the full text of the Apache 2.0
- License can be found in the file
- `/usr/share/common-licenses/Apache-2.0`.

debian/rules

@@ -1,6 +0,0 @@
-#!/usr/bin/make -f
-
-export PYBUILD_NAME=dvc
-%:
-	dh $@ --with python3 --buildsystem=pybuild
-

debian/source/format

@@ -1 +0,0 @@
-3.0 (native)

debian/watch

@@ -1,4 +0,0 @@
-# please also check http://pypi.debian.net/dvc/watch
-version=3
-opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
-http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))

dvc/command/experiments.py

@@ -3,7 +3,7 @@ import logging
 from collections import Counter, OrderedDict, defaultdict
 from collections.abc import Mapping
 from datetime import date, datetime
-from itertools import groupby
+from fnmatch import fnmatch
 from typing import Dict, Iterable, Optional
 
 import dvc.prompt as prompt
@@ -27,32 +27,23 @@ def _filter_name(names, label, filter_strs):
 
     for filter_s in filter_strs:
         path, _, name = filter_s.rpartition(":")
-        path_filters[path].append(tuple(name.split(".")))
+        path_filters[path].append(name)
 
     for path, filters in path_filters.items():
         if path:
             match_paths = [path]
         else:
             match_paths = names.keys()
-        for length, groups in groupby(filters, len):
-            for group in groups:
-                for match_path in match_paths:
-                    possible_names = [
-                        tuple(name.split(".")) for name in names[match_path]
-                    ]
-                    matches = [
-                        name
-                        for name in possible_names
-                        if name[:length] == group
-                    ]
-                    if not matches:
-                        name = ".".join(group)
-                        raise InvalidArgumentError(
-                            f"'{name}' does not match any known {label}"
-                        )
-                    ret[match_path].update(
-                        {".".join(match): None for match in matches}
+        for match_path in match_paths:
+            for f in filters:
+                matches = [
+                    name for name in names[match_path] if fnmatch(name, f)
+                ]
+                if not matches:
+                    raise InvalidArgumentError(
+                        f"'{f}' does not match any known {label}"
                     )
+                ret[match_path].update({match: None for match in matches})
 
     return ret
 
@@ -103,7 +94,6 @@ def _collect_names(all_experiments, **kwargs):
         for exp in experiments.values():
             _update_names(metric_names, exp.get("metrics", {}).items())
             _update_names(param_names, exp.get("params", {}).items())
-
     metric_names = _filter_names(
         metric_names,
         "metrics",
@@ -268,7 +258,8 @@ def _extend_row(row, names, items, precision):
     from rich.text import Text
 
     if not items:
-        row.extend(["-"] * len(names))
+        for keys in names.values():
+            row.extend(["-"] * len(keys))
         return
 
     for fname, item in items:
@@ -374,6 +365,7 @@ class CmdExperimentsShow(CmdBase):
                 all_commits=self.args.all_commits,
                 sha_only=self.args.sha,
                 num=self.args.num,
+                param_deps=self.args.param_deps,
             )
 
             if self.args.show_json:
@@ -408,7 +400,9 @@ class CmdExperimentsShow(CmdBase):
                 # NOTE: rich does not have native support for unlimited width
                 # via pager. we override rich table compression by setting
                 # console width to the full width of the table
-                measurement = table.__rich_measure__(console, SHOW_MAX_WIDTH)
+                console_options = console.options
+                console_options.max_width = SHOW_MAX_WIDTH
+                measurement = table.__rich_measure__(console, console_options)
                 console._width = (  # pylint: disable=protected-access
                     measurement.maximum
                 )
@@ -477,6 +471,7 @@ class CmdExperimentsDiff(CmdBase):
                 a_rev=self.args.a_rev,
                 b_rev=self.args.b_rev,
                 all=self.args.all,
+                param_deps=self.args.param_deps,
             )
 
             if self.args.show_json:
@@ -788,6 +783,12 @@ def add_parser(subparsers, parent_parser):
         help="Exclude the specified params from output table.",
         metavar="<params_list>",
     )
+    experiments_show_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     experiments_show_parser.add_argument(
         "--sort-by",
         help="Sort related experiments by the specified metric or param.",
@@ -873,6 +874,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show unchanged metrics/params as well.",
     )
+    experiments_diff_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     experiments_diff_parser.add_argument(
         "--show-json",
         action="store_true",
@@ -1220,6 +1227,7 @@ def _add_run_common(parser):
         "-j",
         "--jobs",
         type=int,
+        default=1,
         help="Run the specified number of experiments at a time in parallel.",
         metavar="<number>",
     )

dvc/command/params.py

@@ -40,6 +40,7 @@ class CmdParamsDiff(CmdBase):
                 b_rev=self.args.b_rev,
                 targets=self.args.targets,
                 all=self.args.all,
+                deps=self.args.deps,
             )
 
             if self.args.show_json:
@@ -111,6 +112,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show unchanged params as well.",
     )
+    params_diff_parser.add_argument(
+        "--deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     params_diff_parser.add_argument(
         "--show-json",
         action="store_true",

dvc/command/plots.py

@@ -1,6 +1,5 @@
 import argparse
 import logging
-import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
@@ -23,6 +22,8 @@ class CmdPlots(CmdBase):
         return {k: v for k, v in props.items() if v is not None}
 
     def run(self):
+        from pathlib import Path
+
         if self.args.show_vega:
             if not self.args.targets:
                 logger.error("please specify a target for `--show-vega`")
@@ -41,20 +42,20 @@ class CmdPlots(CmdBase):
                 logger.info(plots[target])
                 return 0
 
-            path = self.args.out or "plots.html"
-            path = os.path.join(os.getcwd(), path)
-
+            rel: str = self.args.out or "plots.html"
+            path = (Path.cwd() / rel).resolve()
             write(path, plots)
-            url = f"file://{path}"
         except DvcException:
             logger.exception("")
             return 1
 
+        assert path.is_absolute()  # as_uri throws ValueError if not absolute
+        url = path.as_uri()
         logger.info(url)
         if self.args.open:
             import webbrowser
 
-            opened = webbrowser.open(url)
+            opened = webbrowser.open(rel)
             if not opened:
                 logger.error("Failed to open. Please try opening it manually.")
                 return 1

dvc/command/status.py

@@ -44,19 +44,27 @@ class CmdDataStatus(CmdDataBase):
                 self._show(value, indent + 1)
 
     def run(self):
+        from dvc.repo import lock_repo
+        from dvc.ui import ui
+
         indent = 1 if self.args.cloud else 0
-        try:
-            st = self.repo.status(
-                targets=self.args.targets,
-                jobs=self.args.jobs,
-                cloud=self.args.cloud,
-                remote=self.args.remote,
-                all_branches=self.args.all_branches,
-                all_tags=self.args.all_tags,
-                all_commits=self.args.all_commits,
-                with_deps=self.args.with_deps,
-                recursive=self.args.recursive,
-            )
+
+        with lock_repo(self.repo):
+            try:
+                st = self.repo.status(
+                    targets=self.args.targets,
+                    jobs=self.args.jobs,
+                    cloud=self.args.cloud,
+                    remote=self.args.remote,
+                    all_branches=self.args.all_branches,
+                    all_tags=self.args.all_tags,
+                    all_commits=self.args.all_commits,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("")
+                return 1
 
             if self.args.quiet:
                 return bool(st)
@@ -65,19 +73,21 @@ class CmdDataStatus(CmdDataBase):
                 import json
 
                 logger.info(json.dumps(st))
-            elif st:
+                return 0
+
+            if st:
                 self._show(st, indent)
-            elif not self.repo.stages:
-                logger.info(self.EMPTY_PROJECT_MSG)
+                return 0
+
+            # additional hints for the user
+            if not self.repo.stages:
+                ui.write(self.EMPTY_PROJECT_MSG)
             elif self.args.cloud or self.args.remote:
                 remote = self.args.remote or self.repo.config["core"].get(
                     "remote"
                 )
-                logger.info(self.IN_SYNC_MSG.format(remote=remote))
+                ui.write(self.IN_SYNC_MSG.format(remote=remote))
             else:
-                logger.info(self.UP_TO_DATE_MSG)
+                ui.write(self.UP_TO_DATE_MSG)
 
-        except DvcException:
-            logger.exception("")
-            return 1
         return 0

dvc/config_schema.py

@@ -145,6 +145,7 @@ SCHEMA = {
                     "session_token": str,
                     Optional("listobjects", default=False): Bool,  # obsoleted
                     Optional("use_ssl", default=True): Bool,
+                    Optional("ssl_verify", default=True): Bool,
                     "sse": str,
                     "sse_kms_key_id": str,
                     "acl": str,

dvc/dependency/param.py

@@ -39,6 +39,7 @@ class ParamsDependency(LocalDependency):
             path
             or os.path.join(stage.repo.root_dir, self.DEFAULT_PARAMS_FILE),
             info=info,
+            fs=stage.repo.fs,
         )
 
     def dumpd(self):
@@ -84,19 +85,34 @@ class ParamsDependency(LocalDependency):
     def status(self):
         return self.workspace_status()
 
-    def read_params(self):
+    def _read(self):
         if not self.exists:
             return {}
 
         suffix = self.path_info.suffix.lower()
         loader = LOADERS[suffix]
         try:
-            config = loader(self.path_info, fs=self.repo.fs)
+            return loader(self.path_info, fs=self.repo.fs)
         except ParseError as exc:
             raise BadParamFileError(
                 f"Unable to read parameters from '{self}'"
             ) from exc
 
+    def read_params_d(self):
+        config = self._read()
+
+        ret = {}
+        for param in self.params:
+            dpath.util.merge(
+                ret,
+                dpath.util.search(config, param, separator="."),
+                separator=".",
+            )
+        return ret
+
+    def read_params(self):
+        config = self._read()
+
         ret = {}
         for param in self.params:
             try:

dvc/fs/azure.py

@@ -51,6 +51,9 @@ class AzureFileSystem(FSSpecWrapper):
     def _prepare_credentials(self, config):
         from azure.identity.aio import DefaultAzureCredential
 
+        # Disable spam from failed cred types for DefaultAzureCredential
+        logging.getLogger("azure.identity.aio").setLevel(logging.ERROR)
+
         login_info = {}
         login_info["connection_string"] = config.get(
             "connection_string",
@@ -69,8 +72,12 @@ class AzureFileSystem(FSSpecWrapper):
         login_info["client_id"] = config.get("client_id")
         login_info["client_secret"] = config.get("client_secret")
 
-        if not any(login_info.values()):
-            login_info["credentials"] = DefaultAzureCredential()
+        if not any(
+            value for key, value in login_info.items() if key != "account_name"
+        ):
+            login_info["credential"] = DefaultAzureCredential(
+                exclude_interactive_browser_credential=False
+            )
 
         for login_method, required_keys in [  # noqa
             ("connection string", ["connection_string"]),
@@ -80,8 +87,11 @@ class AzureFileSystem(FSSpecWrapper):
             ),
             ("account key", ["account_name", "account_key"]),
             ("SAS token", ["account_name", "sas_token"]),
+            (
+                f"default credentials ({_DEFAULT_CREDS_STEPS})",
+                ["account_name", "credential"],
+            ),
             ("anonymous login", ["account_name"]),
-            (f"default credentials ({_DEFAULT_CREDS_STEPS})", ["credentials"]),
         ]:
             if all(login_info.get(key) is not None for key in required_keys):
                 break

dvc/fs/base.py

@@ -1,6 +1,6 @@
 import logging
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from functools import partial
+from functools import partial, partialmethod
 from multiprocessing import cpu_count
 from typing import Any, ClassVar, Dict, FrozenSet, Optional
 
@@ -38,7 +38,7 @@ class BaseFileSystem:
     scheme = "base"
     REQUIRES: ClassVar[Dict[str, str]] = {}
     PATH_CLS = URLInfo  # type: Any
-    JOBS = 4 * cpu_count()
+    _JOBS = 4 * cpu_count()
 
     CHECKSUM_DIR_SUFFIX = ".dir"
     HASH_JOBS = max(1, min(4, cpu_count() // 2))
@@ -67,7 +67,7 @@ class BaseFileSystem:
         return (
             self.config.get("jobs")
             or (self.repo and self.repo.config["core"].get("jobs"))
-            or self.JOBS
+            or self._JOBS
         )
 
     @cached_property
@@ -258,6 +258,7 @@ class BaseFileSystem:
         name=None,
         no_progress_bar=False,
         jobs=None,
+        _only_file=False,
         **kwargs,
     ):
         if not hasattr(self, "_download"):
@@ -273,12 +274,14 @@ class BaseFileSystem:
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        if self.isdir(from_info):
+        if not _only_file and self.isdir(from_info):
             return self._download_dir(
                 from_info, to_info, name, no_progress_bar, jobs, **kwargs,
             )
         return self._download_file(from_info, to_info, name, no_progress_bar,)
 
+    download_file = partialmethod(download, _only_file=True)
+
     def _download_dir(
         self, from_info, to_info, name, no_progress_bar, jobs, **kwargs,
     ):

dvc/fs/s3.py

@@ -38,6 +38,7 @@ class S3FileSystem(BaseFileSystem):
         self.endpoint_url = config.get("endpointurl")
 
         self.use_ssl = config.get("use_ssl", True)
+        self.ssl_verify = config.get("ssl_verify", True)
 
         self.extra_args = {}
 
@@ -136,6 +137,7 @@ class S3FileSystem(BaseFileSystem):
         return session.resource(
             "s3",
             endpoint_url=self.endpoint_url,
+            verify=self.ssl_verify,
             use_ssl=self.use_ssl,
             config=boto3.session.Config(
                 signature_version="s3v4", s3=s3_config

dvc/fs/ssh/__init__.py

@@ -33,7 +33,7 @@ def ask_password(host, user, port):
 class SSHFileSystem(BaseFileSystem):
     scheme = Schemes.SSH
     REQUIRES = {"paramiko": "paramiko"}
-    JOBS = 4
+    _JOBS = 4
 
     PARAM_CHECKSUM = "md5"
     # At any given time some of the connections will go over network and

dvc/fs/ssh/connection.py

@@ -17,7 +17,7 @@ from ..base import RemoteCmdError
 try:
     import paramiko
 except ImportError:
-    paramiko = None
+    paramiko = None  # type: ignore
 
 
 logger = logging.getLogger(__name__)

dvc/fs/webdav.py

@@ -117,7 +117,7 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             )
 
         # Check whether connection is valid (root should always exist)
-        if not client.check("/"):
+        if not client.check(self.path_info.path):
             raise WebDAVConnectionError(self.hostname)
 
         return client
@@ -213,7 +213,7 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     # Creates directories
     def makedirs(self, path_info):
         # Terminate recursion
-        if path_info.path == "/" or self.exists(path_info):
+        if path_info.path == self.path_info.path or self.exists(path_info):
             return
 
         # Recursively descent to root

dvc/ignore.py

@@ -183,7 +183,12 @@ class DvcIgnoreFilter:
     def __init__(self, fs, root_dir):
         from dvc.repo import Repo
 
-        default_ignore_patterns = [".hg/", ".git/", "{}/".format(Repo.DVC_DIR)]
+        default_ignore_patterns = [
+            ".hg/",
+            ".git/",
+            ".git",
+            "{}/".format(Repo.DVC_DIR),
+        ]
 
         self.fs = fs
         self.root_dir = root_dir

dvc/main.py

@@ -45,6 +45,11 @@ def main(argv=None):  # noqa: C901
 
         logger.trace(args)
 
+        if not args.quiet:
+            from dvc.ui import ui
+
+            ui.enable()
+
         with debugtools(args):
             cmd = args.func(args)
             ret = cmd.run()

dvc/objects/db/base.py

@@ -229,7 +229,7 @@ class ObjectDB:
         keeping all of it in memory.
         """
         num_pages = remote_size / self.fs.LIST_OBJECT_PAGE_SIZE
-        if num_pages < 256 / self.fs.JOBS:
+        if num_pages < 256 / self.fs.jobs:
             # Fetching prefixes in parallel requires at least 255 more
             # requests, for small enough remotes it will be faster to fetch
             # entire cache without splitting it into prefixes.
@@ -263,7 +263,7 @@ class ObjectDB:
                 )
 
             with ThreadPoolExecutor(
-                max_workers=jobs or self.fs.JOBS
+                max_workers=jobs or self.fs.jobs
             ) as executor:
                 in_remote = executor.map(list_with_update, traverse_prefixes,)
                 yield from itertools.chain.from_iterable(in_remote)
@@ -331,7 +331,7 @@ class ObjectDB:
                 return ret
 
             with ThreadPoolExecutor(
-                max_workers=jobs or self.fs.JOBS
+                max_workers=jobs or self.fs.jobs
             ) as executor:
                 path_infos = map(self.hash_to_path_info, hashes)
                 in_remote = executor.map(exists_with_progress, path_infos)

dvc/objects/db/ssh.py

@@ -64,10 +64,10 @@ class SSHObjectDB(ObjectDB):
                 return self.batch_exists(chunks, callback=pbar.update_msg)
 
             with ThreadPoolExecutor(
-                max_workers=jobs or self.fs.JOBS
+                max_workers=jobs or self.fs.jobs
             ) as executor:
                 path_infos = [self.hash_to_path_info(x) for x in hashes]
-                chunks = to_chunks(path_infos, num_chunks=self.fs.JOBS)
+                chunks = to_chunks(path_infos, num_chunks=self.fs.jobs)
                 results = executor.map(exists_with_progress, chunks)
                 in_remote = itertools.chain.from_iterable(results)
                 ret = list(itertools.compress(hashes, in_remote))

dvc/parsing/context.py

@@ -401,9 +401,9 @@ class Context(CtxDict):
     ):
         path, _, keys_str = item.partition(":")
         select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
-        path_info = wdir / path
 
-        abspath = os.path.abspath(path_info)
+        abspath = os.path.abspath(wdir / path)
+        path_info = PathInfo(abspath)
         if abspath in self.imports:
             if not select_keys and self.imports[abspath] is None:
                 return  # allow specifying complete filepath multiple times

dvc/remote/base.py

@@ -309,7 +309,7 @@ class Remote:
         )
 
         if download:
-            func = _log_exceptions(self.fs.download, "download")
+            func = _log_exceptions(self.fs.download_file, "download")
             status = STATUS_DELETED
             desc = "Downloading"
         else:
@@ -318,7 +318,7 @@ class Remote:
             desc = "Uploading"
 
         if jobs is None:
-            jobs = self.fs.JOBS
+            jobs = self.fs.jobs
 
         dir_status, file_status, dir_contents = self._status(
             cache,

dvc/repo/__init__.py

@@ -141,6 +141,7 @@ class Repo:
         from dvc.repo.params import Params
         from dvc.repo.plots import Plots
         from dvc.repo.stage import StageLoad
+        from dvc.scm import SCM
         from dvc.stage.cache import StageCache
         from dvc.state import State, StateNoop
 
@@ -149,6 +150,9 @@ class Repo:
             "repo_factory": repo_factory,
         }
 
+        if rev and not scm:
+            scm = SCM(root_dir or os.curdir)
+
         self.root_dir, self.dvc_dir, self.tmp_dir = self._get_repo_dirs(
             root_dir=root_dir, scm=scm, rev=rev, uninitialized=uninitialized
         )

dvc/repo/experiments/__init__.py

@@ -324,7 +324,6 @@ class Experiments:
 
         if reset:
             self.reset_checkpoints()
-            kwargs["force"] = True
 
         if not (queue or tmp_dir):
             staged, _, _ = self.scm.status()

dvc/repo/experiments/diff.py

@@ -6,7 +6,7 @@ from dvc.utils.diff import format_dict
 logger = logging.getLogger(__name__)
 
 
-def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
+def diff(repo, *args, a_rev=None, b_rev=None, param_deps=False, **kwargs):
     from dvc.repo.experiments.show import _collect_experiment_commit
 
     if repo.scm.no_commits:
@@ -14,15 +14,17 @@ def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
 
     if a_rev:
         rev = repo.scm.resolve_rev(a_rev)
-        old = _collect_experiment_commit(repo, rev)
+        old = _collect_experiment_commit(repo, rev, param_deps=param_deps)
     else:
-        old = _collect_experiment_commit(repo, "HEAD")
+        old = _collect_experiment_commit(repo, "HEAD", param_deps=param_deps)
 
     if b_rev:
         rev = repo.scm.resolve_rev(b_rev)
-        new = _collect_experiment_commit(repo, rev)
+        new = _collect_experiment_commit(repo, rev, param_deps=param_deps)
     else:
-        new = _collect_experiment_commit(repo, "workspace")
+        new = _collect_experiment_commit(
+            repo, "workspace", param_deps=param_deps
+        )
 
     with_unchanged = kwargs.pop("all", False)
 

dvc/repo/experiments/executor/base.py

@@ -271,7 +271,7 @@ class BaseExecutor(ABC):
             # - if run without --reset, the checkpoint out will be checked out
             #   using any hash present in dvc.lock (or removed if no entry
             #   exists in dvc.lock)
-            checkpoint_reset = kwargs.pop("reset", False)
+            checkpoint_reset: bool = kwargs.pop("reset", False)
             dvc_checkout(
                 dvc,
                 targets=targets,
@@ -283,7 +283,10 @@ class BaseExecutor(ABC):
             )
 
             checkpoint_func = partial(
-                cls.checkpoint_callback, dvc.scm, name, repro_force
+                cls.checkpoint_callback,
+                dvc.scm,
+                name,
+                repro_force or checkpoint_reset,
             )
             stages = dvc_reproduce(
                 dvc,
@@ -295,12 +298,18 @@ class BaseExecutor(ABC):
 
             exp_hash = cls.hash_exp(stages)
             try:
+                is_checkpoint = any(stage.is_checkpoint for stage in stages)
+                if is_checkpoint and checkpoint_reset:
+                    # For reset checkpoint stages, we need to force overwriting
+                    # existing checkpoint refs even though repro may not have
+                    # actually been run with --force
+                    repro_force = True
                 cls.commit(
                     dvc.scm,
                     exp_hash,
                     exp_name=name,
                     force=repro_force,
-                    checkpoint=any(stage.is_checkpoint for stage in stages),
+                    checkpoint=is_checkpoint,
                 )
             except UnchangedExperimentError:
                 pass

dvc/repo/experiments/show.py

@@ -12,17 +12,23 @@ from dvc.scm.base import SCMError
 logger = logging.getLogger(__name__)
 
 
-def _collect_experiment_commit(repo, rev, stash=False, sha_only=True):
+def _collect_experiment_commit(
+    repo, exp_rev, stash=False, sha_only=True, param_deps=False
+):
     res = defaultdict(dict)
-    for rev in repo.brancher(revs=[rev]):
+    for rev in repo.brancher(revs=[exp_rev]):
         if rev == "workspace":
+            if exp_rev != "workspace":
+                continue
             res["timestamp"] = None
         else:
             commit = repo.scm.resolve_commit(rev)
             res["timestamp"] = datetime.fromtimestamp(commit.commit_time)
 
-        configs = _collect_configs(repo, rev=rev)
-        params = _read_params(repo, configs, rev)
+        params, params_path_infos = _collect_configs(repo, rev=rev)
+        params = _read_params(
+            repo, params, params_path_infos, rev, deps=param_deps
+        )
         if params:
             res["params"] = params
 
@@ -83,6 +89,7 @@ def show(
     all_commits=False,
     sha_only=False,
     num=1,
+    param_deps=False,
 ):
     res = defaultdict(OrderedDict)
 
@@ -110,7 +117,7 @@ def show(
 
     for rev in revs:
         res[rev]["baseline"] = _collect_experiment_commit(
-            repo, rev, sha_only=sha_only
+            repo, rev, sha_only=sha_only, param_deps=param_deps
         )
 
         if rev == "workspace":
@@ -127,14 +134,19 @@ def show(
             ref_info = ExpRefInfo.from_ref(exp_ref)
             assert ref_info.baseline_sha == rev
             _collect_experiment_branch(
-                res[rev], repo, exp_ref, rev, sha_only=sha_only
+                res[rev],
+                repo,
+                exp_ref,
+                rev,
+                sha_only=sha_only,
+                param_deps=param_deps,
             )
 
     # collect queued (not yet reproduced) experiments
     for stash_rev, entry in repo.experiments.stash_revs.items():
         if entry.baseline_rev in revs:
             experiment = _collect_experiment_commit(
-                repo, stash_rev, stash=True
+                repo, stash_rev, stash=True, param_deps=param_deps
             )
             res[entry.baseline_rev][stash_rev] = experiment
 

dvc/repo/params/show.py

@@ -1,6 +1,7 @@
 import logging
 from collections import defaultdict
-from typing import TYPE_CHECKING, List
+from copy import copy
+from typing import TYPE_CHECKING, List, Tuple
 
 from dvc.dependency.param import ParamsDependency
 from dvc.exceptions import DvcException
@@ -27,7 +28,9 @@ def _is_params(dep: "BaseOutput"):
     return isinstance(dep, ParamsDependency)
 
 
-def _collect_configs(repo: "Repo", rev, targets=None) -> List["DvcPath"]:
+def _collect_configs(
+    repo: "Repo", rev, targets=None
+) -> Tuple[List["BaseOutput"], List["DvcPath"]]:
     params, path_infos = collect(
         repo,
         targets=targets or [],
@@ -35,31 +38,45 @@ def _collect_configs(repo: "Repo", rev, targets=None) -> List["DvcPath"]:
         output_filter=_is_params,
         rev=rev,
     )
-    path_infos.extend([p.path_info for p in params])
+    all_path_infos = path_infos + [p.path_info for p in params]
     if not targets:
         default_params = (
             PathInfo(repo.root_dir) / ParamsDependency.DEFAULT_PARAMS_FILE
         )
-        if default_params not in path_infos:
+        if default_params not in all_path_infos:
             path_infos.append(default_params)
-    return list(path_infos)
+    return params, path_infos
 
 
-def _read_params(repo, configs, rev):
-    res = {}
-    for config in configs:
-        if not repo.fs.exists(config):
-            continue
+def _read_path_info(fs, path_info, rev):
+    if not fs.exists(path_info):
+        return None
 
-        suffix = config.suffix.lower()
-        loader = LOADERS[suffix]
-        try:
-            res[str(config)] = loader(config, fs=repo.fs)
-        except ParseError:
-            logger.debug(
-                "failed to read '%s' on '%s'", config, rev, exc_info=True
-            )
-            continue
+    suffix = path_info.suffix.lower()
+    loader = LOADERS[suffix]
+    try:
+        return loader(path_info, fs=fs)
+    except ParseError:
+        logger.debug(
+            "failed to read '%s' on '%s'", path_info, rev, exc_info=True
+        )
+        return None
+
+
+def _read_params(repo, params, params_path_infos, rev, deps=False):
+    res = defaultdict(dict)
+    path_infos = copy(params_path_infos)
+
+    if deps:
+        for param in params:
+            res[str(param.path_info)].update(param.read_params_d())
+    else:
+        path_infos += [param.path_info for param in params]
+
+    for path_info in path_infos:
+        from_path = _read_path_info(repo.fs, path_info, rev)
+        if from_path:
+            res[str(path_info)] = from_path
 
     return res
 
@@ -79,12 +96,12 @@ def _collect_vars(repo, params):
 
 
 @locked
-def show(repo, revs=None, targets=None):
+def show(repo, revs=None, targets=None, deps=False):
     res = {}
 
     for branch in repo.brancher(revs=revs):
-        configs = _collect_configs(repo, branch, targets)
-        params = _read_params(repo, configs, branch)
+        params, params_path_infos = _collect_configs(repo, branch, targets)
+        params = _read_params(repo, params, params_path_infos, branch, deps)
         vars_params = _collect_vars(repo, params)
 
         # NOTE: only those that are not added as a ParamDependency are included

dvc/scm/git/backend/dulwich.py

@@ -182,11 +182,16 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
     def commit(self, msg: str, no_verify: bool = False):
         from dulwich.errors import CommitError
         from dulwich.porcelain import commit
+        from dulwich.repo import InvalidUserIdentity
 
         try:
             commit(self.root_dir, message=msg, no_verify=no_verify)
         except CommitError as exc:
             raise SCMError("Git commit failed") from exc
+        except InvalidUserIdentity as exc:
+            raise SCMError(
+                "Git username and email must be configured"
+            ) from exc
 
     def checkout(
         self,
@@ -528,6 +533,8 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
         message: Optional[str] = None,
         include_untracked: Optional[bool] = False,
     ) -> Tuple[Optional[str], bool]:
+        from dulwich.repo import InvalidUserIdentity
+
         from dvc.scm.git import Stash
 
         if include_untracked or ref == Stash.DEFAULT_STASH:
@@ -537,7 +544,12 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
 
         stash = self._get_stash(ref)
         message_b = message.encode("utf-8") if message else None
-        rev = stash.push(message=message_b)
+        try:
+            rev = stash.push(message=message_b)
+        except InvalidUserIdentity as exc:
+            raise SCMError(
+                "Git username and email must be configured"
+            ) from exc
         return os.fsdecode(rev), True
 
     def _stash_apply(self, rev: str):

dvc/scm/git/backend/pygit2.py

@@ -80,6 +80,15 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
 
         return RefdbFsBackend(self.repo)
 
+    @property
+    def default_signature(self):
+        try:
+            return self.repo.default_signature
+        except KeyError as exc:
+            raise SCMError(
+                "Git username and email must be configured"
+            ) from exc
+
     @staticmethod
     def clone(
         url: str,
@@ -328,7 +337,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         from dvc.scm.git import Stash
 
         oid = self.repo.stash(
-            self.repo.default_signature,
+            self.default_signature,
             message=message,
             include_untracked=include_untracked,
         )
@@ -503,7 +512,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
             raise MergeConflictError("Merge contained conflicts")
 
         if commit:
-            user = self.repo.default_signature
+            user = self.default_signature
             tree = self.repo.index.write_tree()
             merge_commit = self.repo.create_commit(
                 "HEAD", user, user, msg, tree, [self.repo.head.target, rev]

dvc/types.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 from dvc.path_info import PathInfo, URLInfo
 
@@ -11,3 +11,5 @@ AnyPath = Union[str, DvcPath, StrPath]
 
 OptStr = Optional[str]
 TargetType = Union[List[str], str]
+DictStrAny = Dict[str, Any]
+DictAny = Dict[Any, Any]

dvc/ui/__init__.py

@@ -1,14 +1,21 @@
 import sys
 from collections import defaultdict
-from typing import Any, Dict, Iterable, Optional, TextIO
-
-from funcy import cached_property
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterable,
+    Optional,
+    Sequence,
+    TextIO,
+    Union,
+)
 
 from dvc.progress import Tqdm
 from dvc.utils import colorize
 
-NEWLINE = "\n"
-SEP = " "
+if TYPE_CHECKING:
+    from dvc.ui.table import Headers, Styles, TableData
 
 
 class Formatter:
@@ -30,14 +37,23 @@ class Console:
         formatter: Formatter = None,
         output: TextIO = None,
         error: TextIO = None,
-        disable: bool = False,
+        enable: bool = False,
     ) -> None:
-        self._input: TextIO = sys.stdin
-        self._output: TextIO = output or sys.stdout
-        self._error: TextIO = error or sys.stderr
-
+        self._output: Optional[TextIO] = output
+        self._error: Optional[TextIO] = error
         self.formatter: Formatter = formatter or Formatter()
-        self.disabled: bool = disable
+        self._enabled: bool = enable
+
+    @property
+    def output(self) -> TextIO:
+        return self._output or sys.stdout
+
+    @property
+    def error_output(self) -> TextIO:
+        return self._error or sys.stderr
+
+    def enable(self) -> None:
+        self._enabled = True
 
     def success(self, message: str) -> None:
         self.write(message, style="success")
@@ -52,8 +68,8 @@ class Console:
         self,
         *objects: Any,
         style: str = None,
-        sep: str = SEP,
-        end: str = NEWLINE,
+        sep: str = None,
+        end: str = None,
         flush: bool = False,
     ) -> None:
         return self.write(
@@ -61,7 +77,7 @@ class Console:
             style=style,
             sep=sep,
             end=end,
-            file=self._error,
+            file=self.error_output,
             flush=flush,
         )
 
@@ -69,20 +85,21 @@ class Console:
         self,
         *objects: Any,
         style: str = None,
-        sep: str = SEP,
-        end: str = NEWLINE,
+        sep: str = None,
+        end: str = None,
         file: TextIO = None,
         flush: bool = False,
+        force: bool = False,
     ) -> None:
-        if self.disabled:
+        if not self._enabled and not force:
             return
 
-        file = file or self._output
+        file = file or self.output
         values = (self.formatter.format(obj, style=style) for obj in objects)
         return print(*values, sep=sep, end=end, file=file, flush=flush)
 
     def progress(self, *args, **kwargs) -> Tqdm:
-        kwargs.setdefault("file", self._error)
+        kwargs.setdefault("file", self.error_output)
         return Tqdm(*args, **kwargs)
 
     def prompt(
@@ -120,47 +137,64 @@ class Console:
             return False
         return answer.startswith("y")
 
-    @cached_property
+    @property
     def rich_console(self):
         """rich_console is only set to stdout for now."""
         from rich import console
 
-        return console.Console(file=self._output)
+        # FIXME: Getting IO Operation on closed file error
+        #  when testing with capsys, therefore we are creating
+        #  one instance each time as a temporary workaround.
+        return console.Console(file=self.output)
 
-    def rich_table(self, pager: bool = True):
-        pass
+    def table(
+        self,
+        data: "TableData",
+        headers: "Headers" = None,
+        markdown: bool = False,
+        rich_table: bool = False,
+        force: bool = True,
+        pager: bool = False,
+        header_styles: Sequence["Styles"] = None,
+        row_styles: Sequence["Styles"] = None,
+        borders: Union[bool, str] = False,
+    ) -> None:
+        from dvc.ui import table as t
 
-    def table(self, header, rows, markdown: bool = False):
-        from tabulate import tabulate
+        if not data and not markdown:
+            return
 
-        if not rows and not markdown:
-            return ""
+        if not markdown and rich_table:
+            if force or self._enabled:
+                return t.rich_table(
+                    self,
+                    data,
+                    headers,
+                    pager=pager,
+                    header_styles=header_styles,
+                    row_styles=row_styles,
+                    borders=borders,
+                )
 
-        ret = tabulate(
-            rows,
-            header,
-            tablefmt="github" if markdown else "plain",
-            disable_numparse=True,
-            # None will be shown as "" by default, overriding
-            missingval="—",
+            return
+
+        return t.plain_table(
+            self, data, headers, markdown=markdown, pager=pager, force=force,
         )
 
-        if markdown:
-            # NOTE: md table is incomplete without the trailing newline
-            ret += "\n"
 
-        self.write(ret)
+ui = Console()
 
 
 if __name__ == "__main__":
-    ui = Console()
+    ui.enable()
 
     ui.write("No default remote set")
     ui.success("Everything is up to date.")
     ui.warn("Run queued experiments will be removed.")
     ui.error("too few arguments.")
 
-    ui.table("keys", {"Path": ["scores.json"], "auc": ["0.5674"]})
+    ui.table([("scores.json", "0.5674")], headers=["Path", "auc"])
     ui.table(
-        "keys", {"Path": ["scores.json"], "auc": ["0.5674"]}, markdown=True
+        [("scores.json", "0.5674")], headers=["Path", "auc"], markdown=True
     )

dvc/ui/table.py

@@ -0,0 +1,119 @@
+from contextlib import contextmanager
+from itertools import zip_longest
+from typing import TYPE_CHECKING, Iterator, List, Sequence, Union
+
+from dvc.types import DictStrAny
+
+if TYPE_CHECKING:
+    from rich.console import Console as RichConsole
+    from rich.table import Table
+    from rich.text import Text
+
+    from dvc.ui import Console
+
+
+SHOW_MAX_WIDTH = 1024
+
+
+CellT = Union[str, "Text"]  # Text is mostly compatible with str
+Row = Sequence[CellT]
+TableData = List[Row]
+Headers = Sequence[str]
+Styles = DictStrAny
+
+
+def plain_table(
+    ui: "Console",
+    data: TableData,
+    headers: Headers = None,
+    markdown: bool = False,
+    pager: bool = False,
+    force: bool = True,
+) -> None:
+    from tabulate import tabulate
+
+    text: str = tabulate(
+        data,
+        headers if headers is not None else (),
+        tablefmt="github" if markdown else "plain",
+        disable_numparse=True,
+        # None will be shown as "" by default, overriding
+        missingval="-",
+    )
+    if markdown:
+        # NOTE: md table is incomplete without the trailing newline
+        text += "\n"
+
+    if pager:
+        from dvc.utils.pager import pager as _pager
+
+        _pager(text)
+    else:
+        ui.write(text, force=force)
+
+
+@contextmanager
+def console_width(
+    table: "Table", console: "RichConsole", val: int
+) -> Iterator[None]:
+    # NOTE: rich does not have native support for unlimited width
+    # via pager. we override rich table compression by setting
+    # console width to the full width of the table
+    # pylint: disable=protected-access
+
+    console_options = console.options
+    original = console_options.max_width
+    con_width = console._width
+
+    try:
+        console_options.max_width = val
+        measurement = table.__rich_measure__(console, console_options)
+        console._width = measurement.maximum
+
+        yield
+    finally:
+        console_options.max_width = original
+        console._width = con_width
+
+
+def rich_table(
+    ui: "Console",
+    data: TableData,
+    headers: Headers = None,
+    pager: bool = False,
+    header_styles: Sequence[Styles] = None,
+    row_styles: Sequence[Styles] = None,
+    borders: Union[bool, str] = False,
+) -> None:
+    from rich import box
+
+    from dvc.utils.table import Table
+
+    border_style = {
+        True: box.HEAVY_HEAD,  # is a default in rich,
+        False: None,
+        "simple": box.SIMPLE,
+        "minimal": box.MINIMAL,
+    }
+
+    table = Table(box=border_style[borders])
+    hs: Sequence[Styles] = header_styles or []
+    rs: Sequence[Styles] = row_styles or []
+
+    for header, style in zip_longest(headers or [], hs):
+        table.add_column(header, **(style or {}))
+
+    for row, style in zip_longest(data, rs):
+        table.add_row(*row, **(style or {}))
+
+    console = ui.rich_console
+
+    if not pager:
+        console.print(table)
+        return
+
+    from dvc.utils.pager import DvcPager
+
+    with console_width(table, console, SHOW_MAX_WIDTH):
+        with console.pager(pager=DvcPager(), styles=True):
+            console.print(table)

dvc/utils/pager.py

@@ -46,7 +46,7 @@ def find_pager():
     return pydoc.plainpager
 
 
-def pager(text):
+def pager(text: str) -> None:
     find_pager()(text)
 
 

dvc/utils/table.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING, List, cast
 
 from rich.style import StyleType
 from rich.table import Column as RichColumn
@@ -8,6 +8,7 @@ from rich.table import Table as RichTable
 if TYPE_CHECKING:
     from rich.console import (
         Console,
+        ConsoleOptions,
         JustifyMethod,
         OverflowMethod,
         RenderableType,
@@ -56,7 +57,7 @@ class Table(RichTable):
         self.columns.append(column)
 
     def _calculate_column_widths(
-        self, console: "Console", max_width: int
+        self, console: "Console", options: "ConsoleOptions"
     ) -> List[int]:
         """Calculate the widths of each column, including padding, not
         including borders.
@@ -64,22 +65,23 @@ class Table(RichTable):
         Adjacent collapsed columns will be removed until there is only a single
         truncated column remaining.
         """
-        widths = super()._calculate_column_widths(console, max_width)
+        widths = super()._calculate_column_widths(console, options)
         last_collapsed = -1
-        for i in range(len(self.columns) - 1, -1, -1):
-            if widths[i] == 1 and self.columns[i].collapse:
+        columns = cast(List[Column], self.columns)
+        for i in range(len(columns) - 1, -1, -1):
+            if widths[i] == 0 and columns[i].collapse:
                 if last_collapsed >= 0:
                     del widths[last_collapsed]
-                    del self.columns[last_collapsed]
+                    del columns[last_collapsed]
                     if self.box:
-                        max_width += 1
-                    for column in self.columns[last_collapsed:]:
+                        options.max_width += 1
+                    for column in columns[last_collapsed:]:
                         column._index -= 1
                 last_collapsed = i
                 padding = self._get_padding_width(i)
                 if (
-                    self.columns[i].overflow == "ellipsis"
-                    and (sum(widths) + padding) <= max_width
+                    columns[i].overflow == "ellipsis"
+                    and (sum(widths) + padding) <= options.max_width
                 ):
                     # Set content width to 1 (plus padding) if we can fit a
                     # single unicode ellipsis in this column
@@ -88,7 +90,7 @@ class Table(RichTable):
                 last_collapsed = -1
         return widths
 
-    def _collapse_widths(
+    def _collapse_widths(  # type: ignore[override]
         self, widths: List[int], wrapable: List[bool], max_width: int,
     ) -> List[int]:
         """Collapse columns right-to-left if possible to fit table into
@@ -97,7 +99,8 @@ class Table(RichTable):
         If table is still too wide after collapsing, rich's automatic overflow
         handling will be used.
         """
-        collapsible = [column.collapse for column in self.columns]
+        columns = cast(List[Column], self.columns)
+        collapsible = [column.collapse for column in columns]
         total_width = sum(widths)
         excess_width = total_width - max_width
         if any(collapsible):

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.0.7"
+_BASE_VERSION = "2.0.18"
 
 
 def _generate_version(base_version):

scripts/build-requirements.txt

@@ -1,2 +1,2 @@
-PyInstaller==3.5
+PyInstaller==4.2
 numpy==1.19.3

scripts/build.py

@@ -0,0 +1,103 @@
+import argparse
+import importlib
+import pathlib
+import textwrap
+from subprocess import STDOUT, check_call
+
+path = pathlib.Path(__file__).parent.absolute()
+dvc = path.parent / "dvc"
+pyinstaller = path / "pyinstaller"
+innosetup = path / "innosetup"
+fpm = path / "fpm"
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "pkg", choices=["deb", "rpm", "osxpkg", "exe"], help="package type"
+)
+parser.add_argument("--sign-application", default=False, action="store_true")
+parser.add_argument("--application-id")
+parser.add_argument("--sign-installer", default=False, action="store_true")
+parser.add_argument("--installer-id")
+parser.add_argument("--notarize", default=False, action="store_true")
+parser.add_argument("--apple-id-username")
+parser.add_argument("--apple-id-password")
+args = parser.parse_args()
+
+(dvc / "utils" / "build.py").write_text(f'PKG = "{args.pkg}"')
+
+# Autogenerate version, similar to what we do in setup.py
+spec = importlib.util.spec_from_file_location(
+    "dvc.version", dvc / "version.py"
+)
+dvc_version = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(dvc_version)
+version = dvc_version.__version__
+
+(dvc / "version.py").write_text(
+    textwrap.dedent(
+        f"""\
+        # AUTOGENERATED at build time by scripts/build.py
+        __version__ = "{version}"
+    """
+    )
+)
+
+check_call(
+    ["python", "build.py"], cwd=pyinstaller, stderr=STDOUT,
+)
+
+if args.sign_application:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not args.application_id:
+        print("--sign-application requires --application-id")
+        exit(1)
+    check_call(
+        ["python", "sign.py", "--application-id", args.application_id],
+        cwd=pyinstaller,
+        stderr=STDOUT,
+    )
+
+if args.pkg == "exe":
+    check_call(
+        ["python", "build.py"], cwd=innosetup, stderr=STDOUT,
+    )
+else:
+    check_call(
+        ["python", "build.py", args.pkg], cwd=fpm, stderr=STDOUT,
+    )
+
+if args.sign_installer:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not all(
+        [args.installer_id, args.apple_id_username, args.apple_id_password]
+    ):
+        print("--sign-installer requires --installer-id")
+        exit(1)
+    check_call(
+        ["python", "sign.py", "--installer-id", args.installer_id],
+        cwd=fpm,
+        stderr=STDOUT,
+    )
+
+if args.notarize:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not all([args.apple_id_username, args.apple_id_password]):
+        print(
+            "--notarize requires --apple-id-username and --apple-id-password"
+        )
+        exit(1)
+    check_call(
+        [
+            "python",
+            "notarize.py",
+            "--apple-id-username",
+            args.apple_id_username,
+            "--apple-id-password",
+            args.apple_id_password,
+        ],
+        cwd=fpm,
+        stderr=STDOUT,
+    )

scripts/build_posix.sh

@@ -1,139 +0,0 @@
-#!/bin/bash
-
-set -e
-set -x
-
-FPM_FLAGS='--description "Data Version Control - datasets, models, and experiments versioning for ML or data science projects"'
-FPM_FLAGS+=' -n dvc -s dir -f'
-FPM_FLAGS+=' --url https://dvc.org'
-FPM_FLAGS+=' --license "Apache License 2.0"'
-
-if [[ "$(uname)" == 'Linux' ]]; then
-  INSTALL_DIR=usr
-  FPM_FLAGS+=' --depends "git >= 1.7.0"'  # needed for gitpython
-else
-  INSTALL_DIR=usr/local
-  FPM_FLAGS+=' --osxpkg-identifier-prefix com.iterative'
-  FPM_FLAGS+=' --after-install scripts/fpm/after-install.sh'
-  FPM_FLAGS+=' --after-remove scripts/fpm/after-remove.sh'
-fi
-
-BUILD_DIR=build
-BIN_DIR=$BUILD_DIR/$INSTALL_DIR/bin
-LIB_DIR=$BUILD_DIR/$INSTALL_DIR/lib
-
-FPM_PACKAGE_DIRS="usr"
-ZSH_CMPLT_DIR=usr/share/zsh/site-functions/_dvc
-if [[ "$(uname)" == 'Linux' ]]; then
-  BASH_CMPLT_DIR=etc/bash_completion.d
-  FPM_PACKAGE_DIRS="$FPM_PACKAGE_DIRS etc"
-else
-  BASH_CMPLT_DIR=usr/local/etc/bash_completion.d
-fi
-
-print_error() {
-  echo -e "\e[31m$1\e[0m" >&2
-}
-
-if [ ! -d "dvc" ]; then
-  print_error "Please run this script from repository root"
-  exit 1
-fi
-
-trap 'print_error "FAIL"; exit 1' ERR
-
-print_info() {
-  echo -e "\e[32m$1\e[0m"
-}
-
-command_exists() {
-  command -v $1 >/dev/null 2>&1
-}
-
-fpm_build() {
-  print_info "Building $1..."
-  VERSION=$(python -c "import dvc; from dvc import __version__; print(str(__version__))")
-  eval "fpm -t $1 $FPM_FLAGS -v $VERSION -C $BUILD_DIR $FPM_PACKAGE_DIRS"
-}
-
-cleanup() {
-  print_info "Cleaning up..."
-  rm -rf build
-}
-
-install_dependencies() {
-  print_info "Installing fpm..."
-  if command_exists dnf; then
-    sudo dnf install ruby-devel gcc make rpm-build
-  elif command_exists yum; then
-    sudo yum install ruby-devel gcc make rpm-build
-  elif command_exists apt-get; then
-    sudo apt-get update -y
-    sudo apt-get install ruby-dev build-essential rpm python-pip python-dev
-  elif command_exists brew; then
-    brew install ruby
-  else
-    echo "Unable to install fpm dependencies" && exit 1
-  fi
-
-  gem install --no-document fpm
-
-  # https://github.com/iterative/dvc/issues/2970
-  gem uninstall -i /Users/travis/.rvm/gems/ruby-2.4.3@global rubygems-bundler
-
-  print_info "Installing requirements..."
-  pip install .[all]
-  pip install -r scripts/build-requirements.txt
-}
-
-build_dvc() {
-  print_info "Building dvc binary..."
-  pyinstaller \
-    --additional-hooks-dir $(pwd)/scripts/hooks dvc/__main__.py \
-    --name dvc \
-    --distpath $LIB_DIR \
-    --specpath $BUILD_DIR
-
-  $LIB_DIR/dvc/dvc --help
-
-  # NOTE: in osxpkg fpm replaces symlinks with actual file that it
-  # points to, so we need to use after-install hook. See FPM_FLAGS
-  # above.
-  if [[ "$(uname)" == 'Linux' ]]; then
-    mkdir -p $BIN_DIR
-    pushd $BIN_DIR
-    ln -s ../lib/dvc/dvc dvc
-    popd
-    $BIN_DIR/dvc --help
-  fi
-
-  # NOTE: temporarily not adding scripts to mac package. See [1]
-  # [1] https://github.com/iterative/dvc/issues/2585
-  if [[ "$(uname)" == 'Linux' ]]; then
-    mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR
-    $LIB_DIR/dvc/dvc completion -s bash > $BUILD_DIR/$BASH_CMPLT_DIR/dvc
-
-    mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR
-    $LIB_DIR/dvc/dvc completion -s zsh > $BUILD_DIR/$ZSH_CMPLT_DIR/_dvc
-  fi
-}
-
-build() {
-  cleanup
-  echo "PKG = \"$1\"" >dvc/utils/build.py
-  build_dvc
-  fpm_build $1
-}
-
-install_dependencies
-
-if [[ "$(uname)" == 'Linux' ]]; then
-  build rpm
-  build deb
-else
-  build osxpkg
-fi
-
-cleanup
-
-print_info "Successfully built dvc packages"

scripts/build_windows.cmd

@@ -1,48 +0,0 @@
-@echo Off
-
-echo ====== Starting to build dvc installer for Windows... ======
-
-if not exist dvc\NUL (echo Error: Please run this script from repository root && goto :error)
-
-rmdir /Q /S build
-rmdir /Q /S dist
-del /Q /S dvc.spec
-del /Q /S "dvc-*.exe"
-
-where pip
-if %errorlevel% neq 0 (echo Error: pip not found && goto :error)
-
-where choco
-if %errorlevel% neq 0 (echo Error: choco not found && goto :error)
-
-choco install InnoSetup
-call refreshenv
-where iscc
-if %errorlevel% neq 0 (echo Error: Couldn't find Inno Setup compiler. && goto :error)
-
-echo ====== Installing requirements... ======
-echo PKG = "exe" > dvc\utils\build.py
-call pip install .[all] || goto :error
-call pip install -r scripts\build-requirements.txt || goto :error
-call dvc pull || goto :error
-
-echo ====== Building dvc binary... ======
-call pyinstaller --additional-hooks-dir scripts\hooks dvc/__main__.py --name dvc --specpath build
-
-echo ====== Testing dvc binary... ======
-call dist\dvc\dvc.exe version || goto :error
-
-echo ====== Copying additional files... ======
-copy scripts\innosetup\addSymLinkPermissions.ps1 dist\ || goto :error
-
-echo ====== Building dvc installer... ======
-set PYTHONPATH=%cd%
-call python scripts\innosetup\config_gen.py || goto :error
-call iscc scripts\innosetup\setup.iss || goto :error
-
-echo ====== DONE ======
-goto :EOF
-
-:error
-echo ====== FAIL ======
-exit /b 1

scripts/fpm/.gitignore

@@ -0,0 +1,4 @@
+/build
+/*.deb
+/*.rpm
+/*.pkg

scripts/fpm/build.py

@@ -0,0 +1,96 @@
+import argparse
+import os
+import pathlib
+import shutil
+from subprocess import STDOUT, check_call, check_output
+
+path = pathlib.Path(__file__).parent.absolute()
+dvc = path.parent.parent / "dvc"
+pyinstaller = path.parent / "pyinstaller"
+
+build = path / "build"
+install = build / "usr"
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "pkg", choices=["deb", "rpm", "osxpkg"], help="package type"
+)
+args = parser.parse_args()
+
+flags = [
+    "--description",
+    '"Data Version Control | Git for Data & Models"',
+    "-n",
+    "dvc",
+    "-s",
+    "dir",
+    "-f",
+    "--license",
+    '"Apache License 2.0"',
+]
+
+if args.pkg == "osxpkg":
+    install /= "local"
+    bash_dir = install / "etc" / "bash_completion.d"
+    dirs = ["usr"]
+    flags.extend(
+        [
+            "--osxpkg-identifier-prefix",
+            "com.iterative",
+            "--after-install",
+            path / "after-install.sh",
+            "--after-remove",
+            path / "after-remove.sh",
+        ]
+    )
+else:
+    bash_dir = build / "etc" / "bash_completion.d"
+    dirs = ["usr", "etc"]
+    flags.extend(["--depends", "git >= 1.7.0"])  # needed for gitpython
+
+try:
+    shutil.rmtree(build)
+except FileNotFoundError:
+    pass
+
+lib = install / "lib"
+lib.mkdir(parents=True)
+shutil.copytree(pyinstaller / "dist" / "dvc", lib / "dvc")
+
+if args.pkg != "osxpkg":
+    # NOTE: in osxpkg fpm replaces symlinks with actual file that it
+    # points to, so we need to use after-install hook.
+    (install / "bin").mkdir()
+    os.symlink("../lib/dvc/dvc", install / "bin" / "dvc")
+
+bash_dir.mkdir(parents=True)
+bash_completion = check_output(
+    [lib / "dvc" / "dvc", "completion", "-s", "bash"], text=True
+)
+(bash_dir / "dvc").write_text(bash_completion)
+
+zsh_dir = install / "share" / "zsh" / "site-functions"
+zsh_dir.mkdir(parents=True)
+zsh_completion = check_output(
+    [lib / "dvc" / "dvc", "completion", "-s", "zsh"], text=True
+)
+(zsh_dir / "_dvc").write_text(zsh_completion)
+
+version = check_output([lib / "dvc" / "dvc", "--version"], text=True).strip()
+
+check_call(
+    [
+        "fpm",
+        "--verbose",
+        "-t",
+        args.pkg,
+        *flags,
+        "-v",
+        version,
+        "-C",
+        build,
+        *dirs,
+    ],
+    cwd=path,
+    stderr=STDOUT,
+)

scripts/fpm/notarize.py

@@ -0,0 +1,64 @@
+import argparse
+import json
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "path",
+    nargs="?",
+    help="Path to the osxpkg to notarize. If not specified - try to find one.",
+)
+parser.add_argument(
+    "--apple-id-username", required=True, help="Apple ID username."
+)
+parser.add_argument(
+    "--apple-id-password",
+    required=True,
+    help=(
+        "Apple ID app-specific password. Note that this is not a regular "
+        "Apple ID password, so you need to generate one at "
+        "https://appleid.apple.com/account/manage"
+    ),
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+
+if args.path:
+    pkg = pathlib.Path(args.path)
+else:
+    pkgs = list(path.glob("*.pkg"))
+    if not pkgs:
+        print("No pkgs found")
+        exit(1)
+
+    if len(pkgs) > 1:
+        print("Too many packages")
+        exit(1)
+
+    (pkg,) = pkgs
+
+
+config = {
+    "notarize": {
+        "path": os.fspath(pkg),
+        "bundle_id": "com.iterative.dvc",
+        "staple": True,
+    },
+    "apple_id": {
+        "username": args.apple_id_username,
+        "password": args.apple_id_password,
+    },
+}
+
+(path / "config.json").write_text(json.dumps(config))
+
+check_call(
+    ["gon", "config.json"], cwd=path, stderr=STDOUT,
+)

scripts/fpm/sign.py

@@ -0,0 +1,54 @@
+import argparse
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "path",
+    nargs="?",
+    help="Path to the osxpkg to sign. If not specified - try to find one.",
+)
+parser.add_argument(
+    "--installer-id",
+    required=True,
+    help="Certificate ID (should be added to the keychain).",
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+
+if args.path:
+    pkg = pathlib.Path(args.path)
+else:
+    pkgs = list(path.glob("*.pkg"))
+    if not pkgs:
+        print("No pkgs found")
+        exit(1)
+
+    if len(pkgs) > 1:
+        print("Too many packages")
+        exit(1)
+
+    (pkg,) = pkgs
+
+unsigned = pkg.with_suffix(".unsigned")
+os.rename(pkg, unsigned)
+check_call(
+    [
+        "productsign",
+        "--sign",
+        args.installer_id,
+        os.fspath(unsigned),
+        os.fspath(pkg),
+    ],
+    stderr=STDOUT,
+)
+
+check_call(
+    ["pkgutil", "--check-signature", os.fspath(pkg)], stderr=STDOUT,
+)

scripts/innosetup/.gitignore

@@ -1,4 +1,6 @@
-dvc.ico
-dvc_left.bmp
-dvc_up.bmp
+/dvc.ico
+/dvc_left.bmp
+/dvc_up.bmp
 /config.ini
+/build
+/dvc*.exe

scripts/innosetup/build.py

@@ -0,0 +1,42 @@
+import argparse
+import configparser
+import os
+import pathlib
+import shutil
+from subprocess import STDOUT, check_call, check_output
+
+path = pathlib.Path(__file__).parent.absolute()
+config = path / "config.ini"
+
+dvc = path.parent.parent / "dvc"
+pyinstaller = path.parent / "pyinstaller"
+
+build = path / "build"
+install = build / "usr"
+
+parser = argparse.ArgumentParser()
+args = parser.parse_args()
+
+try:
+    shutil.rmtree(build)
+except FileNotFoundError:
+    pass
+
+build.mkdir()
+shutil.copytree(pyinstaller / "dist" / "dvc", build / "dvc")
+shutil.copy(path / "addSymLinkPermissions.ps1", build)
+
+version = check_output(
+    [os.fspath(build / "dvc" / "dvc"), "--version"], text=True
+).strip()
+
+cfg = configparser.ConfigParser()
+cfg.add_section("Version")
+cfg.set("Version", "version", version)
+
+with (path / "config.ini").open("w") as fobj:
+    cfg.write(fobj)
+
+check_call(
+    ["iscc", "setup.iss"], cwd=path, stderr=STDOUT,
+)

scripts/innosetup/config_gen.py

@@ -1,11 +0,0 @@
-# This script generates config.ini for setup.iss script
-import configparser
-
-from dvc import __version__
-
-config = configparser.ConfigParser()
-config.add_section("Version")
-config.set("Version", "Version", __version__)
-
-with open("scripts/innosetup/config.ini", "w") as f:
-    config.write(f)

scripts/innosetup/setup.iss

@@ -1,8 +1,8 @@
 #define MyAppName "DVC (Data Version Control)"
-#define MyAppVersion ReadIni(".\scripts\innosetup\config.ini", "Version", "version", "unknown")
+#define MyAppVersion ReadIni(SourcePath + "\config.ini", "Version", "version", "unknown")
 #define MyAppPublisher "Dmitry Petrov"
-#define MyAppURL "https://dataversioncontrol.com/"
-#define MyAppDir "..\..\dist\dvc"
+#define MyAppURL "https://dvc.org"
+#define MyAppDir SourcePath + "\build\dvc"
 
 [Setup]
 AppId={{8258CE8A-110E-4E0D-AE60-FEE00B15F041}
@@ -19,7 +19,7 @@ LicenseFile=..\..\LICENSE
 OutputBaseFilename=dvc-{#MyAppVersion}
 Compression=lzma2/max
 SolidCompression=yes
-OutputDir=..\..\
+OutputDir=.
 ChangesEnvironment=yes
 SetupIconFile=dvc.ico
 WizardSmallImageFile=dvc_up.bmp

scripts/pyinstaller/.gitignore

@@ -0,0 +1,4 @@
+/build
+/dist
+/dvc.spec
+/dvc

scripts/pyinstaller/build.py

@@ -0,0 +1,22 @@
+import os
+import pathlib
+from subprocess import STDOUT, check_call
+
+path = pathlib.Path(__file__).parent.absolute()
+hooks = path / "hooks"
+dvc = path.parent.parent / "dvc"
+entry = dvc / "__main__.py"
+
+check_call(
+    [
+        "pyinstaller",
+        "--additional-hooks-dir",
+        os.fspath(hooks),
+        "--name",
+        "dvc",
+        "-y",
+        os.fspath(entry),
+    ],
+    cwd=path,
+    stderr=STDOUT,
+)

scripts/pyinstaller/entitlements.plist

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<!-- These are required for binaries built by PyInstaller -->
+	<key>com.apple.security.cs.allow-jit</key>
+	<true/>
+	<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+	<true/>
+</dict>
+</plist>

scripts/pyinstaller/sign.py

@@ -0,0 +1,39 @@
+import argparse
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--application-id",
+    required=True,
+    help="Certificate ID (should be added to the keychain).",
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+dvc = path / "dist" / "dvc"
+for root, _, fnames in os.walk(dvc):
+    for fname in fnames:
+        fpath = os.path.join(root, fname)
+        print(f"signing {fpath}")
+        check_call(
+            [
+                "codesign",
+                "--force",
+                "--verbose",
+                "-s",
+                args.application_id,
+                "-o",
+                "runtime",
+                "--entitlements",
+                "entitlements.plist",
+                fpath,
+            ],
+            stderr=STDOUT,
+            timeout=180,
+        )

setup.py

@@ -1,5 +1,6 @@
 import importlib.util
 import os
+from pathlib import Path
 
 from setuptools import find_packages, setup
 from setuptools.command.build_py import build_py as _build_py
@@ -50,7 +51,7 @@ install_requires = [
     "colorama>=0.3.9",
     "configobj>=5.0.6",
     "gitpython>3",
-    "dulwich>=0.20.20",
+    "dulwich>=0.20.21",
     "pygit2>=1.5.0",
     "setuptools>=34.0.0",
     "nanotime>=0.5.2",
@@ -82,24 +83,24 @@ install_requires = [
     "pygtrie==2.3.2",
     "dpath>=2.0.1,<3",
     "shtab>=1.3.4,<2",
-    "rich>=9.0.0",
+    "rich>=10.0.0",
     "dictdiffer>=0.8.1",
     "python-benedict>=0.21.1",
     "pyparsing==2.4.7",
     "typing_extensions>=3.7.4",
-    "fsspec>=0.8.5",
+    "fsspec==0.9.0",
     "diskcache>=5.2.1",
 ]
 
 
 # Extra dependencies for remote integrations
 
-gs = ["gcsfs>=0.7.2"]
+gs = ["gcsfs==0.8.0"]
 gdrive = ["pydrive2>=1.8.1", "six >= 1.13.0"]
 s3 = ["boto3>=1.9.201"]
-azure = ["adlfs>=0.6.3", "azure-identity>=1.4.0", "knack"]
+azure = ["adlfs==0.7.1", "azure-identity>=1.4.0", "knack"]
 # https://github.com/Legrandin/pycryptodome/issues/465
-oss = ["oss2==2.6.1", "pycryptodome<3.9.9"]
+oss = ["oss2==2.6.1", "pycryptodome>=3.10"]
 ssh = ["paramiko[invoke]>=2.7.0"]
 
 # Remove the env marker if/when pyarrow is available for Python3.9
@@ -113,49 +114,9 @@ webdav = ["webdavclient3>=3.14.5"]
 ssh_gssapi = ["paramiko[invoke,gssapi]>=2.7.0"]
 all_remotes = gs + s3 + azure + ssh + oss + gdrive + hdfs + webhdfs + webdav
 
-# Extra dependecies to run tests
-tests_requirements = [
-    "wheel>=0.31.1",
-    # Test requirements:
-    "pytest>=6.0.1,<6.2.2",
-    "pytest-docker>=0.7.2",
-    "pytest-timeout>=1.3.3",
-    "pytest-cov>=2.6.1",
-    "pytest-xdist>=1.26.1",
-    "pytest-mock==1.11.2",
-    "pytest-lazy-fixture",
-    "pytest-tap",
-    "flaky>=3.5.3",
-    "mock>=3.0.0",
-    "xmltodict>=0.11.0",
-    "google-compute-engine==2.8.13",
-    "Pygments",  # required by collective.checkdocs,
-    "collective.checkdocs",
-    "psutil",
-    "pydocstyle<4.0",
-    "jaraco.windows==3.9.2",
-    "mock-ssh-server>=0.8.2",
-    "moto==1.3.16.dev122",
-    # moto's indirect dependency that is causing problems with flufl.lock's
-    # dependency (atpublic). See https://github.com/iterative/dvc/pull/4853
-    "aws-sam-translator<1.29.0",
-    # for moto's indirect dependency.
-    # See https://github.com/iterative/dvc/pull/4879
-    "urllib3<1.26.0",
-    "rangehttpserver==1.2.0",
-    "beautifulsoup4==4.4.0",
-    "pylint==2.7.2",
-    "pylint-pytest==0.3.0",
-    "pylint-plugin-utils==0.6",
-    "wget",
-    "filelock",
-    "mypy",
-    "wsgidav",
-    "crc32c",
-    "google-cloud-storage==1.19.0",
-    # pypi doesn't allow for direct dependencies
-    #    "gdrivefs @ git+https://github.com/intake/gdrivefs.git",
-]
+tests_requirements = (
+    Path("test_requirements.txt").read_text().strip().splitlines()
+)
 
 setup(
     name="dvc",

snap/snapcraft.yaml

@@ -1,59 +0,0 @@
-name: dvc
-summary: Data Version Control
-description: Git for Data & Models https://dvc.org
-adopt-info: dvc
-grade: stable
-confinement: classic  # for `dvc run`ning external binaries
-base: core18
-license: Apache-2.0
-layout:
-  /etc/dvc:
-    bind: $SNAP_DATA/etc/dvc
-  /usr/share/git-core:
-    bind: $SNAP/usr/share/git-core
-  /usr/share/gitweb:
-    bind: $SNAP/usr/share/gitweb
-  /usr/lib/git-core:
-    bind: $SNAP/usr/lib/git-core
-parts:
-  dvc:
-    plugin: python
-    python-packages: [wheel]
-    source: .
-    build-snaps:
-    - snapd
-    build-packages:
-    - git
-    - libc-dev
-    stage-packages:
-    - git
-    override-pull: |
-        snapcraftctl pull
-        git diff --quiet || error_dirty_build
-        echo 'PKG = "snap"' > dvc/utils/build.py
-        # install all optional extras
-        sed -ri 's/(=install_requires)/\1+all_remotes+hdfs/' setup.py
-        # ensure dvc knows the state isn't really dirty
-        sed -rin 's/.*git.*diff.*--quiet.*//' dvc/version.py
-    override-build: |
-        snapcraftctl build
-        # prevent user site packages interfering with this snap - reference:
-        # https://github.com/snapcore/snapcraft/blob/19393ef36cd773a28131cec10cc0bfb3bf9c7e77/tools/snapcraft-override-build.sh#L18
-        sed -ri 's/^(ENABLE_USER_SITE = )None$/\1False/' $SNAPCRAFT_PART_INSTALL/usr/lib/python*/site.py
-        # make plugin aware that a file (generated later) needs to be staged
-        touch $SNAPCRAFT_PART_INSTALL/completion.sh
-    override-stage: |
-        snapcraftctl stage
-        snapcraftctl set-version $(usr/bin/python3 -m dvc --version)
-        usr/bin/python3 -m dvc completion -s bash > completion.sh
-apps:
-  dvc:
-    command: bin/dvc
-    completer: completion.sh
-    environment:
-      GIT_PYTHON_GIT_EXECUTABLE: $SNAP/usr/bin/git
-      XDG_CONFIG_DIRS: /etc
-  git:
-    command: usr/bin/git
-    environment:
-      GIT_DISCOVERY_ACROSS_FILESYSTEM: 1

test_requirements.txt

@@ -0,0 +1,48 @@
+wheel==0.36.2
+
+# Test requirements
+pytest==6.2.3
+pytest-cov==2.11.1
+pytest-xdist==2.2.1
+pytest-mock==3.5.1
+pytest-lazy-fixture==0.6.3
+pytest-timeout==1.4.2
+pytest-docker==0.10.1
+
+flaky==3.7.0
+mock==4.0.3
+rangehttpserver==1.2.0
+mock-ssh-server==0.9.0
+# moto's indirect dependency that is causing problems with flufl.lock's
+# dependency (atpublic). See https://github.com/iterative/dvc/pull/4853
+moto==2.0.5
+beautifulsoup4==4.9.3
+wget==3.2
+filelock==3.0.12
+wsgidav==3.1.0
+crc32c==2.2
+xmltodict==0.12.0
+google-compute-engine==2.8.13
+google-cloud-storage==1.37.1
+
+urllib3==1.26.4
+
+# required by collective.checkdocs
+Pygments==2.8.1
+collective.checkdocs==0.2
+pydocstyle==6.0.0
+jaraco.windows==5.5.0
+
+# pylint requirements
+pylint==2.7.4
+# we use this to suppress pytest-related false positives in our tests.
+pylint-pytest==1.0.3
+# we use this to suppress some messages in tests, eg: foo/bar naming,
+# and, protected method calls in our tests
+pylint-plugin-utils==0.6
+
+# type-checking
+mypy==0.812
+
+# pypi doesn't allow for direct dependencies
+# gdrivefs @ git+https://github.com/intake/gdrivefs.git

tests/__main__.py

@@ -14,7 +14,7 @@ os.putenv(
 )
 os.putenv("DVC_HOME", REPO_ROOT)
 
+opts = "-v -n=auto --dist loadscope --cov=dvc --durations=0"
 params = " ".join(sys.argv[1:])
-
-cmd = f"pytest -v -n=auto --dist loadscope --cov=dvc --durations=0 {params}"
+cmd = f"{sys.executable} -m pytest {opts} {params}"
 check_call(cmd, shell=True)

tests/conftest.py

@@ -43,6 +43,13 @@ def reset_loglevel(request, caplog):
         yield
 
 
+@pytest.fixture(autouse=True)
+def enable_ui():
+    from dvc.ui import ui
+
+    ui.enable()
+
+
 @pytest.fixture(scope="session", autouse=True)
 def _close_pools():
     from dvc.fs.pool import close_pools

tests/func/experiments/test_show.py

@@ -5,8 +5,11 @@ from datetime import datetime
 import pytest
 from funcy import first
 
+from dvc.exceptions import InvalidArgumentError
 from dvc.main import main
 from dvc.repo.experiments.base import ExpRefInfo
+from dvc.utils.serialize import dump_yaml
+from tests.func.test_repro_multistage import COPY_SCRIPT
 
 
 def test_show_simple(tmp_dir, scm, dvc, exp_stage):
@@ -157,47 +160,156 @@ def test_show_checkpoint_branch(
     assert f"({branch_rev[:7]})" in cap.out
 
 
-def test_show_filter(tmp_dir, scm, dvc, exp_stage, capsys):
+@pytest.mark.parametrize(
+    "i_metrics,i_params,e_metrics,e_params,included,excluded",
+    [
+        (
+            "foo",
+            "foo",
+            None,
+            None,
+            ["foo"],
+            ["bar", "train/foo", "nested.foo"],
+        ),
+        (
+            None,
+            None,
+            "foo",
+            "foo",
+            ["bar", "train/foo", "nested.foo"],
+            ["foo"],
+        ),
+        (
+            "foo,bar",
+            "foo,bar",
+            None,
+            None,
+            ["foo", "bar"],
+            ["train/foo", "train/bar", "nested.foo", "nested.bar"],
+        ),
+        (
+            "metrics.yaml:foo,bar",
+            "params.yaml:foo,bar",
+            None,
+            None,
+            ["foo", "bar"],
+            ["train/foo", "train/bar", "nested.foo", "nested.bar"],
+        ),
+        (
+            "train/*",
+            "train/*",
+            None,
+            None,
+            ["train/foo", "train/bar"],
+            ["foo", "bar", "nested.foo", "nested.bar"],
+        ),
+        (
+            None,
+            None,
+            "train/*",
+            "train/*",
+            ["foo", "bar", "nested.foo", "nested.bar"],
+            ["train/foo", "train/bar"],
+        ),
+        (
+            "train/*",
+            "train/*",
+            "*foo",
+            "*foo",
+            ["train/bar"],
+            ["train/foo", "foo", "bar", "nested.foo", "nested.bar"],
+        ),
+        (
+            "nested.*",
+            "nested.*",
+            None,
+            None,
+            ["nested.foo", "nested.bar"],
+            ["foo", "bar", "train/foo", "train/bar"],
+        ),
+        (
+            None,
+            None,
+            "nested.*",
+            "nested.*",
+            ["foo", "bar", "train/foo", "train/bar"],
+            ["nested.foo", "nested.bar"],
+        ),
+        (
+            "*.*",
+            "*.*",
+            "*.bar",
+            "*.bar",
+            ["nested.foo"],
+            ["foo", "bar", "nested.bar", "train/foo", "train/bar"],
+        ),
+    ],
+)
+def test_show_filter(
+    tmp_dir,
+    scm,
+    dvc,
+    capsys,
+    i_metrics,
+    i_params,
+    e_metrics,
+    e_params,
+    included,
+    excluded,
+):
     capsys.readouterr()
-    assert (
-        main(
-            [
-                "exp",
-                "show",
-                "--no-pager",
-                "--no-timestamp",
-                "--include-metrics=foo",
-                "--include-params=foo",
-            ]
-        )
-        == 0
-    )
-    cap = capsys.readouterr()
-
     div = "│" if os.name == "nt" else "┃"
-    assert f"{div} foo {div} foo {div}" in cap.out
 
-    assert (
-        main(
-            [
-                "exp",
-                "show",
-                "--no-pager",
-                "--no-timestamp",
-                "--exclude-metrics=foo",
-                "--exclude-params=foo",
-            ]
-        )
-        == 0
+    tmp_dir.gen("copy.py", COPY_SCRIPT)
+    params_file = tmp_dir / "params.yaml"
+    params_data = {
+        "foo": 1,
+        "bar": 1,
+        "train/foo": 1,
+        "train/bar": 1,
+        "nested": {"foo": 1, "bar": 1},
+    }
+    dump_yaml(params_file, params_data)
+
+    dvc.run(
+        cmd="python copy.py params.yaml metrics.yaml",
+        metrics_no_cache=["metrics.yaml"],
+        params=["foo"],
+        name="copy-file",
+        deps=["copy.py"],
     )
+    scm.add(
+        [
+            "dvc.yaml",
+            "dvc.lock",
+            "copy.py",
+            "params.yaml",
+            "metrics.yaml",
+            ".gitignore",
+        ]
+    )
+    scm.commit("init")
+
+    command = ["exp", "show", "--no-pager", "--no-timestamp"]
+    if i_metrics is not None:
+        command.append(f"--include-metrics={i_metrics}")
+    if i_params is not None:
+        command.append(f"--include-params={i_params}")
+    if e_metrics is not None:
+        command.append(f"--exclude-metrics={e_metrics}")
+    if e_params is not None:
+        command.append(f"--exclude-params={e_params}")
+
+    assert main(command) == 0
     cap = capsys.readouterr()
 
-    assert f"{div} foo {div}" not in cap.out
+    for i in included:
+        assert f"{div} {i} {div}" in cap.out
+    for e in excluded:
+        assert f"{div} {e} {div}" not in cap.out
 
 
 def test_show_multiple_commits(tmp_dir, scm, dvc, exp_stage):
-    from dvc.exceptions import InvalidArgumentError
-
     init_rev = scm.get_rev()
     tmp_dir.scm_gen("file", "file", "commit")
     next_rev = scm.get_rev()

tests/func/params/test_diff.py

@@ -64,19 +64,6 @@ def test_diff_deleted(tmp_dir, scm, dvc):
     }
 
 
-def test_diff_deleted_config(tmp_dir, scm, dvc):
-    tmp_dir.gen("params.yaml", "foo: bar")
-    dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)
-    scm.add(["params.yaml", "Dvcfile"])
-    scm.commit("bar")
-
-    (tmp_dir / "params.yaml").unlink()
-
-    assert dvc.params.diff() == {
-        "params.yaml": {"foo": {"old": "bar", "new": None}}
-    }
-
-
 def test_diff_list(tmp_dir, scm, dvc):
     tmp_dir.gen("params.yaml", "foo:\n- bar\n- baz")
     dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True)

tests/func/params/test_show.py

@@ -80,20 +80,31 @@ def test_show_branch(tmp_dir, scm, dvc):
     }
 
 
-def test_pipeline_tracked_params(tmp_dir, scm, dvc, run_copy):
+def test_pipeline_params(tmp_dir, scm, dvc, run_copy):
     from dvc.dvcfile import PIPELINE_FILE
 
-    tmp_dir.gen({"foo": "foo", "params.yaml": "foo: bar\nxyz: val"})
+    tmp_dir.gen(
+        {"foo": "foo", "params.yaml": "foo: bar\nxyz: val\nabc: ignore"}
+    )
     run_copy("foo", "bar", name="copy-foo-bar", params=["foo,xyz"])
     scm.add(["params.yaml", PIPELINE_FILE])
     scm.commit("add stage")
 
-    tmp_dir.scm_gen("params.yaml", "foo: baz\nxyz: val", commit="baz")
-    tmp_dir.scm_gen("params.yaml", "foo: qux\nxyz: val", commit="qux")
+    tmp_dir.scm_gen(
+        "params.yaml", "foo: baz\nxyz: val\nabc: ignore", commit="baz"
+    )
+    tmp_dir.scm_gen(
+        "params.yaml", "foo: qux\nxyz: val\nabc: ignore", commit="qux"
+    )
 
-    assert dvc.params.show(revs=["master"]) == {
+    assert dvc.params.show(revs=["master"], deps=True) == {
         "master": {"params.yaml": {"foo": "qux", "xyz": "val"}}
     }
+    assert dvc.params.show(revs=["master"]) == {
+        "master": {
+            "params.yaml": {"foo": "qux", "xyz": "val", "abc": "ignore"}
+        }
+    }
 
 
 def test_show_no_repo(tmp_dir):

tests/func/parsing/test_resolver.py

@@ -1,10 +1,11 @@
+import os
 from copy import deepcopy
 
 import pytest
 
 from dvc.parsing import DEFAULT_PARAMS_FILE, DataResolver, ResolveError
 from dvc.parsing.context import Context
-from dvc.utils.serialize import dump_json, dump_yaml
+from dvc.utils.serialize import dump_json, dump_yaml, dumps_yaml
 
 from . import CONTEXT_DATA, RESOLVED_DVC_YAML_DATA, TEMPLATED_DVC_YAML_DATA
 
@@ -67,6 +68,19 @@ def test_load_vars_from_file(tmp_dir, dvc):
     assert resolver.context == expected
 
 
+def test_load_vars_with_relpath(tmp_dir, scm, dvc):
+    tmp_dir.scm_gen(DEFAULT_PARAMS_FILE, dumps_yaml(DATA), commit="add params")
+
+    subdir = tmp_dir / "subdir"
+    d = {"vars": [os.path.relpath(tmp_dir / DEFAULT_PARAMS_FILE, subdir)]}
+
+    revisions = ["HEAD", "workspace"]
+    for rev in dvc.brancher(revs=["HEAD"]):
+        assert rev == revisions.pop()
+        resolver = DataResolver(dvc, subdir, d)
+        assert resolver.context == deepcopy(DATA)
+
+
 def test_partial_vars_doesnot_exist(tmp_dir, dvc):
     dump_yaml("test_params.yaml", {"sub1": "sub1"})
 

tests/func/plots/test_show.py

@@ -708,7 +708,7 @@ def test_show_from_subdir(tmp_dir, dvc, caplog):
     with subdir.chdir(), caplog.at_level(logging.INFO, "dvc"):
         assert main(["plots", "show", "metric.json"]) == 0
 
-    assert f"file://{str(subdir)}" in caplog.text
+    assert subdir.as_uri() in caplog.text
     assert (subdir / "plots.html").exists()
 
 

tests/func/test_api.py

@@ -182,6 +182,15 @@ def test_open_not_cached(dvc):
         api.read(metric_file)
 
 
+def test_open_rev(tmp_dir, scm, dvc):
+    tmp_dir.scm_gen("foo", "foo", commit="foo")
+
+    (tmp_dir / "foo").write_text("bar")
+
+    with api.open("foo", rev="master") as fobj:
+        assert fobj.read() == "foo"
+
+
 @pytest.mark.parametrize("as_external", [True, False])
 @pytest.mark.parametrize("remote", [pytest.lazy_fixture("ssh")], indirect=True)
 @pytest.mark.parametrize(

tests/func/test_external_repo.py

@@ -52,7 +52,7 @@ def test_cache_reused(erepo_dir, mocker, local_cloud):
         erepo_dir.dvc_gen("file", "text", commit="add file")
     erepo_dir.dvc.push()
 
-    download_spy = mocker.spy(LocalFileSystem, "download")
+    download_spy = mocker.spy(LocalFileSystem, "download_file")
 
     # Use URL to prevent any fishy optimizations
     url = f"file://{erepo_dir}"

tests/func/test_ignore.py

@@ -112,7 +112,7 @@ def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
     assert (
         DvcIgnorePatterns(
             *merge_patterns(
-                _to_pattern_info_list([".hg/", ".git/", ".dvc/"]),
+                _to_pattern_info_list([".hg/", ".git/", ".git", ".dvc/"]),
                 os.fspath(tmp_dir),
                 _to_pattern_info_list([os.path.basename(dname)]),
                 top_ignore_path,
@@ -344,7 +344,7 @@ def test_pattern_trie_fs(tmp_dir, dvc):
     )
 
     base_pattern = (
-        _to_pattern_info_list([".hg/", ".git/", ".dvc/"]),
+        _to_pattern_info_list([".hg/", ".git/", ".git", ".dvc/"]),
         os.fspath(tmp_dir),
     )
     first_pattern = merge_patterns(

tests/unit/command/test_experiments.py

@@ -38,6 +38,7 @@ def test_experiments_diff(dvc, scm, mocker):
             "HEAD~10",
             "HEAD~1",
             "--all",
+            "--param-deps",
             "--show-json",
             "--show-md",
             "--old",
@@ -53,7 +54,7 @@ def test_experiments_diff(dvc, scm, mocker):
     assert cmd.run() == 0
 
     m.assert_called_once_with(
-        cmd.repo, a_rev="HEAD~10", b_rev="HEAD~1", all=True
+        cmd.repo, a_rev="HEAD~10", b_rev="HEAD~1", all=True, param_deps=True
     )
 
 
@@ -66,6 +67,7 @@ def test_experiments_show(dvc, scm, mocker):
             "--all-branches",
             "--all-commits",
             "--sha",
+            "--param-deps",
             "-n",
             "1",
         ]
@@ -84,6 +86,7 @@ def test_experiments_show(dvc, scm, mocker):
         all_commits=True,
         sha_only=True,
         num=1,
+        param_deps=True,
     )
 
 
@@ -93,7 +96,7 @@ def test_experiments_run(dvc, scm, mocker):
         "name": None,
         "queue": False,
         "run_all": False,
-        "jobs": None,
+        "jobs": 1,
         "tmp_dir": False,
         "checkpoint_resume": None,
         "reset": False,

tests/unit/command/test_params.py

@@ -18,6 +18,7 @@ def test_params_diff(dvc, mocker):
             "--show-json",
             "--show-md",
             "--no-path",
+            "--deps",
         ]
     )
     assert cli_args.func == CmdParamsDiff
@@ -33,6 +34,21 @@ def test_params_diff(dvc, mocker):
         b_rev="HEAD~1",
         targets=["target"],
         all=True,
+        deps=True,
+    )
+
+
+def test_params_diff_from_cli(dvc, mocker):
+    cli_args = parse_args(["params", "diff"])
+    assert cli_args.func == CmdParamsDiff
+
+    cmd = cli_args.func(cli_args)
+    m = mocker.patch("dvc.repo.params.diff.diff", return_value={})
+
+    assert cmd.run() == 0
+
+    m.assert_called_once_with(
+        cmd.repo, a_rev=None, b_rev=None, all=False, targets=None, deps=False,
     )
 
 

tests/unit/command/test_plots.py

@@ -1,4 +1,8 @@
 import logging
+import os
+import posixpath
+
+import pytest
 
 from dvc.cli import parse_args
 from dvc.command.plots import CmdPlotsDiff, CmdPlotsShow
@@ -116,12 +120,11 @@ def test_plots_diff_open(tmp_dir, dvc, mocker, caplog):
     )
 
     assert cmd.run() == 0
+    mocked_open.assert_called_once_with("plots.html")
 
-    expected_url = f"file://{tmp_dir / 'plots.html'}"
+    expected_url = posixpath.join(tmp_dir.as_uri(), "plots.html")
     assert expected_url in caplog.text
 
-    mocked_open.assert_called_once_with(expected_url)
-
 
 def test_plots_diff_open_failed(tmp_dir, dvc, mocker, caplog):
     mocked_open = mocker.patch("webbrowser.open", return_value=False)
@@ -132,12 +135,36 @@ def test_plots_diff_open_failed(tmp_dir, dvc, mocker, caplog):
     )
 
     assert cmd.run() == 1
-
-    expected_url = f"file://{tmp_dir / 'plots.html'}"
-    mocked_open.assert_called_once_with(expected_url)
+    mocked_open.assert_called_once_with("plots.html")
 
     error_message = "Failed to open. Please try opening it manually."
+    expected_url = posixpath.join(tmp_dir.as_uri(), "plots.html")
+
     assert caplog.record_tuples == [
         ("dvc.command.plots", logging.INFO, expected_url),
         ("dvc.command.plots", logging.ERROR, error_message),
     ]
+
+
+@pytest.mark.parametrize(
+    "output, expected_url_path",
+    [
+        ("plots file with spaces.html", "plots%20file%20with%20spaces.html"),
+        (os.path.join("dir", "..", "plots.html"), "plots.html"),
+    ],
+    ids=["quote", "resolve"],
+)
+def test_plots_path_is_quoted_and_resolved_properly(
+    tmp_dir, dvc, mocker, caplog, output, expected_url_path
+):
+    cli_args = parse_args(
+        ["plots", "diff", "--targets", "datafile", "--out", output]
+    )
+    cmd = cli_args.func(cli_args)
+    mocker.patch(
+        "dvc.repo.plots.diff.diff", return_value={"datafile": "filledtemplate"}
+    )
+
+    assert cmd.run() == 0
+    expected_url = posixpath.join(tmp_dir.as_uri(), expected_url_path)
+    assert expected_url in caplog.text

tests/unit/command/test_status.py

@@ -45,7 +45,7 @@ def test_cloud_status(tmp_dir, dvc, mocker):
 
 
 @pytest.mark.parametrize("status", [{}, {"a": "b", "c": [1, 2, 3]}, [1, 2, 3]])
-def test_status_show_json(mocker, caplog, status):
+def test_status_show_json(dvc, mocker, caplog, status):
     cli_args = parse_args(["status", "--show-json"])
     assert cli_args.func == CmdDataStatus
 
@@ -60,7 +60,7 @@ def test_status_show_json(mocker, caplog, status):
 @pytest.mark.parametrize(
     "status, ret", [({}, 0), ({"a": "b", "c": [1, 2, 3]}, 1), ([1, 2, 3], 1)]
 )
-def test_status_quiet(mocker, caplog, capsys, status, ret):
+def test_status_quiet(dvc, mocker, caplog, capsys, status, ret):
     cli_args = parse_args(["status", "-q"])
     assert cli_args.func == CmdDataStatus
 
@@ -71,5 +71,45 @@ def test_status_quiet(mocker, caplog, capsys, status, ret):
     assert cmd.run() == ret
     assert not caplog.messages
     captured = capsys.readouterr()
-    assert not captured.err
     assert not captured.out
+
+
+def test_status_empty(dvc, mocker, capsys):
+    cli_args = parse_args(["status"])
+    assert cli_args.func == CmdDataStatus
+
+    cmd = cli_args.func(cli_args)
+
+    spy = mocker.spy(cmd.repo.stage, "collect_repo")
+
+    assert cmd.run() == 0
+
+    captured = capsys.readouterr()
+    assert "no data or pipelines tracked" in captured.out
+    # stages should only be collected once
+    assert spy.call_count == 1
+
+
+@pytest.mark.parametrize(
+    "cloud_opts, expected_message",
+    [
+        (["--cloud"], "Cache and remote 'default' are in sync"),
+        (["--remote", "remote1"], "Cache and remote 'remote1' are in sync"),
+        ([], "Data and pipelines are up to date"),
+    ],
+)
+def test_status_up_to_date(dvc, mocker, capsys, cloud_opts, expected_message):
+    cli_args = parse_args(["status", *cloud_opts])
+    assert cli_args.func == CmdDataStatus
+
+    cmd = cli_args.func(cli_args)
+
+    mocker.patch.dict(cmd.repo.config, {"core": {"remote": "default"}})
+    mocker.patch.object(cmd.repo, "status", autospec=True, return_value={})
+    mocker.patch.object(
+        cmd.repo.stage, "collect_repo", return_value=[object()], autospec=True
+    )
+
+    assert cmd.run() == 0
+    captured = capsys.readouterr()
+    assert expected_message in captured.out

tests/unit/fs/test_hdfs.py

@@ -2,6 +2,7 @@ import os
 
 from dvc.fs.hdfs import _hadoop_fs_checksum
 from dvc.path_info import URLInfo
+from dvc.utils import fix_env
 
 
 def test_hadoop_fs_checksum(mocker):
@@ -22,7 +23,7 @@ def test_hadoop_fs_checksum(mocker):
         shell=True,
         close_fds=os.name != "nt",
         executable=os.getenv("SHELL") if os.name != "nt" else None,
-        env=os.environ,
+        env=fix_env(os.environ),
         stdin=-1,
         stdout=-1,
         stderr=-1,

tests/unit/fs/test_s3.py

@@ -29,6 +29,22 @@ def test_init(dvc):
     assert fs.path_info == url
 
 
+def test_verify_ssl_default_param(dvc):
+    config = {
+        "url": url,
+    }
+    fs = S3FileSystem(dvc, config)
+
+    assert fs.ssl_verify
+
+
+def test_ssl_verify_bool_param(dvc):
+    config = {"url": url, "ssl_verify": False}
+    fs = S3FileSystem(dvc, config)
+
+    assert fs.ssl_verify == config["ssl_verify"]
+
+
 def test_grants(dvc):
     config = {
         "url": url,

tests/unit/remote/test_base.py

@@ -70,7 +70,7 @@ def test_hashes_exist(object_exists, traverse, dvc):
     # large remote, large local
     object_exists.reset_mock()
     traverse.reset_mock()
-    odb.fs.JOBS = 16
+    odb.fs._JOBS = 16
     with mock.patch.object(odb, "list_hashes", return_value=list(range(256))):
         hashes = list(range(1000000))
         odb.hashes_exist(hashes)
@@ -94,7 +94,7 @@ def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc):
     odb.fs.path_info = PathInfo("foo")
 
     # parallel traverse
-    size = 256 / odb.fs.JOBS * odb.fs.LIST_OBJECT_PAGE_SIZE
+    size = 256 / odb.fs._JOBS * odb.fs.LIST_OBJECT_PAGE_SIZE
     list(odb.list_hashes_traverse(size, {0}))
     for i in range(1, 16):
         list_hashes.assert_any_call(

tests/unit/test_external_repo.py

@@ -68,7 +68,7 @@ def test_subrepo_is_constructed_properly(
 
         list(repo.repo_fs.walk(repo.root_dir))  # drain
         assert spy.call_count == 1
-        subrepo = spy.return_value
+        subrepo = spy.spy_return
 
         assert repo.url == str(tmp_dir)
         assert repo.config["cache"]["dir"] == str(cache_dir)

tests/unit/test_imports.py

@@ -1,4 +1,5 @@
 import subprocess
+import sys
 
 
 def test_no_remote_imports():
@@ -15,7 +16,7 @@ def test_no_remote_imports():
 
     code = "import dvc.main, sys; print(' '.join(sys.modules))"
     res = subprocess.run(
-        ["python", "-c", code], stdout=subprocess.PIPE, check=True
+        [sys.executable, "-c", code], stdout=subprocess.PIPE, check=True
     )
     modules = res.stdout.decode().split()
     assert not set(modules) & remote_modules

tests/unit/test_params.py

@@ -6,23 +6,28 @@ from dvc.utils.serialize import dumps_yaml
 def test_params_order(tmp_dir, dvc):
     tmp_dir.gen(
         {
-            "params.yaml": dumps_yaml({"p1": 1}),
-            "p1.yaml": dumps_yaml({"p2": 1}),
-            "sub": {"p2.yaml": dumps_yaml({"p3": 1})},
+            "params.yaml": dumps_yaml({"p": 1}),
+            "params1.yaml": dumps_yaml({"p1": 1}),
+            "sub": {"params2.yaml": dumps_yaml({"p2": 1})},
         }
     )
 
-    p2_path = os.path.join("sub", "p2.yaml")
+    params_path = os.path.join("..", "params.yaml")
+    p2_path = os.path.join("sub", "params2.yaml")
     dvc.stage.add(
-        params=[{p2_path: ["p3"]}, {"p1.yaml": ["p2"]}],
+        params=[{p2_path: ["p2"]}, {"params1.yaml": ["p1"]}],
         cmd="cmd1",
         name="stage1",
     )
     with (tmp_dir / "sub").chdir():
-        dvc.stage.add(params=["p1"], cmd="cmd2", name="stage2")
+        dvc.stage.add(params=[{params_path: ["p"]}], cmd="cmd2", name="stage2")
 
     # params are sorted during dumping, therefore p1 is first
-    assert list(dvc.params.show()[""]) == ["p1.yaml", p2_path, "params.yaml"]
+    assert list(dvc.params.show()[""]) == [
+        "params1.yaml",
+        p2_path,
+        "params.yaml",
+    ]
 
 
 def test_repro_unicode(tmp_dir, dvc):

tests/unit/ui/test_console.py

@@ -0,0 +1,28 @@
+from pytest import CaptureFixture
+
+from dvc.ui import Console
+
+
+def test_write(capsys: CaptureFixture[str]):
+    """Test that ui.write works."""
+    console = Console(enable=True)
+    message = "hello world"
+    console.write(message)
+    console.error_write(message)
+
+    captured = capsys.readouterr()
+    assert captured.out == f"{message}\n"
+    assert captured.err == f"{message}\n"
+
+
+def test_capsys_works(capsys: CaptureFixture[str]):
+    """Sanity check that capsys can capture outputs from a global ui."""
+    from dvc.ui import ui
+
+    message = "hello world"
+    ui.write(message)
+    ui.error_write(message)
+
+    captured = capsys.readouterr()
+    assert captured.out == f"{message}\n"
+    assert captured.err == f"{message}\n"

tests/unit/ui/test_table.py

@@ -0,0 +1,164 @@
+import textwrap
+
+import pytest
+from pytest import CaptureFixture
+from pytest_mock import MockerFixture
+from rich.style import Style
+
+from dvc.ui import ui
+
+
+def test_plain(capsys: CaptureFixture[str]):
+    ui.table(
+        [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")],
+        headers=["first", "second"],
+    )
+    captured = capsys.readouterr()
+    assert captured.out == textwrap.dedent(
+        """\
+        first    second
+        foo      bar
+        foo1     bar1
+        foo2     bar2
+    """
+    )
+
+
+def test_plain_md(capsys: CaptureFixture[str]):
+    ui.table(
+        [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")],
+        headers=["first", "second"],
+        markdown=True,
+    )
+    captured = capsys.readouterr()
+    assert captured.out == textwrap.dedent(
+        """\
+        | first   | second   |
+        |---------|----------|
+        | foo     | bar      |
+        | foo1    | bar1     |
+        | foo2    | bar2     |\n
+    """
+    )
+
+
+def test_plain_pager(mocker: MockerFixture):
+    pager_mock = mocker.patch("dvc.utils.pager.pager")
+    ui.table(
+        [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")],
+        headers=["first", "second"],
+        pager=True,
+    )
+
+    pager_mock.assert_called_once_with(
+        textwrap.dedent(
+            """\
+            first    second
+            foo      bar
+            foo1     bar1
+            foo2     bar2"""
+        )
+    )
+
+
+def test_plain_headerless(capsys: CaptureFixture[str]):
+    ui.table([("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")],)
+    captured = capsys.readouterr()
+    assert captured.out == textwrap.dedent(
+        """\
+        foo   bar
+        foo1  bar1
+        foo2  bar2
+    """
+    )
+
+
+def test_rich_simple(capsys: CaptureFixture[str]):
+    ui.table(
+        [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")],
+        headers=["first", "second"],
+        rich_table=True,
+    )
+    # not able to test the actual style for now
+    captured = capsys.readouterr()
+    assert [
+        row.strip() for row in captured.out.splitlines() if row.strip()
+    ] == ["first  second", "foo    bar", "foo1   bar1", "foo2   bar2"]
+
+
+def test_rich_headerless(capsys: CaptureFixture[str]):
+    ui.table(
+        [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")], rich_table=True,
+    )
+    captured = capsys.readouterr()
+    assert [
+        row.strip() for row in captured.out.splitlines() if row.strip()
+    ] == ["foo   bar", "foo1  bar1", "foo2  bar2"]
+
+
+def test_rich_border(capsys: CaptureFixture[str]):
+    ui.table(
+        [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")],
+        headers=["first", "second"],
+        rich_table=True,
+        borders="simple",
+    )
+    captured = capsys.readouterr()
+    assert [
+        row.strip() for row in captured.out.splitlines() if row.strip()
+    ] == [
+        "first   second",
+        "────────────────",
+        "foo     bar",
+        "foo1    bar1",
+        "foo2    bar2",
+    ]
+
+
+@pytest.mark.parametrize(
+    "extra_opts",
+    [
+        {"header_styles": [{"style": Style(bold=True)}]},
+        {"row_styles": [{"style": Style(bold=True)}]},
+    ],
+)
+def test_rich_styles(capsys: CaptureFixture[str], extra_opts):
+    ui.table(
+        [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")],
+        headers=["first", "second"],
+        rich_table=True,
+        **extra_opts
+    )
+    # not able to test the actual style for now
+    captured = capsys.readouterr()
+    assert [
+        row.strip() for row in captured.out.splitlines() if row.strip()
+    ] == ["first  second", "foo    bar", "foo1   bar1", "foo2   bar2"]
+
+
+def test_rich_pager(mocker: MockerFixture):
+    pager_mock = mocker.patch("dvc.utils.pager.pager")
+
+    ui.table(
+        [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")],
+        headers=["first", "second"],
+        rich_table=True,
+        pager=True,
+    )
+    received_text = pager_mock.call_args[0][0]
+    assert [
+        row.strip() for row in received_text.splitlines() if row.strip()
+    ] == ["first  second", "foo    bar", "foo1   bar1", "foo2   bar2"]
+
+
+@pytest.mark.parametrize("rich_table", [True, False])
+def test_empty(capsys: CaptureFixture[str], rich_table: str):
+    ui.table([], rich_table=rich_table)
+    out, err = capsys.readouterr()
+    assert (out, err) == ("", "")
+
+
+def test_empty_markdown(capsys: CaptureFixture[str]):
+    ui.table([], headers=["Col1", "Col2"], markdown=True)
+    out, err = capsys.readouterr()
+    assert (out, err) == ("| Col1   | Col2   |\n|--------|--------|\n\n", "")