dvc: bump to 2.1.0

I found that it's harder to style a header with the list using index,
they are to be applied by header name. Otherwise, we might end up using
a different header for coloring, or worse throw some error because the
header name and style don't match.
Row does have it's property of index, so it makes sense there.

.coveragerc

@@ -13,3 +13,4 @@ exclude_lines =
     raise NotImplementedError
     # pragma: no cover
     #  pragma: no cover
+    \.\.\.$

.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+
+updates:
+  - directory: "/"
+    package-ecosystem: "pip"
+    schedule:
+      interval: "daily"
+    labels:
+      - "maintenance"
+
+  - directory: "/"
+    package-ecosystem: "github-actions"
+    schedule:
+      interval: "daily"
+    labels:
+      - "maintenance"

.github/workflows/packages.yaml

@@ -6,91 +6,130 @@ on:
   release:
     types: [released, prereleased]
 jobs:
-  linux:
+  deb:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+    - uses: actions/[email protected].4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2.2.2
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Set up Ruby 2.4
       uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.4
-    - name: Build
-      run: ./scripts/build_posix.sh
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build deb
+      run: python scripts/build.py deb
     - name: Publish deb
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_path: scripts/fpm/dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
         asset_content_type: binary/octet-stream
-    - name: Publish rpm
-      if: github.event_name == 'release'
-      uses: actions/[email protected]
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
-        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
-        asset_content_type: binary/octet-stream
-  mac:
-    runs-on: macos-10.15
+  rpm:
+    runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+    - uses: actions/[email protected].4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2.2.2
       with:
-        python-version: 3.7
+        python-version: 3.8
     - name: Set up Ruby 2.4
       uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.4
-    - name: Build
-      run: ./scripts/build_posix.sh
-    - name: Publish
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build rpm
+      run: python scripts/build.py rpm
+    - name: Publish rpm
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_content_type: binary/octet-stream
+  osxpkg:
+    runs-on: macos-10.15
+    steps:
+    - uses: actions/[email protected]
+    - name: Set up Python 3.8
+      uses: actions/[email protected]
+      with:
+        python-version: 3.8
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Install deps
+      run: |
+        # available lxml (webdav dependency) wheels are built with an old
+        # SDK, which breaks osxpkg notarization. Building from scratch
+        # instead.
+        pip install .[all] --no-binary lxml
+        pip install -r scripts/build-requirements.txt
+        gem install --no-document fpm
+    - name: Build
+      run: python scripts/build.py osxpkg
+    - name: Publish
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}.pkg
         asset_name: dvc-${{ github.event.release.tag_name }}.pkg
         asset_content_type: binary/octet-stream
-  windows:
+  exe:
     runs-on: windows-2019
     steps:
-    - uses: actions/[email protected].2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+    - uses: actions/[email protected].4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2.2.2
       with:
-        python-version: 3.7
+        python-version: 3.8
+    - name: Install deps
+      run: |
+        pip install .[all]
+        pip install -r scripts/build-requirements.txt
+    - name: Pull images
+      run: |
+        dvc pull
     - name: Build
-      run: scripts\build_windows.cmd
+      run: python scripts\build.py exe
       shell: cmd
     - name: Publish
       if: github.event_name == 'release'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       with:
         upload_url: ${{ github.event.release.upload_url }}
-        asset_path: dvc-${{ github.event.release.tag_name }}.exe
+        asset_path: scripts/innosetup/dvc-${{ github.event.release.tag_name }}.exe
         asset_name: dvc-${{ github.event.release.tag_name }}.exe
         asset_content_type: binary/octet-stream
-  python:
+  pip:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.7
     - name: Install
@@ -103,29 +142,3 @@ jobs:
       with:
         user: __token__
         password: ${{ secrets.PYPI_TOKEN }}
-  snap:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/[email protected]
-    - name: Build the package
-      uses: snapcore/action-build@v1
-      id: snapcraft
-#    - name: Test the package
-#      run: |
-#        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
-#        dvc version
-#        sudo snap remove dvc
-    - name: Publish to the edge channel
-      uses: snapcore/action-publish@v1
-      if: github.event_name == 'push'
-      with:
-        store_login: ${{ secrets.STORE_LOGIN }}
-        snap: ${{ steps.snapcraft.outputs.snap }}
-        release: edge,v2/edge
-    - name: Publish to the stable channel
-      uses: snapcore/action-publish@v1
-      if: ${{ github.event_name == 'release' && github.event.release.released }}
-      with:
-        store_login: ${{ secrets.STORE_LOGIN }}
-        snap: ${{ steps.snapcraft.outputs.snap }}
-        release: stable,v2/stable

.github/workflows/tests.yaml

@@ -12,29 +12,27 @@ env:
   DVC_TEST: "true"
   HOMEBREW_NO_AUTO_UPDATE: 1
   SHELL: /bin/bash
-  GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: 3.8
     - name: Install requirements
-      run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
+      run: |
+        pip install wheel
+        pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
     - name: install temporary dependencies
       run: |
         pip install git+https://github.com/isidentical/gdrivefs@service-account
         pip install git+https://github.com/isidentical/pydata-google-auth@service-account
     - name: Check README
       run: python setup.py checkdocs
-    - uses: pre-commit/[email protected].0
+    - uses: pre-commit/[email protected].3
   tests:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -51,9 +49,9 @@ jobs:
             pyv: "3.9"
 
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.2
       with:
         python-version: ${{ matrix.pyv }}
     - name: get pip cache dir
@@ -62,7 +60,7 @@ jobs:
         echo "::set-output name=dir::$(pip cache dir)"
     - name: set pip cache
       id: pip-cache
-      uses: actions/cache@v2
+      uses: actions/cache@v2.1.5
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
@@ -83,14 +81,20 @@ jobs:
         git config --global user.name "DVC Tester"
     - name: setup gs creds
       shell: bash
+      env:
+        GS_CREDS_JSON: ${{ secrets.GS_CREDS_JSON }}
       if: env.GS_CREDS_JSON != ''
       run: |
         mkdir -p scripts/ci
         echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
     - name: run tests
-      run: python -m tests --all --cov-report=xml --cov-report=term --tap-combined -n=4
+      env:
+        GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      run: python -m tests --all --cov-report=xml --cov-report=term -n=4
     - name: upload coverage report
-      uses: codecov/codecov-action@v1.0.13
+      uses: codecov/codecov-action@v1.4.1
       with:
         file: ./coverage.xml
         fail_ci_if_error: false

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 5.6.4
+    rev: 5.8.0
   - hooks:
       - id: flake8
         language_version: python3
@@ -18,13 +18,13 @@ repos:
           - flake8-debugger
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 3.9.0
   - hooks:
       -  id: mypy
          language_version: python3
          files: ^dvc/
     repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.790'
+    rev: 'v0.812'
   - repo: local
     hooks:
     - id: pylint

MANIFEST.in

@@ -1,2 +1,3 @@
 include fastentrypoints.py
 include LICENSE
+include test_requirements.txt

README.rst

@@ -129,7 +129,11 @@ Conda (Anaconda)
 
 .. code-block:: bash
 
-   conda install -c conda-forge dvc
+   conda install -c conda-forge mamba # installs much faster than conda
+   mamba install -c conda-forge dvc
+
+Depending on the remote storage type you plan to use to keep and share your data, you might need to
+install optional dependencies: `dvc-s3`, `dvc-azure`, `dvc-gdrive`, `dvc-gs`, `dvc-oss`, `dvc-ssh`.
 
 pip (PyPI)
 ----------

debian/changelog

@@ -1,5 +0,0 @@
-dvc (0.75.b1) experimental; urgency=low
-
-  * First native Debian package version.
-
- -- Dmitry Petrov <[email protected]>  Sat, 07 Dec 2019 09:49:28 +0300

debian/compat

@@ -1 +0,0 @@
-10

debian/control

@@ -1,33 +0,0 @@
-Source: dvc
-Maintainer: Dmitry Petrov <[email protected]>
-Section: python
-Priority: optional
-Build-Depends: dh-python, python3-setuptools, python3-all, debhelper (>= 9)
-Standards-Version: 4.0.0
-Homepage: http://dvc.org
-Vcs-Git: https://github.com/iterative/dvc.git
-Vcs-browser: https://github.com/iterative/dvc
-
-Package: dvc
-Architecture: all
-Depends: ${misc:Depends}, ${python3:Depends}, python3-flufl.lock (>=3.2)
-Recommends: python3-arrow (=0.14.0),
-            python3-paramiko (>=2.5.0),
-            python3-gssapi (>= 2.5.0),
-            python3-boto3 (>=1.9.86),
-            python3-pydrive (=1.3.1),
-            python3-azure-storage-blob (=12.1.0),
-            python3-oss2 (=2.6.1),
-            python3-google-cloud-storage (=1.19.0)
-Description: Git for data scientists - manage your code and data together
- Data Version Control or DVC is an open-source tool for data science and machine learning projects.
- .
- Key features:
- .
- 1. Simple command line Git-like experience. Does not require installing and maintaining any databases. Does not depend on any proprietary online services.
- .
- 2. Management and versioning of datasets and machine learning models. Data is saved in S3, Google cloud, Azure, Alibaba cloud, SSH server, HDFS, or even local HDD RAID.
- .
- 3. Makes projects reproducible and shareable; helping to answer questions about how a model was built.
- .
- 4. Helps manage experiments with Git tags/branches and metrics tracking.

debian/copyright

@@ -1,11 +0,0 @@
-Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: DVC
-Upstream-Contact: Dmitry Petrov <[email protected]>
-Source: https://github.com/iterative/dvc
-
-Files: *
-Copyright: Dmitry Petrov <[email protected]>
-License: Apache 2.0
- On Debian systems, the full text of the Apache 2.0
- License can be found in the file
- `/usr/share/common-licenses/Apache-2.0`.

debian/rules

@@ -1,6 +0,0 @@
-#!/usr/bin/make -f
-
-export PYBUILD_NAME=dvc
-%:
-	dh $@ --with python3 --buildsystem=pybuild
-

debian/source/format

@@ -1 +0,0 @@
-3.0 (native)

debian/watch

@@ -1,4 +0,0 @@
-# please also check http://pypi.debian.net/dvc/watch
-version=3
-opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
-http://pypi.debian.net/dvc/dvc-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))

dvc/command/data_sync.py

@@ -156,6 +156,7 @@ def add_parser(subparsers, _parent_parser):
         help="Fetch cache for all tags.",
     )
     pull_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -224,6 +225,7 @@ def add_parser(subparsers, _parent_parser):
         help="Push cache for all tags.",
     )
     push_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -290,6 +292,7 @@ def add_parser(subparsers, _parent_parser):
         help="Fetch cache for all tags.",
     )
     fetch_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -370,6 +373,7 @@ def add_parser(subparsers, _parent_parser):
         "for all tags.",
     )
     status_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,

dvc/command/diff.py

@@ -18,9 +18,9 @@ def _digest(checksum):
 
 
 def _show_md(diff, show_hash=False, hide_missing=False):
-    from dvc.utils.diff import table
+    from dvc.ui import ui
 
-    header = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
+    headers = ["Status", "Hash", "Path"] if show_hash else ["Status", "Path"]
     rows = []
     statuses = ["added", "deleted", "renamed", "modified"]
     if not hide_missing:
@@ -39,7 +39,7 @@ def _show_md(diff, show_hash=False, hide_missing=False):
             else:
                 rows.append([status, path])
 
-    return table(header, rows, True)
+    ui.table(rows, headers=headers, markdown=True)
 
 
 class CmdDiff(CmdBase):

dvc/command/experiments.py

@@ -1,10 +1,11 @@
 import argparse
 import logging
 from collections import Counter, OrderedDict, defaultdict
-from collections.abc import Mapping
 from datetime import date, datetime
-from itertools import groupby
-from typing import Dict, Iterable, Optional
+from fnmatch import fnmatch
+from typing import TYPE_CHECKING, Dict, Iterable, Optional
+
+from funcy import constantly, iffy, lmap
 
 import dvc.prompt as prompt
 from dvc.command import completion
@@ -15,10 +16,15 @@ from dvc.command.repro import add_arguments as add_repro_arguments
 from dvc.exceptions import DvcException, InvalidArgumentError
 from dvc.utils.flatten import flatten
 
+if TYPE_CHECKING:
+    from dvc.compare import TabularData
+
+
 logger = logging.getLogger(__name__)
 
 
 SHOW_MAX_WIDTH = 1024
+FILL_VALUE = "-"
 
 
 def _filter_name(names, label, filter_strs):
@@ -27,32 +33,23 @@ def _filter_name(names, label, filter_strs):
 
     for filter_s in filter_strs:
         path, _, name = filter_s.rpartition(":")
-        path_filters[path].append(tuple(name.split(".")))
+        path_filters[path].append(name)
 
     for path, filters in path_filters.items():
         if path:
             match_paths = [path]
         else:
             match_paths = names.keys()
-        for length, groups in groupby(filters, len):
-            for group in groups:
-                for match_path in match_paths:
-                    possible_names = [
-                        tuple(name.split(".")) for name in names[match_path]
-                    ]
-                    matches = [
-                        name
-                        for name in possible_names
-                        if name[:length] == group
-                    ]
-                    if not matches:
-                        name = ".".join(group)
-                        raise InvalidArgumentError(
-                            f"'{name}' does not match any known {label}"
-                        )
-                    ret[match_path].update(
-                        {".".join(match): None for match in matches}
+        for match_path in match_paths:
+            for f in filters:
+                matches = [
+                    name for name in names[match_path] if fnmatch(name, f)
+                ]
+                if not matches:
+                    raise InvalidArgumentError(
+                        f"'{f}' does not match any known {label}"
                     )
+                ret[match_path].update({match: None for match in matches})
 
     return ret
 
@@ -103,7 +100,6 @@ def _collect_names(all_experiments, **kwargs):
         for exp in experiments.values():
             _update_names(metric_names, exp.get("metrics", {}).items())
             _update_names(param_names, exp.get("params", {}).items())
-
     metric_names = _filter_names(
         metric_names,
         "metrics",
@@ -111,7 +107,7 @@ def _collect_names(all_experiments, **kwargs):
         kwargs.get("exclude_metrics"),
     )
     param_names = _filter_names(
-        (param_names),
+        param_names,
         "params",
         kwargs.get("include_params"),
         kwargs.get("exclude_params"),
@@ -126,7 +122,6 @@ def _collect_rows(
     metric_names,
     param_names,
     precision=DEFAULT_PRECISION,
-    no_timestamp=False,
     sort_by=None,
     sort_order=None,
 ):
@@ -143,56 +138,59 @@ def _collect_rows(
 
     new_checkpoint = True
     for i, (rev, exp) in enumerate(experiments.items()):
-        row = []
-        style = None
-        queued = "*" if exp.get("queued", False) else ""
+        queued = str(exp.get("queued") or "")
+        is_baseline = rev == "baseline"
+
+        if is_baseline:
+            name_rev = base_rev[:7] if Git.is_sha(base_rev) else base_rev
+        else:
+            name_rev = rev[:7]
+
+        exp_name = exp.get("name", "")
+        if is_baseline:
+            name = exp_name or name_rev
+        elif exp_name:
+            name = f"{rev[:7]} [[bold]{exp['name']}[/]]"
+        else:
+            name = name_rev
 
         tip = exp.get("checkpoint_tip")
+
+        parent_rev = exp.get("checkpoint_parent", "")
+        parent_exp = experiments.get(parent_rev, {})
+        parent_tip = parent_exp.get("checkpoint_tip")
+
         parent = ""
-        if rev == "baseline":
-            if Git.is_sha(base_rev):
-                name_rev = base_rev[:7]
+        if is_baseline:
+            tree = ""
+        elif tip:
+            if tip == parent_tip:
+                tree = "│ ╓" if new_checkpoint else "│ ╟"
+            elif parent_rev == base_rev:
+                tree = "├─╨"
             else:
-                name_rev = base_rev
-            name = exp.get("name", name_rev)
-            row.append(f"{name}")
-            style = "bold"
+                tree = "│ ╟"
+                parent = parent_rev[:7]
         else:
-            if tip:
-                parent_rev = exp.get("checkpoint_parent", "")
-                parent_exp = experiments.get(parent_rev, {})
-                parent_tip = parent_exp.get("checkpoint_tip")
-                if tip == parent_tip:
-                    if new_checkpoint:
-                        tree = "│ ╓"
-                    else:
-                        tree = "│ ╟"
-                    new_checkpoint = False
-                else:
-                    if parent_rev == base_rev:
-                        tree = "├─╨"
-                    else:
-                        tree = "│ ╟"
-                        parent = f" ({parent_rev[:7]})"
-                    new_checkpoint = True
-            else:
-                if i < len(experiments) - 1:
-                    tree = "├──"
-                else:
-                    tree = "└──"
-                new_checkpoint = True
-            name = exp.get("name", rev[:7])
-            row.append(f"{tree} {queued}{name}{parent}")
+            tree = "├──" if i < len(experiments) - 1 else "└──"
 
-        if not no_timestamp:
-            row.append(_format_time(exp.get("timestamp")))
+        if not is_baseline:
+            new_checkpoint = not (tip and tip == parent_tip)
 
+        row = [
+            name,
+            queued,
+            tree,
+            _format_time(exp.get("timestamp")),
+            rev == "baseline",
+            parent,
+        ]
         _extend_row(
             row, metric_names, exp.get("metrics", {}).items(), precision
         )
         _extend_row(row, param_names, exp.get("params", {}).items(), precision)
 
-        yield row, style
+        yield row
 
 
 def _sort_column(sort_by, metric_names, param_names):
@@ -233,7 +231,7 @@ def _sort_exp(experiments, sort_path, sort_name, typ, reverse):
             return _sort((tip, experiments[tip]))
         data = exp.get(typ, {}).get(sort_path, {})
         val = flatten(data).get(sort_name)
-        return (val is None, val)
+        return val is None, val
 
     ret = OrderedDict()
     if "baseline" in experiments:
@@ -245,7 +243,7 @@ def _sort_exp(experiments, sort_path, sort_name, typ, reverse):
 
 def _format_time(timestamp):
     if timestamp is None:
-        return "-"
+        return FILL_VALUE
     if timestamp.date() == date.today():
         fmt = "%I:%M %p"
     else:
@@ -253,42 +251,23 @@ def _format_time(timestamp):
     return timestamp.strftime(fmt)
 
 
-def _format_field(val, precision=DEFAULT_PRECISION):
-    if isinstance(val, float):
-        fmt = f"{{:.{precision}g}}"
-        return fmt.format(val)
-    if isinstance(val, Mapping):
-        return {k: _format_field(v) for k, v in val.items()}
-    if isinstance(val, list):
-        return [_format_field(x) for x in val]
-    return str(val)
-
-
 def _extend_row(row, names, items, precision):
     from rich.text import Text
 
+    from dvc.compare import _format_field, with_value
+
     if not items:
-        row.extend(["-"] * len(names))
+        row.extend(FILL_VALUE for keys in names.values() for _ in keys)
         return
 
     for fname, item in items:
-        if isinstance(item, dict):
-            item = flatten(item)
-        else:
-            item = {fname: item}
+        item = flatten(item) if isinstance(item, dict) else {fname: item}
         for name in names[fname]:
-            if name in item:
-                value = item[name]
-                if value is None:
-                    text = "-"
-                else:
-                    # wrap field data in rich.Text, otherwise rich may
-                    # interpret unescaped braces from list/dict types as rich
-                    # markup tags
-                    text = Text(str(_format_field(value, precision)))
-                row.append(text)
-            else:
-                row.append("-")
+            value = with_value(item.get(name), FILL_VALUE)
+            # wrap field data in rich.Text, otherwise rich may
+            # interpret unescaped braces from list/dict types as rich
+            # markup tags
+            row.append(Text(str(_format_field(value, precision))))
 
 
 def _parse_filter_list(param_list):
@@ -302,9 +281,62 @@ def _parse_filter_list(param_list):
     return ret
 
 
-def _experiments_table(all_experiments, **kwargs):
-    from dvc.utils.table import Table
+def experiments_table(
+    all_experiments,
+    metric_headers,
+    metric_names,
+    param_headers,
+    param_names,
+    sort_by=None,
+    sort_order=None,
+    precision=DEFAULT_PRECISION,
+) -> "TabularData":
+    from funcy import lconcat
 
+    from dvc.compare import TabularData
+
+    headers = [
+        "Experiment",
+        "queued",
+        "ident_guide",
+        "Created",
+        "is_baseline",
+        "parent",
+    ]
+    td = TabularData(
+        lconcat(headers, metric_headers, param_headers), fill_value=FILL_VALUE
+    )
+    for base_rev, experiments in all_experiments.items():
+        rows = _collect_rows(
+            base_rev,
+            experiments,
+            metric_names,
+            param_names,
+            sort_by=sort_by,
+            sort_order=sort_order,
+            precision=precision,
+        )
+        td.extend(rows)
+
+    return td
+
+
+def prepare_exp_id(kwargs):
+    exp_name = kwargs["Experiment"]
+    ident = kwargs.get("ident_guide")
+    queued = kwargs.get("queued")
+    parent = kwargs.get("parent")
+    return "{}{}{}{}".format(
+        f"{ident} " if ident else "",
+        "*" if queued else "",
+        exp_name,
+        f" ({parent})" if parent else "",
+    )
+
+
+def show_experiments(
+    all_experiments, pager=True, no_timestamp=False, **kwargs
+):
     include_metrics = _parse_filter_list(kwargs.pop("include_metrics", []))
     exclude_metrics = _parse_filter_list(kwargs.pop("exclude_metrics", []))
     include_params = _parse_filter_list(kwargs.pop("include_params", []))
@@ -317,44 +349,68 @@ def _experiments_table(all_experiments, **kwargs):
         include_params=include_params,
         exclude_params=exclude_params,
     )
+    metric_headers = _normalize_headers(metric_names)
+    param_headers = _normalize_headers(param_names)
 
-    table = Table()
-    table.add_column(
-        "Experiment", no_wrap=True, header_style="black on grey93"
-    )
-    if not kwargs.get("no_timestamp", False):
-        table.add_column("Created", header_style="black on grey93")
-    _add_data_columns(
-        table,
+    td = experiments_table(
+        all_experiments,
+        metric_headers,
         metric_names,
-        justify="right",
-        no_wrap=True,
-        header_style="black on cornsilk1",
-    )
-    _add_data_columns(
-        table, param_names, justify="left", header_style="black on light_cyan1"
+        param_headers,
+        param_names,
+        kwargs.get("sort_by"),
+        kwargs.get("sort_order"),
+        kwargs.get("precision"),
     )
 
-    for base_rev, experiments in all_experiments.items():
-        for row, _, in _collect_rows(
-            base_rev, experiments, metric_names, param_names, **kwargs,
-        ):
-            table.add_row(*row)
+    if no_timestamp:
+        td.drop("Created")
 
-    return table
+    baseline_styler = iffy(constantly({"style": "bold"}), default={})
+    row_styles = lmap(baseline_styler, td.column("is_baseline"))
+    td.drop("is_baseline")
+
+    merge_headers = ["Experiment", "queued", "ident_guide", "parent"]
+    td.column("Experiment")[:] = map(prepare_exp_id, td.as_dict(merge_headers))
+    td.drop(*merge_headers[1:])
+
+    headers = {"metrics": metric_headers, "params": param_headers}
+    styles = {
+        "Experiment": {"no_wrap": True, "header_style": "black on grey93"},
+        "Created": {"header_style": "black on grey93"},
+    }
+    header_bg_colors = {"metrics": "cornsilk1", "params": "light_cyan1"}
+    styles.update(
+        {
+            header: {
+                "justify": "left" if typ == "metrics" else "params",
+                "header_style": f"black on {header_bg_colors[typ]}",
+                "collapse": idx != 0,
+                "no_wrap": typ == "metrics",
+            }
+            for typ, hs in headers.items()
+            for idx, header in enumerate(hs)
+        }
+    )
+
+    td.render(
+        pager=pager,
+        borders=True,
+        rich_table=True,
+        header_styles=styles,
+        row_styles=row_styles,
+    )
 
 
-def _add_data_columns(table, names, **kwargs):
+def _normalize_headers(names):
     count = Counter(
         name for path in names for name in names[path] for path in names
     )
-    first = True
-    for path in names:
-        for name in names[path]:
-            col_name = name if count[name] == 1 else f"{path}:{name}"
-            kwargs["collapse"] = False if first else True
-            table.add_column(col_name, **kwargs)
-            first = False
+    return [
+        name if count[name] == 1 else f"{path}:{name}"
+        for path in names
+        for name in names[path]
+    ]
 
 
 def _format_json(item):
@@ -365,8 +421,6 @@ def _format_json(item):
 
 class CmdExperimentsShow(CmdBase):
     def run(self):
-        from rich.console import Console
-
         try:
             all_experiments = self.repo.experiments.show(
                 all_branches=self.args.all_branches,
@@ -374,20 +428,18 @@ class CmdExperimentsShow(CmdBase):
                 all_commits=self.args.all_commits,
                 sha_only=self.args.sha,
                 num=self.args.num,
+                param_deps=self.args.param_deps,
             )
+        except DvcException:
+            logger.exception("failed to show experiments")
+            return 1
 
-            if self.args.show_json:
-                import json
+        if self.args.show_json:
+            import json
 
-                logger.info(json.dumps(all_experiments, default=_format_json))
-                return 0
-
-            if self.args.precision is None:
-                precision = DEFAULT_PRECISION
-            else:
-                precision = self.args.precision
-
-            table = _experiments_table(
+            logger.info(json.dumps(all_experiments, default=_format_json))
+        else:
+            show_experiments(
                 all_experiments,
                 include_metrics=self.args.include_metrics,
                 exclude_metrics=self.args.exclude_metrics,
@@ -396,29 +448,9 @@ class CmdExperimentsShow(CmdBase):
                 no_timestamp=self.args.no_timestamp,
                 sort_by=self.args.sort_by,
                 sort_order=self.args.sort_order,
-                precision=precision,
+                precision=self.args.precision or DEFAULT_PRECISION,
+                pager=not self.args.no_pager,
             )
-
-            console = Console()
-            if self.args.no_pager:
-                console.print(table)
-            else:
-                from dvc.utils.pager import DvcPager
-
-                # NOTE: rich does not have native support for unlimited width
-                # via pager. we override rich table compression by setting
-                # console width to the full width of the table
-                measurement = table.__rich_measure__(console, SHOW_MAX_WIDTH)
-                console._width = (  # pylint: disable=protected-access
-                    measurement.maximum
-                )
-                with console.pager(pager=DvcPager(), styles=True):
-                    console.print(table)
-
-        except DvcException:
-            logger.exception("failed to show experiments")
-            return 1
-
         return 0
 
 
@@ -432,43 +464,6 @@ class CmdExperimentsApply(CmdBase):
         return 0
 
 
-def _show_diff(
-    diff,
-    title="",
-    markdown=False,
-    no_path=False,
-    old=False,
-    precision=DEFAULT_PRECISION,
-):
-    from dvc.utils.diff import table
-
-    rows = []
-    for fname, diff_ in diff.items():
-        sorted_diff = OrderedDict(sorted(diff_.items()))
-        for item, change in sorted_diff.items():
-            row = [] if no_path else [fname]
-            row.append(item)
-            if old:
-                row.append(_format_field(change.get("old"), precision))
-            row.append(_format_field(change["new"], precision))
-            row.append(
-                _format_field(
-                    change.get("diff", "diff not supported"), precision
-                )
-            )
-            rows.append(row)
-
-    header = [] if no_path else ["Path"]
-    header.append(title)
-    if old:
-        header.extend(["Old", "New"])
-    else:
-        header.append("Value")
-    header.append("Change")
-
-    return table(header, rows, markdown)
-
-
 class CmdExperimentsDiff(CmdBase):
     def run(self):
 
@@ -477,47 +472,56 @@ class CmdExperimentsDiff(CmdBase):
                 a_rev=self.args.a_rev,
                 b_rev=self.args.b_rev,
                 all=self.args.all,
+                param_deps=self.args.param_deps,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(diff))
-            else:
-                if self.args.precision is None:
-                    precision = DEFAULT_PRECISION
-                else:
-                    precision = self.args.precision
-
-                diffs = [("metrics", "Metric"), ("params", "Param")]
-                for key, title in diffs:
-                    table = _show_diff(
-                        diff[key],
-                        title=title,
-                        markdown=self.args.show_md,
-                        no_path=self.args.no_path,
-                        old=self.args.old,
-                        precision=precision,
-                    )
-                    if table:
-                        logger.info(table)
-                        logger.info("")
-
         except DvcException:
             logger.exception("failed to show experiments diff")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            logger.info(json.dumps(diff))
+        else:
+            from dvc.compare import show_diff
+
+            precision = self.args.precision or DEFAULT_PRECISION
+            diffs = [("metrics", "Metric"), ("params", "Param")]
+            for idx, (key, title) in enumerate(diffs):
+                if idx:
+                    from dvc.ui import ui
+
+                    # we are printing tables even in `--quiet` mode
+                    # so we should also be printing the "table" separator
+                    ui.write(force=True)
+
+                show_diff(
+                    diff[key],
+                    title=title,
+                    markdown=self.args.show_md,
+                    no_path=self.args.no_path,
+                    old=self.args.old,
+                    on_empty_diff="diff not supported",
+                    precision=precision if key == "metrics" else None,
+                )
+
         return 0
 
 
 class CmdExperimentsRun(CmdRepro):
     def run(self):
-        from dvc.command.metrics import _show_metrics
+        from dvc.compare import show_metrics
 
-        if self.args.reset and self.args.checkpoint_resume:
-            raise InvalidArgumentError(
-                "--reset and --rev are mutually exclusive."
-            )
+        if self.args.checkpoint_resume:
+            if self.args.reset:
+                raise InvalidArgumentError(
+                    "--reset and --rev are mutually exclusive."
+                )
+            if not (self.args.queue or self.args.tmp_dir):
+                raise InvalidArgumentError(
+                    "--rev can only be used in conjunction with "
+                    "--queue or --temp."
+                )
 
         if self.args.reset:
             logger.info("Any existing checkpoints will be reset and re-run.")
@@ -537,7 +541,7 @@ class CmdExperimentsRun(CmdRepro):
         if self.args.metrics and results:
             metrics = self.repo.metrics.show(revs=list(results))
             metrics.pop("workspace", None)
-            logger.info(_show_metrics(metrics))
+            logger.info(show_metrics(metrics))
 
         return 0
 
@@ -788,6 +792,12 @@ def add_parser(subparsers, parent_parser):
         help="Exclude the specified params from output table.",
         metavar="<params_list>",
     )
+    experiments_show_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     experiments_show_parser.add_argument(
         "--sort-by",
         help="Sort related experiments by the specified metric or param.",
@@ -873,6 +883,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show unchanged metrics/params as well.",
     )
+    experiments_diff_parser.add_argument(
+        "--param-deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     experiments_diff_parser.add_argument(
         "--show-json",
         action="store_true",
@@ -925,9 +941,8 @@ def add_parser(subparsers, parent_parser):
         type=str,
         dest="checkpoint_resume",
         help=(
-            "Continue the specified checkpoint experiment. "
-            "(Only required for explicitly resuming checkpoints in queued "
-            "or temp dir runs.)"
+            "Continue the specified checkpoint experiment. Can only be used "
+            "in conjunction with --queue or --temp."
         ),
         metavar="<experiment_rev>",
     ).complete = completion.EXPERIMENT
@@ -972,6 +987,7 @@ def add_parser(subparsers, parent_parser):
         help="Keep experiments derived from all Git tags.",
     )
     experiments_gc_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,
@@ -1220,6 +1236,7 @@ def _add_run_common(parser):
         "-j",
         "--jobs",
         type=int,
+        default=1,
         help="Run the specified number of experiments at a time in parallel.",
         metavar="<number>",
     )

dvc/command/gc.py

@@ -97,6 +97,7 @@ def add_parser(subparsers, parent_parser):
         help="Keep data files for all Git tags.",
     )
     gc_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,

dvc/command/live.py

@@ -4,7 +4,6 @@ import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, fix_subparsers
-from dvc.utils.html import write
 
 logger = logging.getLogger(__name__)
 
@@ -16,7 +15,7 @@ class CmdLive(CmdBase):
         metrics, plots = self.repo.live.show(target=target, revs=revs)
 
         html_path = self.args.target + ".html"
-        write(html_path, plots, metrics)
+        self.repo.plots.write_html(html_path, plots, metrics)
 
         logger.info(f"\nfile://{os.path.abspath(html_path)}")
 

dvc/command/ls/__init__.py

@@ -34,7 +34,11 @@ class CmdList(CmdBaseNoRepo):
                 recursive=self.args.recursive,
                 dvc_only=self.args.dvc_only,
             )
-            if entries:
+            if self.args.show_json:
+                import json
+
+                logger.info(json.dumps(entries))
+            elif entries:
                 entries = _prettify(entries, sys.stdout.isatty())
                 logger.info("\n".join(entries))
             return 0
@@ -65,6 +69,9 @@ def add_parser(subparsers, parent_parser):
     list_parser.add_argument(
         "--dvc-only", action="store_true", help="Show only DVC outputs."
     )
+    list_parser.add_argument(
+        "--show-json", action="store_true", help="Show output in JSON format."
+    )
     list_parser.add_argument(
         "--rev",
         nargs="?",

dvc/command/metrics.py

@@ -11,64 +11,6 @@ logger = logging.getLogger(__name__)
 DEFAULT_PRECISION = 5
 
 
-def _show_metrics(
-    metrics,
-    markdown=False,
-    all_branches=False,
-    all_tags=False,
-    all_commits=False,
-    precision=None,
-):
-    from dvc.utils.diff import format_dict, table
-    from dvc.utils.flatten import flatten
-
-    # When `metrics` contains a `None` key, it means that some files
-    # specified as `targets` in `repo.metrics.show` didn't contain any metrics.
-    missing = metrics.pop(None, None)
-    with_rev = any([all_branches, all_tags, all_commits])
-    header_set = set()
-    rows = []
-
-    if precision is None:
-        precision = DEFAULT_PRECISION
-
-    def _round(val):
-        if isinstance(val, float):
-            return round(val, precision)
-        return val
-
-    for _branch, val in metrics.items():
-        for _fname, metric in val.items():
-            if not isinstance(metric, dict):
-                header_set.add("")
-                continue
-            for key, _val in flatten(format_dict(metric)).items():
-                header_set.add(key)
-    header = sorted(header_set)
-    for branch, val in metrics.items():
-        for fname, metric in val.items():
-            row = []
-            if with_rev:
-                row.append(branch)
-            row.append(fname)
-            if not isinstance(metric, dict):
-                row.append(str(metric))
-                rows.append(row)
-                continue
-            flattened_val = flatten(format_dict(metric))
-
-            for i in header:
-                row.append(_round(flattened_val.get(i)))
-            rows.append(row)
-    header.insert(0, "Path")
-    if with_rev:
-        header.insert(0, "Revision")
-
-    if missing:
-        raise BadMetricError(missing)
-    return table(header, rows, markdown)
-
-
 class CmdMetricsBase(CmdBase):
     UNINITIALIZED = True
 
@@ -83,61 +25,37 @@ class CmdMetricsShow(CmdMetricsBase):
                 all_commits=self.args.all_commits,
                 recursive=self.args.recursive,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(metrics))
-            else:
-                table = _show_metrics(
-                    metrics,
-                    self.args.show_md,
-                    self.args.all_branches,
-                    self.args.all_tags,
-                    self.args.all_commits,
-                )
-                if table:
-                    logger.info(table)
         except DvcException:
             logger.exception("")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            logger.info(json.dumps(metrics))
+        else:
+            from dvc.compare import show_metrics
+
+            # When `metrics` contains a `None` key, it means that some files
+            # specified as `targets` in `repo.metrics.show` didn't contain any
+            # metrics.
+            missing = metrics.pop(None, None)
+            if missing:
+                raise BadMetricError(missing)
+
+            show_metrics(
+                metrics,
+                markdown=self.args.show_md,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                all_commits=self.args.all_commits,
+                precision=self.args.precision or DEFAULT_PRECISION,
+                round_digits=True,
+            )
+
         return 0
 
 
-def _show_diff(diff, markdown=False, no_path=False, precision=None):
-    from collections import OrderedDict
-
-    from dvc.utils.diff import table
-
-    if precision is None:
-        precision = DEFAULT_PRECISION
-
-    def _round(val):
-        if isinstance(val, float):
-            return round(val, precision)
-
-        return val
-
-    rows = []
-    for fname, mdiff in diff.items():
-        sorted_mdiff = OrderedDict(sorted(mdiff.items()))
-        for metric, change in sorted_mdiff.items():
-            row = [] if no_path else [fname]
-            row.append(metric)
-            row.append(_round(change.get("old")))
-            row.append(_round(change["new"]))
-            row.append(_round(change.get("diff")))
-            rows.append(row)
-
-    header = [] if no_path else ["Path"]
-    header.append("Metric")
-    header.extend(["Old", "New"])
-    header.append("Change")
-
-    return table(header, rows, markdown)
-
-
 class CmdMetricsDiff(CmdMetricsBase):
     def run(self):
         try:
@@ -148,25 +66,26 @@ class CmdMetricsDiff(CmdMetricsBase):
                 recursive=self.args.recursive,
                 all=self.args.all,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(diff))
-            else:
-                table = _show_diff(
-                    diff,
-                    self.args.show_md,
-                    self.args.no_path,
-                    precision=self.args.precision,
-                )
-                if table:
-                    logger.info(table)
-
         except DvcException:
             logger.exception("failed to show metrics diff")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            logger.info(json.dumps(diff))
+        else:
+            from dvc.compare import show_diff
+
+            show_diff(
+                diff,
+                title="Metric",
+                markdown=self.args.show_md,
+                no_path=self.args.no_path,
+                precision=self.args.precision or DEFAULT_PRECISION,
+                round_digits=True,
+            )
+
         return 0
 
 
@@ -219,6 +138,7 @@ def add_parser(subparsers, parent_parser):
         help="Show metrics for all tags.",
     )
     metrics_show_parser.add_argument(
+        "-A",
         "--all-commits",
         action="store_true",
         default=False,

dvc/command/params.py

@@ -1,6 +1,5 @@
 import argparse
 import logging
-from collections import OrderedDict
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
@@ -9,27 +8,6 @@ from dvc.exceptions import DvcException
 logger = logging.getLogger(__name__)
 
 
-def _show_diff(diff, markdown=False, no_path=False):
-    from dvc.utils.diff import table
-
-    rows = []
-    for fname, pdiff in diff.items():
-        sorted_pdiff = OrderedDict(sorted(pdiff.items()))
-        for param, change in sorted_pdiff.items():
-            row = [] if no_path else [fname]
-            row.append(param)
-            row.append(change["old"])
-            row.append(change["new"])
-            rows.append(row)
-
-    header = [] if no_path else ["Path"]
-    header.append("Param")
-    header.append("Old")
-    header.append("New")
-
-    return table(header, rows, markdown)
-
-
 class CmdParamsDiff(CmdBase):
     UNINITIALIZED = True
 
@@ -40,21 +18,27 @@ class CmdParamsDiff(CmdBase):
                 b_rev=self.args.b_rev,
                 targets=self.args.targets,
                 all=self.args.all,
+                deps=self.args.deps,
             )
-
-            if self.args.show_json:
-                import json
-
-                logger.info(json.dumps(diff))
-            else:
-                table = _show_diff(diff, self.args.show_md, self.args.no_path)
-                if table:
-                    logger.info(table)
-
         except DvcException:
             logger.exception("failed to show params diff")
             return 1
 
+        if self.args.show_json:
+            import json
+
+            logger.info(json.dumps(diff))
+        else:
+            from dvc.compare import show_diff
+
+            show_diff(
+                diff,
+                title="Param",
+                markdown=self.args.show_md,
+                no_path=self.args.no_path,
+                show_changes=False,
+            )
+
         return 0
 
 
@@ -111,6 +95,12 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Show unchanged params as well.",
     )
+    params_diff_parser.add_argument(
+        "--deps",
+        action="store_true",
+        default=False,
+        help="Show only params that are stage dependencies.",
+    )
     params_diff_parser.add_argument(
         "--show-json",
         action="store_true",

dvc/command/plots.py

@@ -1,12 +1,10 @@
 import argparse
 import logging
-import os
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import DvcException
 from dvc.utils import format_link
-from dvc.utils.html import write
 
 logger = logging.getLogger(__name__)
 
@@ -23,6 +21,8 @@ class CmdPlots(CmdBase):
         return {k: v for k, v in props.items() if v is not None}
 
     def run(self):
+        from pathlib import Path
+
         if self.args.show_vega:
             if not self.args.targets:
                 logger.error("please specify a target for `--show-vega`")
@@ -41,20 +41,23 @@ class CmdPlots(CmdBase):
                 logger.info(plots[target])
                 return 0
 
-            path = self.args.out or "plots.html"
-            path = os.path.join(os.getcwd(), path)
+            rel: str = self.args.out or "plots.html"
+            path = (Path.cwd() / rel).resolve()
+            self.repo.plots.write_html(
+                path, plots=plots, html_template_path=self.args.html_template
+            )
 
-            write(path, plots)
-            url = f"file://{path}"
         except DvcException:
             logger.exception("")
             return 1
 
+        assert path.is_absolute()  # as_uri throws ValueError if not absolute
+        url = path.as_uri()
         logger.info(url)
         if self.args.open:
             import webbrowser
 
-            opened = webbrowser.open(url)
+            opened = webbrowser.open(rel)
             if not opened:
                 logger.error("Failed to open. Please try opening it manually.")
                 return 1
@@ -92,7 +95,7 @@ class CmdPlotsModify(CmdPlots):
 def add_parser(subparsers, parent_parser):
     PLOTS_HELP = (
         "Commands to visualize and compare plot metrics in structured files "
-        "(JSON, YAML, CSV, TSV)"
+        "(JSON, YAML, CSV, TSV)."
     )
 
     plots_parser = subparsers.add_parser(
@@ -242,3 +245,9 @@ def _add_output_arguments(parser):
         default=False,
         help="Open plot file directly in the browser.",
     )
+    parser.add_argument(
+        "--html-template",
+        default=None,
+        help="Custom HTML template for VEGA visualization.",
+        metavar="<path>",
+    )

dvc/command/repro.py

@@ -3,7 +3,6 @@ import logging
 
 from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
-from dvc.command.metrics import _show_metrics
 from dvc.command.status import CmdDataStatus
 
 logger = logging.getLogger(__name__)
@@ -20,8 +19,10 @@ class CmdRepro(CmdBase):
             )
 
         if self.args.metrics:
+            from dvc.compare import show_metrics
+
             metrics = self.repo.metrics.show()
-            logger.info(_show_metrics(metrics))
+            show_metrics(metrics)
 
         return 0
 

dvc/command/stage.py

@@ -84,7 +84,7 @@ class CmdStageList(CmdBase):
         return dict.fromkeys(collected).keys()
 
     def run(self):
-        from dvc.utils.diff import table
+        from dvc.ui import ui
 
         def log_error(relpath: str, exc: Exception):
             if self.args.fail:
@@ -98,8 +98,7 @@ class CmdStageList(CmdBase):
         names_only = self.args.names_only
 
         data = prepare_stages_data(stages, description=not names_only)
-        if data:
-            print(table(header=(), rows=data.items()))
+        ui.table(data.items())
 
         return 0
 

dvc/command/status.py

@@ -44,19 +44,27 @@ class CmdDataStatus(CmdDataBase):
                 self._show(value, indent + 1)
 
     def run(self):
+        from dvc.repo import lock_repo
+        from dvc.ui import ui
+
         indent = 1 if self.args.cloud else 0
-        try:
-            st = self.repo.status(
-                targets=self.args.targets,
-                jobs=self.args.jobs,
-                cloud=self.args.cloud,
-                remote=self.args.remote,
-                all_branches=self.args.all_branches,
-                all_tags=self.args.all_tags,
-                all_commits=self.args.all_commits,
-                with_deps=self.args.with_deps,
-                recursive=self.args.recursive,
-            )
+
+        with lock_repo(self.repo):
+            try:
+                st = self.repo.status(
+                    targets=self.args.targets,
+                    jobs=self.args.jobs,
+                    cloud=self.args.cloud,
+                    remote=self.args.remote,
+                    all_branches=self.args.all_branches,
+                    all_tags=self.args.all_tags,
+                    all_commits=self.args.all_commits,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("")
+                return 1
 
             if self.args.quiet:
                 return bool(st)
@@ -65,19 +73,21 @@ class CmdDataStatus(CmdDataBase):
                 import json
 
                 logger.info(json.dumps(st))
-            elif st:
+                return 0
+
+            if st:
                 self._show(st, indent)
-            elif not self.repo.stages:
-                logger.info(self.EMPTY_PROJECT_MSG)
+                return 0
+
+            # additional hints for the user
+            if not self.repo.stages:
+                ui.write(self.EMPTY_PROJECT_MSG)
             elif self.args.cloud or self.args.remote:
                 remote = self.args.remote or self.repo.config["core"].get(
                     "remote"
                 )
-                logger.info(self.IN_SYNC_MSG.format(remote=remote))
+                ui.write(self.IN_SYNC_MSG.format(remote=remote))
             else:
-                logger.info(self.UP_TO_DATE_MSG)
+                ui.write(self.UP_TO_DATE_MSG)
 
-        except DvcException:
-            logger.exception("")
-            return 1
         return 0

dvc/command/unprotect.py

@@ -23,7 +23,7 @@ class CmdUnprotect(CmdBase):
 def add_parser(subparsers, parent_parser):
     UNPROTECT_HELP = (
         "Unprotect tracked files or directories (when hardlinks or symlinks "
-        "have been enabled with `dvc config cache.type`)"
+        "have been enabled with `dvc config cache.type`)."
     )
     unprotect_parser = subparsers.add_parser(
         "unprotect",

dvc/compare.py

@@ -0,0 +1,321 @@
+from collections.abc import Mapping
+from itertools import chain, repeat, zip_longest
+from operator import itemgetter
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    ItemsView,
+    Iterable,
+    Iterator,
+    List,
+    MutableSequence,
+    Sequence,
+    Tuple,
+    Union,
+    overload,
+)
+
+from funcy import reraise
+
+if TYPE_CHECKING:
+    from dvc.ui.table import CellT
+
+
+class Column(List["CellT"]):
+    pass
+
+
+def with_value(value, default):
+    return default if value is None else value
+
+
+class TabularData(MutableSequence[Sequence["CellT"]]):
+    def __init__(self, columns: Sequence[str], fill_value: str = ""):
+        self._columns: Dict[str, Column] = {name: Column() for name in columns}
+        self._keys: List[str] = list(columns)
+        self._fill_value = fill_value
+
+    @property
+    def columns(self) -> List[Column]:
+        return list(map(self.column, self.keys()))
+
+    def column(self, name: str) -> Column:
+        return self._columns[name]
+
+    def items(self) -> ItemsView[str, Column]:
+        projection = {k: self.column(k) for k in self.keys()}
+        return projection.items()
+
+    def keys(self) -> List[str]:
+        return self._keys
+
+    def _iter_col_row(
+        self, row: Sequence["CellT"]
+    ) -> Iterator[Tuple["CellT", Column]]:
+        for val, col in zip_longest(row, self.columns):
+            if col is None:
+                break
+            yield with_value(val, self._fill_value), col
+
+    def append(self, value: Sequence["CellT"]) -> None:
+        for val, col in self._iter_col_row(value):
+            col.append(val)
+
+    def extend(self, values: Iterable[Sequence["CellT"]]) -> None:
+        for row in values:
+            self.append(row)
+
+    def insert(self, index: int, value: Sequence["CellT"]) -> None:
+        for val, col in self._iter_col_row(value):
+            col.insert(index, val)
+
+    def __iter__(self) -> Iterator[List["CellT"]]:
+        return map(list, zip(*self.columns))
+
+    def __getattr__(self, item: str) -> Column:
+        with reraise(KeyError, AttributeError):
+            return self.column(item)
+
+    def __getitem__(self, item: Union[int, slice]):
+        func = itemgetter(item)
+        it = map(func, self.columns)
+        if isinstance(item, slice):
+            it = map(list, zip(*it))
+        return list(it)
+
+    @overload
+    def __setitem__(self, item: int, value: Sequence["CellT"]) -> None:
+        ...
+
+    @overload
+    def __setitem__(
+        self, item: slice, value: Iterable[Sequence["CellT"]]
+    ) -> None:
+        ...
+
+    def __setitem__(self, item, value) -> None:
+        it = value
+        if isinstance(item, slice):
+            n = len(self.columns)
+            normalized_rows = (
+                chain(val, repeat(self._fill_value, n - len(val)))
+                for val in value
+            )
+            # we need to transpose those rows into columnar format
+            # as we work in terms of column-based arrays
+            it = zip(*normalized_rows)
+
+        for i, col in self._iter_col_row(it):
+            col[item] = i
+
+    def __delitem__(self, item: Union[int, slice]) -> None:
+        for col in self.columns:
+            del col[item]
+
+    def __len__(self) -> int:
+        return len(self.columns[0])
+
+    @property
+    def shape(self) -> Tuple[int, int]:
+        return len(self.columns), len(self)
+
+    def drop(self, *col_names: str) -> None:
+        for col_name in col_names:
+            self._keys.remove(col_name)
+            self._columns.pop(col_name)
+
+    def rename(self, from_col_name: str, to_col_name: str) -> None:
+        self._columns[to_col_name] = self._columns.pop(from_col_name)
+        self._keys[self._keys.index(from_col_name)] = to_col_name
+
+    def project(self, *col_names: str) -> None:
+        self.drop(*(set(self._keys) - set(col_names)))
+        self._keys = list(col_names)
+
+    def to_csv(self) -> str:
+        import csv
+        from io import StringIO
+
+        buff = StringIO()
+        writer = csv.writer(buff)
+        writer.writerow(self.keys())
+
+        for row in self:
+            writer.writerow(row)
+        return buff.getvalue()
+
+    def add_column(self, name: str) -> None:
+        self._columns[name] = Column([self._fill_value] * len(self))
+        self._keys.append(name)
+
+    def row_from_dict(self, d: Dict[str, "CellT"]) -> None:
+        keys = self.keys()
+        for key in d:
+            if key not in keys:
+                self.add_column(key)
+
+        row: List["CellT"] = [
+            with_value(d.get(key), self._fill_value) for key in self.keys()
+        ]
+        self.append(row)
+
+    def render(self, **kwargs: Any):
+        from dvc.ui import ui
+
+        ui.table(self, headers=self.keys(), **kwargs)
+
+    def as_dict(self, cols: Iterable[str] = None) -> Iterable[Dict[str, str]]:
+        keys = self.keys() if cols is None else set(cols)
+        return [
+            {k: self._columns[k][i] for k in keys} for i in range(len(self))
+        ]
+
+
+def _normalize_float(val: float, precision: int):
+    return f"{val:.{precision}g}"
+
+
+def _format_field(
+    val: Any, precision: int = None, round_digits: bool = False
+) -> str:
+    def _format(_val):
+        if isinstance(_val, float) and precision:
+            func = round if round_digits else _normalize_float
+            return func(_val, precision)
+        if isinstance(_val, Mapping):
+            return {k: _format(v) for k, v in _val.items()}
+        if isinstance(_val, list):
+            return [_format(x) for x in _val]
+        return _val
+
+    return str(_format(val))
+
+
+def diff_table(
+    diff,
+    title: str,
+    old: bool = True,
+    no_path: bool = False,
+    show_changes: bool = True,
+    precision: int = None,
+    round_digits: bool = False,
+    on_empty_diff: str = None,
+) -> TabularData:
+    headers: List[str] = ["Path", title, "Old", "New", "Change"]
+    fill_value = "-"
+    td = TabularData(headers, fill_value=fill_value)
+
+    for fname, diff_in_file in diff.items():
+        for item, change in sorted(diff_in_file.items()):
+            old_value = with_value(change.get("old"), fill_value)
+            new_value = with_value(change.get("new"), fill_value)
+            diff_value = with_value(
+                change.get("diff", on_empty_diff), fill_value
+            )
+            td.append(
+                [
+                    fname,
+                    str(item),
+                    _format_field(old_value, precision, round_digits),
+                    _format_field(new_value, precision, round_digits),
+                    _format_field(diff_value, precision, round_digits),
+                ]
+            )
+
+    if no_path:
+        td.drop("Path")
+
+    if not show_changes:
+        td.drop("Change")
+
+    if not old:
+        td.drop("Old")
+        td.rename("New", "Value")
+
+    return td
+
+
+def show_diff(
+    diff,
+    title: str,
+    old: bool = True,
+    no_path: bool = False,
+    show_changes: bool = True,
+    precision: int = None,
+    round_digits: bool = False,
+    on_empty_diff: str = None,
+    markdown: bool = False,
+) -> None:
+    td = diff_table(
+        diff,
+        title=title,
+        old=old,
+        no_path=no_path,
+        show_changes=show_changes,
+        precision=precision,
+        round_digits=round_digits,
+        on_empty_diff=on_empty_diff,
+    )
+    td.render(markdown=markdown)
+
+
+def metrics_table(
+    metrics,
+    all_branches: bool = False,
+    all_tags: bool = False,
+    all_commits: bool = False,
+    precision: int = None,
+    round_digits: bool = False,
+):
+    from dvc.utils.diff import format_dict
+    from dvc.utils.flatten import flatten
+
+    td = TabularData(["Revision", "Path"], fill_value="-")
+
+    for branch, val in metrics.items():
+        for fname, metric in val.items():
+            row_data: Dict[str, str] = {
+                "Revision": branch,
+                "Path": fname,
+            }
+            flattened = (
+                flatten(format_dict(metric))
+                if isinstance(metric, dict)
+                else {"": metric}
+            )
+            row_data.update(
+                {
+                    k: _format_field(v, precision, round_digits)
+                    for k, v in flattened.items()
+                }
+            )
+            td.row_from_dict(row_data)
+
+    rev, path, *metrics_headers = td.keys()
+    td.project(rev, path, *sorted(metrics_headers))
+
+    if not any([all_branches, all_tags, all_commits]):
+        td.drop("Revision")
+
+    return td
+
+
+def show_metrics(
+    metrics,
+    markdown: bool = False,
+    all_branches: bool = False,
+    all_tags: bool = False,
+    all_commits: bool = False,
+    precision: int = None,
+    round_digits: bool = False,
+) -> None:
+    td = metrics_table(
+        metrics,
+        all_branches=all_branches,
+        all_tags=all_tags,
+        all_commits=all_commits,
+        precision=precision,
+        round_digits=round_digits,
+    )
+    td.render(markdown=markdown)

dvc/config_schema.py

@@ -145,6 +145,7 @@ SCHEMA = {
                     "session_token": str,
                     Optional("listobjects", default=False): Bool,  # obsoleted
                     Optional("use_ssl", default=True): Bool,
+                    Optional("ssl_verify", default=True): Bool,
                     "sse": str,
                     "sse_kms_key_id": str,
                     "acl": str,
@@ -171,7 +172,7 @@ SCHEMA = {
                     "allow_agent": Bool,
                     **REMOTE_COMMON,
                 },
-                "hdfs": {"user": str, **REMOTE_COMMON},
+                "hdfs": {"user": str, "kerb_ticket": str, **REMOTE_COMMON},
                 "webhdfs": {
                     "hdfscli_config": str,
                     "webhdfs_token": str,
@@ -222,4 +223,5 @@ SCHEMA = {
         # enabled by default. It's of no use, kept for backward compatibility.
         Optional("parametrization", default=True): Bool
     },
+    "plots": {"html_template": str},
 }

dvc/dagascii.py

@@ -3,52 +3,14 @@
 import logging
 import math
 import os
-import pydoc
-import sys
 
 from grandalf.graphs import Edge, Graph, Vertex
 from grandalf.layouts import SugiyamaLayout
 from grandalf.routing import EdgeViewer, route_with_lines
 
-from dvc.env import DVC_PAGER
-from dvc.utils import format_link
-
 logger = logging.getLogger(__name__)
 
 
-DEFAULT_PAGER = "less"
-DEFAULT_PAGER_FORMATTED = "{} --chop-long-lines --clear-screen".format(
-    DEFAULT_PAGER
-)
-
-
-def make_pager(cmd):
-    def pager(text):
-        return pydoc.tempfilepager(pydoc.plain(text), cmd)
-
-    return pager
-
-
-def find_pager():
-    if not sys.stdout.isatty():
-        return pydoc.plainpager
-
-    env_pager = os.getenv(DVC_PAGER)
-    if env_pager:
-        return make_pager(env_pager)
-
-    if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") == 0:
-        return make_pager(DEFAULT_PAGER_FORMATTED)
-
-    logger.warning(
-        "Unable to find `less` in the PATH. Check out "
-        "{} for more info.".format(
-            format_link("https://man.dvc.org/pipeline/show")
-        )
-    )
-    return pydoc.plainpager
-
-
 class VertexViewer:
     """Class to define vertex box boundaries that will be accounted for during
     graph building by grandalf.

dvc/data_cloud.py

@@ -92,24 +92,6 @@ class DataCloud:
             show_checksums=show_checksums,
         )
 
-    def transfer(self, source, jobs=None, remote=None, command=None):
-        """Transfer data items in a cloud-agnostic way.
-
-        Args:
-            source (str): url for the source location.
-            jobs (int): number of jobs that can be running simultaneously.
-            remote (dvc.remote.base.BaseRemote): optional remote to compare
-                cache to. By default remote from core.remote config option
-                is used.
-            command (str): the command which is benefitting from this function
-                (to be used for reporting better error messages).
-        """
-        from dvc.fs import get_cloud_fs
-
-        from_fs = get_cloud_fs(self.repo, url=source)
-        remote = self.get_remote(remote, command)
-        return remote.transfer(from_fs, from_fs.path_info, jobs=jobs)
-
     def status(
         self,
         cache,

dvc/dependency/param.py

@@ -39,6 +39,7 @@ class ParamsDependency(LocalDependency):
             path
             or os.path.join(stage.repo.root_dir, self.DEFAULT_PARAMS_FILE),
             info=info,
+            fs=stage.repo.fs,
         )
 
     def dumpd(self):
@@ -84,19 +85,34 @@ class ParamsDependency(LocalDependency):
     def status(self):
         return self.workspace_status()
 
-    def read_params(self):
+    def _read(self):
         if not self.exists:
             return {}
 
         suffix = self.path_info.suffix.lower()
         loader = LOADERS[suffix]
         try:
-            config = loader(self.path_info, fs=self.repo.fs)
+            return loader(self.path_info, fs=self.repo.fs)
         except ParseError as exc:
             raise BadParamFileError(
                 f"Unable to read parameters from '{self}'"
             ) from exc
 
+    def read_params_d(self):
+        config = self._read()
+
+        ret = {}
+        for param in self.params:
+            dpath.util.merge(
+                ret,
+                dpath.util.search(config, param, separator="."),
+                separator=".",
+            )
+        return ret
+
+    def read_params(self):
+        config = self._read()
+
         ret = {}
         for param in self.params:
             try:

dvc/external_repo.py

@@ -144,7 +144,7 @@ def _cached_clone(url, rev, for_write=False):
     revision checked out. If for_write is set prevents reusing this dir via
     cache.
     """
-    from distutils.dir_util import copy_tree
+    from shutil import copytree
 
     # even if we have already cloned this repo, we may need to
     # fetch/fast-forward to get specified rev
@@ -156,7 +156,7 @@ def _cached_clone(url, rev, for_write=False):
     # Copy to a new dir to keep the clone clean
     repo_path = tempfile.mkdtemp("dvc-erepo")
     logger.debug("erepo: making a copy of %s clone", url)
-    copy_tree(clone_path, repo_path)
+    copytree(clone_path, repo_path)
 
     # Check out the specified revision
     if for_write:

dvc/fs/azure.py

@@ -51,6 +51,9 @@ class AzureFileSystem(FSSpecWrapper):
     def _prepare_credentials(self, config):
         from azure.identity.aio import DefaultAzureCredential
 
+        # Disable spam from failed cred types for DefaultAzureCredential
+        logging.getLogger("azure.identity.aio").setLevel(logging.ERROR)
+
         login_info = {}
         login_info["connection_string"] = config.get(
             "connection_string",
@@ -69,8 +72,12 @@ class AzureFileSystem(FSSpecWrapper):
         login_info["client_id"] = config.get("client_id")
         login_info["client_secret"] = config.get("client_secret")
 
-        if not any(login_info.values()):
-            login_info["credentials"] = DefaultAzureCredential()
+        if not any(
+            value for key, value in login_info.items() if key != "account_name"
+        ):
+            login_info["credential"] = DefaultAzureCredential(
+                exclude_interactive_browser_credential=False
+            )
 
         for login_method, required_keys in [  # noqa
             ("connection string", ["connection_string"]),
@@ -80,8 +87,11 @@ class AzureFileSystem(FSSpecWrapper):
             ),
             ("account key", ["account_name", "account_key"]),
             ("SAS token", ["account_name", "sas_token"]),
+            (
+                f"default credentials ({_DEFAULT_CREDS_STEPS})",
+                ["account_name", "credential"],
+            ),
             ("anonymous login", ["account_name"]),
-            (f"default credentials ({_DEFAULT_CREDS_STEPS})", ["credentials"]),
         ]:
             if all(login_info.get(key) is not None for key in required_keys):
                 break

dvc/fs/base.py

@@ -1,6 +1,6 @@
 import logging
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from functools import partial
+from functools import partial, partialmethod
 from multiprocessing import cpu_count
 from typing import Any, ClassVar, Dict, FrozenSet, Optional
 
@@ -38,7 +38,7 @@ class BaseFileSystem:
     scheme = "base"
     REQUIRES: ClassVar[Dict[str, str]] = {}
     PATH_CLS = URLInfo  # type: Any
-    JOBS = 4 * cpu_count()
+    _JOBS = 4 * cpu_count()
 
     CHECKSUM_DIR_SUFFIX = ".dir"
     HASH_JOBS = max(1, min(4, cpu_count() // 2))
@@ -67,7 +67,7 @@ class BaseFileSystem:
         return (
             self.config.get("jobs")
             or (self.repo and self.repo.config["core"].get("jobs"))
-            or self.JOBS
+            or self._JOBS
         )
 
     @cached_property
@@ -175,9 +175,12 @@ class BaseFileSystem:
         """
         raise NotImplementedError
 
-    def ls(self, path_info, detail=False, **kwargs):
+    def ls(self, path_info, detail=False):
         raise RemoteActionNotImplemented("ls", self.scheme)
 
+    def find(self, path_info, detail=False):
+        raise RemoteActionNotImplemented("find", self.scheme)
+
     def is_empty(self, path_info):
         return False
 
@@ -258,6 +261,7 @@ class BaseFileSystem:
         name=None,
         no_progress_bar=False,
         jobs=None,
+        _only_file=False,
         **kwargs,
     ):
         if not hasattr(self, "_download"):
@@ -273,12 +277,14 @@ class BaseFileSystem:
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        if self.isdir(from_info):
+        if not _only_file and self.isdir(from_info):
             return self._download_dir(
                 from_info, to_info, name, no_progress_bar, jobs, **kwargs,
             )
         return self._download_file(from_info, to_info, name, no_progress_bar,)
 
+    download_file = partialmethod(download, _only_file=True)
+
     def _download_dir(
         self, from_info, to_info, name, no_progress_bar, jobs, **kwargs,
     ):

dvc/fs/fsspec_wrapper.py

@@ -33,15 +33,13 @@ class FSSpecWrapper(BaseFileSystem):
 
         return path or bucket
 
-    def _strip_buckets(self, entries, detail, prefix=None):
+    def _strip_buckets(self, entries, detail=False):
         for entry in entries:
             if detail:
                 entry = self._entry_hook(entry.copy())
                 entry["name"] = self._strip_bucket(entry["name"])
             else:
-                entry = self._strip_bucket(
-                    f"{prefix}/{entry}" if prefix else entry
-                )
+                entry = self._strip_bucket(entry)
             yield entry
 
     def _entry_hook(self, entry):
@@ -93,24 +91,29 @@ class FSSpecWrapper(BaseFileSystem):
     def exists(self, path_info, use_dvcignore=False):
         return self.fs.exists(self._with_bucket(path_info))
 
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
-        if self.isdir(path_info) and self.is_empty(path_info):
-            return None
-
+    def ls(self, path_info, detail=False):
         path = self._with_bucket(path_info)
-        if recursive:
-            for root, _, files in self.fs.walk(path, detail=detail):
-                if detail:
-                    files = files.values()
-                yield from self._strip_buckets(files, detail, prefix=root)
+        files = self.fs.ls(path, detail=detail)
+        yield from self._strip_buckets(files, detail=detail)
+
+    def find(self, path_info, detail=False):
+        path = self._with_bucket(path_info)
+        files = self.fs.find(path, detail=detail)
+        if detail:
+            files = files.values()
+
+        # When calling find() on a file, it returns the same file in a list.
+        # For object-based storages, the same behavior applies to empty
+        # directories since they are represented as files. This condition
+        # checks whether we should yield an empty list (if it is an empty
+        # directory) or just yield the file itself.
+        if len(files) == 1 and files[0] == path and self.isdir(path_info):
             return None
 
-        yield from self._strip_buckets(self.ls(path, detail=detail), detail)
+        yield from self._strip_buckets(files, detail=detail)
 
     def walk_files(self, path_info, **kwargs):
-        for file in self.ls(path_info, recursive=True):
+        for file in self.find(path_info):
             yield path_info.replace(path=file)
 
     def remove(self, path_info):

dvc/fs/gdrive.py

@@ -532,7 +532,7 @@ class GDriveFileSystem(BaseFileSystem):
         query = f"({query}) and trashed=false"
         return self._gdrive_list(query)
 
-    def _ls_recursive(self, path_info, detail=False):
+    def find(self, path_info, detail=False):
         root_path = path_info.path
         seen_paths = set()
 
@@ -569,13 +569,7 @@ class GDriveFileSystem(BaseFileSystem):
                 else:
                     yield item_path
 
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
-        if recursive:
-            yield from self._ls_recursive(path_info, detail=detail)
-            return None
-
+    def ls(self, path_info, detail=False):
         cached = path_info.path in self._ids_cache["dirs"]
         if cached:
             dir_ids = self._ids_cache["dirs"][path_info.path]
@@ -605,8 +599,8 @@ class GDriveFileSystem(BaseFileSystem):
             self._cache_path_id(root_path, *dir_ids)
 
     def walk_files(self, path_info, **kwargs):
-        for filename in self.ls(path_info, recursive=True):
-            yield path_info.replace(path=filename)
+        for file in self.find(path_info):
+            yield path_info.replace(path=file)
 
     def remove(self, path_info):
         item_id = self._get_item_id(path_info)

dvc/fs/hdfs.py

@@ -84,8 +84,9 @@ class HDFSFileSystem(BaseFileSystem):
             path=parsed.path,
         )
 
-    @staticmethod
-    def hdfs(path_info):
+        self.kerb_ticket = config.get("kerb_ticket")
+
+    def hdfs(self, path_info):
         import pyarrow.fs
 
         # NOTE: HadoopFileSystem is not meant to be closed by us and doesn't
@@ -100,6 +101,7 @@ class HDFSFileSystem(BaseFileSystem):
             path_info.host,
             path_info.port,
             user=path_info.user,
+            kerb_ticket=self.kerb_ticket,
         )
 
     @contextmanager

dvc/fs/s3.py

@@ -38,6 +38,7 @@ class S3FileSystem(BaseFileSystem):
         self.endpoint_url = config.get("endpointurl")
 
         self.use_ssl = config.get("use_ssl", True)
+        self.ssl_verify = config.get("ssl_verify", True)
 
         self.extra_args = {}
 
@@ -136,6 +137,7 @@ class S3FileSystem(BaseFileSystem):
         return session.resource(
             "s3",
             endpoint_url=self.endpoint_url,
+            verify=self.ssl_verify,
             use_ssl=self.use_ssl,
             config=boto3.session.Config(
                 signature_version="s3v4", s3=s3_config
@@ -291,11 +293,7 @@ class S3FileSystem(BaseFileSystem):
 
             yield path_info.replace(path=fname)
 
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
-        assert recursive
-
+    def find(self, path_info, detail=False):
         with self._get_bucket(path_info.bucket) as bucket:
             for obj_summary in bucket.objects.filter(Prefix=path_info.path):
                 if detail:

dvc/fs/ssh/__init__.py

@@ -33,7 +33,7 @@ def ask_password(host, user, port):
 class SSHFileSystem(BaseFileSystem):
     scheme = Schemes.SSH
     REQUIRES = {"paramiko": "paramiko"}
-    JOBS = 4
+    _JOBS = 4
 
     PARAM_CHECKSUM = "md5"
     # At any given time some of the connections will go over network and

dvc/fs/ssh/connection.py

@@ -17,7 +17,7 @@ from ..base import RemoteCmdError
 try:
     import paramiko
 except ImportError:
-    paramiko = None
+    paramiko = None  # type: ignore
 
 
 logger = logging.getLogger(__name__)

dvc/fs/webdav.py

@@ -117,7 +117,7 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             )
 
         # Check whether connection is valid (root should always exist)
-        if not client.check("/"):
+        if not client.check(self.path_info.path):
             raise WebDAVConnectionError(self.hostname)
 
         return client
@@ -151,33 +151,30 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
         # Use webdav is_dir to test whether path points to a directory
         return self._client.is_dir(path_info.path)
 
-    # Yields path info to all files
     def walk_files(self, path_info, **kwargs):
-        # Check whether directory exists
         if not self.exists(path_info):
             return
 
-        # Collect directories
-        dirs = deque([path_info.path])
+        for file in self.find(path_info):
+            yield path_info.replace(path=file)
 
-        # Iterate all directories found so far
-        while dirs:
-            # Iterate directory content
-            for entry in self._client.list(dirs.pop(), get_info=True):
-                # Construct path_info to entry
-                info = path_info.replace(path=entry["path"])
-
-                # Check whether entry is a directory
+    def ls(self, path_info, detail=False):
+        for entry in self._client.list(path_info.path):
+            path = entry["path"]
+            if detail:
                 if entry["isdir"]:
-                    # Append new found directory to directory list
-                    dirs.append(info.path)
+                    yield {"type": "directory", "name": path}
                 else:
-                    # Yield path info to non directory
-                    yield info
+                    yield {
+                        "type": "file",
+                        "name": path,
+                        "size": entry["size"],
+                        "etag": entry["etag"],
+                    }
+            else:
+                yield path
 
-    def ls(
-        self, path_info, detail=False, recursive=False
-    ):  # pylint: disable=arguments-differ
+    def find(self, path_info, detail=False):
         dirs = deque([path_info.path])
 
         while dirs:
@@ -197,14 +194,6 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 else:
                     yield path
 
-            if not recursive:
-                for entry in dirs:
-                    if detail:
-                        yield {"type": "directory", "name": entry}
-                    else:
-                        yield entry
-                return None
-
     # Removes file/directory
     def remove(self, path_info):
         # Use webdav client clean (DELETE) method to remove file/directory
@@ -213,7 +202,7 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     # Creates directories
     def makedirs(self, path_info):
         # Terminate recursion
-        if path_info.path == "/" or self.exists(path_info):
+        if path_info.path == self.path_info.path or self.exists(path_info):
             return
 
         # Recursively descent to root

dvc/ignore.py

@@ -183,7 +183,12 @@ class DvcIgnoreFilter:
     def __init__(self, fs, root_dir):
         from dvc.repo import Repo
 
-        default_ignore_patterns = [".hg/", ".git/", "{}/".format(Repo.DVC_DIR)]
+        default_ignore_patterns = [
+            ".hg/",
+            ".git/",
+            ".git",
+            "{}/".format(Repo.DVC_DIR),
+        ]
 
         self.fs = fs
         self.root_dir = root_dir

dvc/main.py

@@ -45,6 +45,11 @@ def main(argv=None):  # noqa: C901
 
         logger.trace(args)
 
+        if not args.quiet:
+            from dvc.ui import ui
+
+            ui.enable()
+
         with debugtools(args):
             cmd = args.func(args)
             ret = cmd.run()

dvc/objects/__init__.py

@@ -1,4 +1,5 @@
 import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 from dvc.progress import Tqdm
 
@@ -7,10 +8,26 @@ from .tree import Tree
 logger = logging.getLogger(__name__)
 
 
-def save(odb, obj, **kwargs):
+def save(odb, obj, jobs=None, **kwargs):
     if isinstance(obj, Tree):
-        for _, entry in Tqdm(obj):
-            odb.add(entry.path_info, entry.fs, entry.hash_info, **kwargs)
+        with ThreadPoolExecutor(max_workers=jobs) as executor:
+            for future in Tqdm(
+                as_completed(
+                    executor.submit(
+                        odb.add,
+                        entry.path_info,
+                        entry.fs,
+                        entry.hash_info,
+                        **kwargs
+                    )
+                    for _, entry in obj
+                ),
+                desc="Saving files",
+                total=len(obj),
+                unit="file",
+            ):
+                future.result()
+
     odb.add(obj.path_info, obj.fs, obj.hash_info, **kwargs)
 
 

dvc/objects/db/base.py

@@ -1,5 +1,6 @@
 import itertools
 import logging
+import os
 from concurrent.futures import ThreadPoolExecutor
 from copy import copy
 from typing import Optional
@@ -42,9 +43,9 @@ class ObjectDB:
             hash_info,
         )
 
-    def add(self, path_info, fs, hash_info, **kwargs):
+    def add(self, path_info, fs, hash_info, move=True, **kwargs):
         try:
-            self.check(hash_info)
+            self.check(hash_info, check_hash=self.verify)
             return
         except (ObjectFormatError, FileNotFoundError):
             pass
@@ -52,11 +53,32 @@ class ObjectDB:
         cache_info = self.hash_to_path_info(hash_info.value)
         # using our makedirs to create dirs with proper permissions
         self.makedirs(cache_info.parent)
-        if isinstance(fs, type(self.fs)):
-            self.fs.move(path_info, cache_info)
-        else:
-            with fs.open(path_info, mode="rb") as fobj:
-                self.fs.upload_fobj(fobj, cache_info)
+        use_move = isinstance(fs, type(self.fs)) and move
+        try:
+            if use_move:
+                self.fs.move(path_info, cache_info)
+            else:
+                with fs.open(path_info, mode="rb") as fobj:
+                    self.fs.upload_fobj(fobj, cache_info)
+        except OSError as exc:
+            # If the target file already exists, we are going to simply
+            # ignore the exception (#4992).
+            #
+            # On Windows, it is not always guaranteed that you'll get
+            # FileExistsError (it might be PermissionError or a bare OSError)
+            # but all of those exceptions raised from the original
+            # FileExistsError so we have a separate check for that.
+            if isinstance(exc, FileExistsError) or (
+                os.name == "nt"
+                and exc.__context__
+                and isinstance(exc.__context__, FileExistsError)
+            ):
+                logger.debug("'%s' file already exists, skipping", path_info)
+                if use_move:
+                    fs.remove(path_info)
+            else:
+                raise
+
         self.protect(cache_info)
         self.fs.repo.state.save(cache_info, self.fs, hash_info)
 
@@ -84,8 +106,10 @@ class ObjectDB:
     def set_exec(self, path_info):  # pylint: disable=unused-argument
         pass
 
-    def check(self, hash_info):
-        """Compare the given hash with the (corresponding) actual one.
+    def check(self, hash_info, check_hash=True):
+        """Compare the given hash with the (corresponding) actual one if
+        check_hash is specified, or just verify the existence of the cache
+        files on the filesystem.
 
         - Use `State` as a cache for computed hashes
             + The entries are invalidated by taking into account the following:
@@ -106,7 +130,7 @@ class ObjectDB:
             return
 
         try:
-            obj.check(self)
+            obj.check(self, check_hash=check_hash)
         except ObjectFormatError:
             logger.warning("corrupted cache file '%s'.", obj.path_info)
             self.fs.remove(obj.path_info)
@@ -229,7 +253,7 @@ class ObjectDB:
         keeping all of it in memory.
         """
         num_pages = remote_size / self.fs.LIST_OBJECT_PAGE_SIZE
-        if num_pages < 256 / self.fs.JOBS:
+        if num_pages < 256 / self.fs.jobs:
             # Fetching prefixes in parallel requires at least 255 more
             # requests, for small enough remotes it will be faster to fetch
             # entire cache without splitting it into prefixes.
@@ -263,7 +287,7 @@ class ObjectDB:
                 )
 
             with ThreadPoolExecutor(
-                max_workers=jobs or self.fs.JOBS
+                max_workers=jobs or self.fs.jobs
             ) as executor:
                 in_remote = executor.map(list_with_update, traverse_prefixes,)
                 yield from itertools.chain.from_iterable(in_remote)
@@ -331,7 +355,7 @@ class ObjectDB:
                 return ret
 
             with ThreadPoolExecutor(
-                max_workers=jobs or self.fs.JOBS
+                max_workers=jobs or self.fs.jobs
             ) as executor:
                 path_infos = map(self.hash_to_path_info, hashes)
                 in_remote = executor.map(exists_with_progress, path_infos)

dvc/objects/db/ssh.py

@@ -64,10 +64,10 @@ class SSHObjectDB(ObjectDB):
                 return self.batch_exists(chunks, callback=pbar.update_msg)
 
             with ThreadPoolExecutor(
-                max_workers=jobs or self.fs.JOBS
+                max_workers=jobs or self.fs.jobs
             ) as executor:
                 path_infos = [self.hash_to_path_info(x) for x in hashes]
-                chunks = to_chunks(path_infos, num_chunks=self.fs.JOBS)
+                chunks = to_chunks(path_infos, num_chunks=self.fs.jobs)
                 results = executor.map(exists_with_progress, chunks)
                 in_remote = itertools.chain.from_iterable(results)
                 ret = list(itertools.compress(hashes, in_remote))

dvc/objects/file.py

@@ -1,7 +1,8 @@
+import errno
 import logging
+import os
 
 from .errors import ObjectFormatError
-from .stage import get_file_hash
 
 logger = logging.getLogger(__name__)
 
@@ -36,7 +37,17 @@ class HashFile:
             and self.hash_info == other.hash_info
         )
 
-    def check(self, odb):
+    def check(self, odb, check_hash=True):
+        from .stage import get_file_hash
+
+        if not check_hash:
+            if not self.fs.exists(self.path_info):
+                raise FileNotFoundError(
+                    errno.ENOENT, os.strerror(errno.ENOENT), self.path_info
+                )
+            else:
+                return None
+
         actual = get_file_hash(
             self.path_info, self.fs, self.hash_info.name, odb.repo.state
         )

dvc/objects/stage.py

@@ -1,14 +1,31 @@
 import errno
 import os
 from concurrent.futures import ThreadPoolExecutor
+from functools import partial
 
 from dvc.exceptions import DvcIgnoreInCollectedDirError
 from dvc.hash_info import HashInfo
 from dvc.ignore import DvcIgnore
+from dvc.objects.file import HashFile
 from dvc.progress import Tqdm
 from dvc.utils import file_md5
 
 
+def _upload_file(path_info, fs, odb):
+    from dvc.utils import tmp_fname
+    from dvc.utils.stream import HashedStreamReader
+
+    tmp_info = odb.fs.path_info / tmp_fname()
+    with fs.open(path_info, mode="rb", chunk_size=fs.CHUNK_SIZE) as stream:
+        stream = HashedStreamReader(stream)
+        odb.fs.upload_fobj(
+            stream, tmp_info, desc=path_info.name, total=fs.getsize(path_info)
+        )
+
+    obj = HashFile(tmp_info, odb.fs, stream.hash_info)
+    return path_info, obj
+
+
 def _get_file_hash(path_info, fs, name):
     info = fs.info(path_info)
     if name in info:
@@ -49,43 +66,64 @@ def get_file_hash(path_info, fs, name, state=None):
     return hash_info
 
 
-def _calculate_hashes(path_info, fs, name, state, **kwargs):
-    def _hash(file_info):
-        return file_info, get_file_hash(file_info, fs, name, state)
+def _get_file_obj(path_info, fs, name, odb=None, state=None, upload=False):
+    if upload:
+        assert odb and name == "md5"
+        return _upload_file(path_info, fs, odb)
 
+    obj = HashFile(
+        path_info, fs, get_file_hash(path_info, fs, name, state=state)
+    )
+    return path_info, obj
+
+
+def _build_objects(path_info, fs, name, odb, state, upload, **kwargs):
     with Tqdm(
-        unit="md5", desc="Computing file/dir hashes (only done once)",
+        unit="md5",
+        desc="Computing file/dir hashes (only done once)",
+        disable=kwargs.pop("no_progress_bar", False),
     ) as pbar:
-        worker = pbar.wrap_fn(_hash)
-        with ThreadPoolExecutor(max_workers=fs.hash_jobs) as executor:
-            pairs = executor.map(worker, fs.walk_files(path_info, **kwargs))
-            return dict(pairs)
+        worker = pbar.wrap_fn(
+            partial(
+                _get_file_obj,
+                fs=fs,
+                name=name,
+                odb=odb,
+                state=state,
+                upload=upload,
+            )
+        )
+        with ThreadPoolExecutor(
+            max_workers=kwargs.pop("jobs", fs.hash_jobs)
+        ) as executor:
+            yield from executor.map(worker, fs.walk_files(path_info, **kwargs))
 
 
-def _iter_hashes(path_info, fs, name, state, **kwargs):
-    if name in fs.DETAIL_FIELDS:
-        for details in fs.ls(path_info, recursive=True, detail=True):
+def _iter_objects(path_info, fs, name, odb, state, upload, **kwargs):
+    if not upload and name in fs.DETAIL_FIELDS:
+        for details in fs.find(path_info, detail=True):
             file_info = path_info.replace(path=details["name"])
             hash_info = HashInfo(
                 name, details[name], size=details.get("size"),
             )
-            yield file_info, hash_info
+            yield file_info, HashFile(file_info, fs, hash_info)
 
         return None
 
-    yield from _calculate_hashes(path_info, fs, name, state, **kwargs).items()
+    yield from _build_objects(
+        path_info, fs, name, odb, state, upload, **kwargs
+    )
 
 
-def _build_tree(path_info, fs, name, state, **kwargs):
-    from .file import HashFile
+def _build_tree(path_info, fs, name, odb, state, upload, **kwargs):
     from .tree import Tree
 
     tree = Tree(None, None, None)
-    for fi, hi in _iter_hashes(path_info, fs, name, state, **kwargs):
-        if DvcIgnore.DVCIGNORE_FILE == fi.name:
-            raise DvcIgnoreInCollectedDirError(fi.parent)
-
-        obj = HashFile(fi, fs, hi)
+    for file_info, obj in _iter_objects(
+        path_info, fs, name, odb, state, upload, **kwargs
+    ):
+        if DvcIgnore.DVCIGNORE_FILE == file_info.name:
+            raise DvcIgnoreInCollectedDirError(file_info.parent)
 
         # NOTE: this is lossy transformation:
         #   "hey\there" -> "hey/there"
@@ -95,14 +133,14 @@ def _build_tree(path_info, fs, name, state, **kwargs):
         #
         # Yes, this is a BUG, as long as we permit "/" in
         # filenames on Windows and "\" on Unix
-        tree.add(fi.relative_to(path_info).parts, obj)
+        tree.add(file_info.relative_to(path_info).parts, obj)
 
     tree.digest()
 
     return tree
 
 
-def _get_tree_obj(path_info, fs, name, odb, state, **kwargs):
+def _get_tree_obj(path_info, fs, name, odb, state, upload, **kwargs):
     from .tree import Tree
 
     value = fs.info(path_info).get(name)
@@ -113,7 +151,7 @@ def _get_tree_obj(path_info, fs, name, odb, state, **kwargs):
         except FileNotFoundError:
             pass
 
-    tree = _build_tree(path_info, fs, name, state, **kwargs)
+    tree = _build_tree(path_info, fs, name, odb, state, upload, **kwargs)
 
     odb.add(tree.path_info, tree.fs, tree.hash_info)
     if name != "md5":
@@ -132,7 +170,7 @@ def _get_tree_obj(path_info, fs, name, odb, state, **kwargs):
     return tree
 
 
-def stage(odb, path_info, fs, name, **kwargs):
+def stage(odb, path_info, fs, name, upload=False, **kwargs):
     assert path_info and (
         isinstance(path_info, str) or path_info.scheme == fs.scheme
     )
@@ -174,12 +212,9 @@ def stage(odb, path_info, fs, name, **kwargs):
         return obj
 
     if fs.isdir(path_info):
-        obj = _get_tree_obj(path_info, fs, name, odb, state, **kwargs)
+        obj = _get_tree_obj(path_info, fs, name, odb, state, upload, **kwargs)
     else:
-        from .file import HashFile
-
-        hash_info = get_file_hash(path_info, fs, name, state)
-        obj = HashFile(path_info, fs, hash_info)
+        _, obj = _get_file_obj(path_info, fs, name, odb, state, upload)
 
     if obj.hash_info and fs.exists(path_info):
         state.save(path_info, fs, obj.hash_info)

dvc/objects/transfer.py

@@ -1,92 +0,0 @@
-import itertools
-import logging
-from concurrent import futures
-
-from dvc.progress import Tqdm
-
-from . import save
-from .file import HashFile
-from .tree import Tree
-
-logger = logging.getLogger(__name__)
-
-
-def _transfer_file(odb, from_fs, from_info):
-    from dvc.utils import tmp_fname
-    from dvc.utils.stream import HashedStreamReader
-
-    tmp_info = odb.fs.path_info / tmp_fname()
-    with from_fs.open(
-        from_info, mode="rb", chunk_size=from_fs.CHUNK_SIZE
-    ) as stream:
-        stream_reader = HashedStreamReader(stream)
-        # Since we don't know the hash beforehand, we'll
-        # upload it to a temporary location and then move
-        # it.
-        odb.fs.upload_fobj(
-            stream_reader,
-            tmp_info,
-            total=from_fs.getsize(from_info),
-            desc=from_info.name,
-        )
-
-    hash_info = stream_reader.hash_info
-    return tmp_info, hash_info
-
-
-def _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
-    rel_path_infos = {}
-    from_infos = from_fs.walk_files(from_info)
-
-    def create_tasks(executor, amount):
-        for entry_info in itertools.islice(from_infos, amount):
-            pbar.total += 1
-            task = executor.submit(
-                pbar.wrap_fn(_transfer_file), odb, from_fs, entry_info
-            )
-            rel_path_infos[task] = entry_info.relative_to(from_info)
-            yield task
-
-    pbar.total = 0
-    with futures.ThreadPoolExecutor(max_workers=jobs) as executor:
-        tasks = set(create_tasks(executor, jobs * 5))
-
-        while tasks:
-            done, tasks = futures.wait(
-                tasks, return_when=futures.FIRST_COMPLETED
-            )
-            tasks.update(create_tasks(executor, len(done)))
-            for task in done:
-                yield rel_path_infos.pop(task), task.result()
-
-
-def _transfer_directory(odb, from_fs, from_info, jobs, no_progress_bar=False):
-    tree = Tree(None, None, None)
-
-    with Tqdm(total=1, unit="Files", disable=no_progress_bar) as pbar:
-        for (
-            entry_info,
-            (entry_tmp_info, entry_hash),
-        ) in _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
-            obj = HashFile(entry_tmp_info, odb.fs, entry_hash)
-            tree.add(entry_info.parts, obj)
-
-    tree.digest()
-    save(odb, tree)
-    return tree.hash_info
-
-
-def transfer(odb, from_fs, from_info, jobs=None, no_progress_bar=False):
-    jobs = jobs or min((from_fs.jobs, odb.fs.jobs))
-
-    if from_fs.isdir(from_info):
-        return _transfer_directory(
-            odb,
-            from_fs,
-            from_info,
-            jobs=jobs,
-            no_progress_bar=no_progress_bar,
-        )
-    tmp_info, hash_info = _transfer_file(odb, from_fs, from_info)
-    odb.add(tmp_info, odb.fs, hash_info)
-    return hash_info

dvc/output/base.py

@@ -17,6 +17,7 @@ from dvc.exceptions import (
     RemoteCacheRequiredError,
 )
 from dvc.hash_info import HashInfo
+from dvc.objects import save as osave
 from dvc.objects.db import NamedCache
 from dvc.objects.errors import ObjectFormatError
 from dvc.objects.stage import stage as ostage
@@ -470,6 +471,42 @@ class BaseOutput:
         if self.scheme == "local" and self.use_scm_ignore:
             self.repo.scm.ignore(self.fspath)
 
+    def transfer(
+        self, source, odb=None, jobs=None, update=False, no_progress_bar=False,
+    ):
+        from dvc.fs import get_cloud_fs
+
+        if odb is None:
+            odb = self.odb
+
+        from_fs = get_cloud_fs(self.repo, url=source)
+        from_info = from_fs.path_info
+
+        # When running import-url --to-remote / add --to-remote/-o ... we
+        # assume that it is unlikely that the odb will contain majority of the
+        # hashes, so we transfer everything as is (even if that file might
+        # already be in the cache) and don't waste an upload to scan the layout
+        # of the source location. But when doing update --to-remote, there is
+        # a high probability that the odb might contain some of the hashes, so
+        # we first calculate all the hashes (but don't transfer anything) and
+        # then only update the missing cache files.
+
+        upload = not (update and from_fs.isdir(from_info))
+        jobs = jobs or min((from_fs.jobs, odb.fs.jobs))
+        obj = ostage(
+            odb,
+            from_info,
+            from_fs,
+            "md5",
+            upload=upload,
+            jobs=jobs,
+            no_progress_bar=no_progress_bar,
+        )
+        osave(odb, obj, jobs=jobs, move=upload)
+
+        self.hash_info = obj.hash_info
+        return obj
+
     def get_files_number(self, filter_info=None):
         if not self.use_cache or not self.hash_info:
             return 0

dvc/output/local.py

@@ -48,7 +48,11 @@ class LocalOutput(BaseOutput):
         return self.FS_CLS.PATH_CLS(abs_p)
 
     def __str__(self):
-        if not self.repo or not self.is_in_repo:
+        if (
+            not self.repo
+            or urlparse(self.def_path).scheme == "remote"
+            or os.path.isabs(self.def_path)
+        ):
             return str(self.def_path)
 
         cur_dir = os.getcwd()
@@ -63,8 +67,15 @@ class LocalOutput(BaseOutput):
 
     @property
     def is_in_repo(self):
-        def_scheme = urlparse(self.def_path).scheme
-        return def_scheme != "remote" and not os.path.isabs(self.def_path)
+        if urlparse(self.def_path).scheme == "remote":
+            return False
+
+        if os.path.isabs(self.def_path):
+            return False
+
+        if self.repo and not self.path_info.isin(self.repo.root_dir):
+            return False
+        return True
 
     def dumpd(self):
         ret = super().dumpd()

dvc/parsing/context.py

@@ -401,9 +401,9 @@ class Context(CtxDict):
     ):
         path, _, keys_str = item.partition(":")
         select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
-        path_info = wdir / path
 
-        abspath = os.path.abspath(path_info)
+        abspath = os.path.abspath(wdir / path)
+        path_info = PathInfo(abspath)
         if abspath in self.imports:
             if not select_keys and self.imports[abspath] is None:
                 return  # allow specifying complete filepath multiple times

dvc/remote/base.py

@@ -309,7 +309,7 @@ class Remote:
         )
 
         if download:
-            func = _log_exceptions(self.fs.download, "download")
+            func = _log_exceptions(self.fs.download_file, "download")
             status = STATUS_DELETED
             desc = "Downloading"
         else:
@@ -318,7 +318,7 @@ class Remote:
             desc = "Uploading"
 
         if jobs is None:
-            jobs = self.fs.JOBS
+            jobs = self.fs.jobs
 
         dir_status, file_status, dir_contents = self._status(
             cache,
@@ -505,17 +505,6 @@ class Remote:
 
         return ret
 
-    def transfer(self, from_fs, from_info, jobs=None, no_progress_bar=False):
-        from dvc.objects.transfer import transfer
-
-        return transfer(
-            self.odb,
-            from_fs,
-            from_info,
-            jobs=jobs,
-            no_progress_bar=no_progress_bar,
-        )
-
     @staticmethod
     def _log_missing_caches(hash_info_dict):
         missing_caches = [

dvc/repo/__init__.py

@@ -141,6 +141,7 @@ class Repo:
         from dvc.repo.params import Params
         from dvc.repo.plots import Plots
         from dvc.repo.stage import StageLoad
+        from dvc.scm import SCM
         from dvc.stage.cache import StageCache
         from dvc.state import State, StateNoop
 
@@ -149,6 +150,9 @@ class Repo:
             "repo_factory": repo_factory,
         }
 
+        if rev and not scm:
+            scm = SCM(root_dir or os.curdir)
+
         self.root_dir, self.dvc_dir, self.tmp_dir = self._get_repo_dirs(
             root_dir=root_dir, scm=scm, rev=rev, uninitialized=uninitialized
         )

dvc/repo/add.py

@@ -164,21 +164,14 @@ def _process_stages(
         (target,) = sub_targets
         (out,) = stage.outs
 
+        odb = None
         if to_remote:
-            out.hash_info = repo.cloud.transfer(
-                target,
-                jobs=kwargs.get("jobs"),
-                remote=kwargs.get("remote"),
-                command="add",
-            )
-        else:
-            from dvc.fs import get_cloud_fs
-            from dvc.objects.transfer import transfer
+            remote = repo.cloud.get_remote(kwargs.get("remote"), "add")
+            odb = remote.odb
 
-            from_fs = get_cloud_fs(repo, url=target)
-            out.hash_info = transfer(
-                out.odb, from_fs, from_fs.path_info, jobs=kwargs.get("jobs"),
-            )
+        out.transfer(target, odb=odb, jobs=kwargs.get("jobs"))
+
+        if to_cache:
             out.checkout()
 
         Dvcfile(repo, stage.path).dump(stage)

dvc/repo/commit.py

@@ -37,20 +37,32 @@ def prompt_to_commit(stage, changes, force=False):
 
 @locked
 def commit(
-    self, target, with_deps=False, recursive=False, force=False,
+    self,
+    target,
+    with_deps=False,
+    recursive=False,
+    force=False,
+    allow_missing=False,
+    data_only=False,
 ):
     from dvc.dvcfile import Dvcfile
 
-    stages_info = self.stage.collect_granular(
-        target, with_deps=with_deps, recursive=recursive
-    )
+    stages_info = [
+        info
+        for info in self.stage.collect_granular(
+            target, with_deps=with_deps, recursive=recursive
+        )
+        if not data_only or info.stage.is_data_source
+    ]
     for stage_info in stages_info:
         stage = stage_info.stage
         changes = stage.changed_entries()
         if any(changes):
             prompt_to_commit(stage, changes, force=force)
-            stage.save()
-        stage.commit(filter_info=stage_info.filter_info)
+            stage.save(allow_missing=allow_missing)
+        stage.commit(
+            filter_info=stage_info.filter_info, allow_missing=allow_missing
+        )
 
         Dvcfile(self, stage.path).dump(stage, update_pipeline=False)
     return [s.stage for s in stages_info]

dvc/repo/diff.py

@@ -7,6 +7,7 @@ from dvc.exceptions import PathMissingError
 from dvc.objects.stage import get_file_hash
 from dvc.objects.stage import stage as ostage
 from dvc.repo import locked
+from dvc.repo.experiments.utils import fix_exp_head
 
 logger = logging.getLogger(__name__)
 
@@ -28,7 +29,8 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
 
     repo_fs = RepoFileSystem(self)
 
-    b_rev = b_rev if b_rev else "workspace"
+    a_rev = fix_exp_head(self.scm, a_rev)
+    b_rev = fix_exp_head(self.scm, b_rev) if b_rev else "workspace"
     results = {}
     missing_targets = {}
     for rev in self.brancher(revs=[a_rev, b_rev]):
@@ -161,6 +163,7 @@ def _dir_output_paths(repo_fs, output, targets=None):
             if targets is None or any(
                 fname.isin_or_eq(target) for target in targets
             ):
+                # pylint: disable=no-member
                 yield str(fname), get_file_hash(fname, repo_fs, "md5").value
     except NoRemoteError:
         logger.warning("dir cache entry for '%s' is missing", output)

dvc/repo/experiments/__init__.py

@@ -186,11 +186,15 @@ class Experiments:
                     if params:
                         self._update_params(params)
 
+                    # DVC commit data deps to preserve state across workspace
+                    # & tempdir runs
+                    self._stash_commit_deps(*args, **kwargs)
+
                     if resume_rev:
                         if branch:
                             branch_name = ExpRefInfo.from_ref(branch).name
                         else:
-                            branch_name = ""
+                            branch_name = f"{resume_rev[:7]}"
                         if self.scm.is_dirty():
                             logger.info(
                                 "Modified checkpoint experiment based on "
@@ -198,6 +202,39 @@ class Experiments:
                                 branch_name,
                             )
                             branch = None
+                        elif (
+                            not branch
+                            or self.scm.get_ref(branch) != resume_rev
+                        ):
+                            msg = [
+                                (
+                                    "Nothing to do for unchanged checkpoint "
+                                    f"'{resume_rev[:7]}'. "
+                                )
+                            ]
+                            if branch:
+                                msg.append(
+                                    "To resume from the head of this "
+                                    "experiment, use "
+                                    f"'dvc exp apply {branch_name}'."
+                                )
+                            else:
+                                names = [
+                                    ref_info.name
+                                    for ref_info in exp_refs_by_rev(
+                                        self.scm, resume_rev
+                                    )
+                                ]
+                                if len(names) > 3:
+                                    names[3:] = [
+                                        f"... ({len(names) - 3} more)"
+                                    ]
+                                msg.append(
+                                    "To resume an experiment containing this "
+                                    "checkpoint, apply one of these heads:\n"
+                                    "\t{}".format(", ".join(names))
+                                )
+                            raise DvcException("".join(msg))
                         else:
                             logger.info(
                                 "Existing checkpoint experiment '%s' will be "
@@ -233,15 +270,38 @@ class Experiments:
                         baseline_rev[:7],
                     )
                 finally:
-                    # Reset any of our changes before prior unstashing
                     if resume_rev:
+                        # NOTE: this set_ref + reset() is equivalent to
+                        # `git reset orig_head` (our SCM reset() only operates
+                        # on HEAD rather than any arbitrary commit)
                         self.scm.set_ref(
                             "HEAD", orig_head, message="dvc: restore HEAD"
                         )
+                        self.scm.reset()
+                    # Revert any of our changes before prior unstashing
                     self.scm.reset(hard=True)
 
         return stash_rev
 
+    def _stash_commit_deps(self, *args, **kwargs):
+        if len(args):
+            targets = args[0]
+        else:
+            targets = kwargs.get("targets")
+        if isinstance(targets, str):
+            targets = [targets]
+        elif not targets:
+            targets = [None]
+        for target in targets:
+            self.repo.commit(
+                target,
+                with_deps=True,
+                recursive=kwargs.get("recursive", False),
+                force=True,
+                allow_missing=True,
+                data_only=True,
+            )
+
     def _stash_msg(
         self,
         rev: str,
@@ -324,7 +384,6 @@ class Experiments:
 
         if reset:
             self.reset_checkpoints()
-            kwargs["force"] = True
 
         if not (queue or tmp_dir):
             staged, _, _ = self.scm.status()
@@ -425,22 +484,34 @@ class Experiments:
         """
         assert resume_rev
 
-        allow_multiple = "params" in kwargs
-        branch: Optional[str] = self.get_branch_by_rev(
-            resume_rev, allow_multiple=allow_multiple
-        )
-        if not branch:
-            raise DvcException(
-                "Could not find checkpoint experiment " f"'{resume_rev[:7]}'"
+        branch: Optional[str] = None
+        try:
+            allow_multiple = bool(kwargs.get("params", None))
+            branch = self.get_branch_by_rev(
+                resume_rev, allow_multiple=allow_multiple
             )
+            if not branch:
+                raise DvcException(
+                    "Could not find checkpoint experiment "
+                    f"'{resume_rev[:7]}'"
+                )
+            baseline_rev = self._get_baseline(branch)
+        except MultipleBranchError as exc:
+            baselines = {
+                info.baseline_sha
+                for info in exc.ref_infos
+                if info.baseline_sha
+            }
+            if len(baselines) == 1:
+                baseline_rev = baselines.pop()
+            else:
+                raise
 
-        baseline_rev = self._get_baseline(branch)
         logger.debug(
-            "Resume from checkpoint '%s' with baseline '%s'",
-            resume_rev,
+            "Checkpoint run from '%s' with baseline '%s'",
+            resume_rev[:7],
             baseline_rev,
         )
-
         return self._stash_exp(
             *args,
             resume_rev=resume_rev,
@@ -685,10 +756,10 @@ class Experiments:
 
         # NOTE: the stash commit to be popped already contains all the current
         # workspace changes plus CLI modifed --params changes.
-        # `reset --hard` here will not lose any data (pop without reset would
-        # result in conflict between workspace params and stashed CLI params).
-        self.scm.reset(hard=True)
-        with self.scm.detach_head(entry.rev):
+        # `checkout --force` here will not lose any data (popping stash commit
+        # will result in conflict between workspace params and stashed CLI
+        # params, but we always want the stashed version).
+        with self.scm.detach_head(entry.rev, force=True):
             rev = self.stash.pop()
             self.scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
             if entry.branch:
@@ -775,7 +846,10 @@ class Experiments:
         if not ref_infos:
             return None
         if len(ref_infos) > 1 and not allow_multiple:
-            raise MultipleBranchError(rev)
+            for ref_info in ref_infos:
+                if self.scm.get_ref(str(ref_info)) == rev:
+                    return str(ref_info)
+            raise MultipleBranchError(rev, ref_infos)
         return str(ref_infos[0])
 
     def get_exact_name(self, rev: str):

dvc/repo/experiments/apply.py

@@ -32,7 +32,9 @@ def apply(repo, rev, force=True, **kwargs):
         raise InvalidExpRevError(rev) from exc
 
     stash_rev = exp_rev in exps.stash_revs
-    if not stash_rev and not exps.get_branch_by_rev(exp_rev):
+    if not stash_rev and not exps.get_branch_by_rev(
+        exp_rev, allow_multiple=True
+    ):
         raise InvalidExpRevError(exp_rev)
 
     # Note that we don't use stash_workspace() here since we need finer control

dvc/repo/experiments/base.py

@@ -74,12 +74,13 @@ class InvalidExpRevError(InvalidArgumentError):
 
 
 class MultipleBranchError(DvcException):
-    def __init__(self, rev):
+    def __init__(self, rev, ref_infos):
         super().__init__(
             f"Ambiguous commit '{rev[:7]}' belongs to multiple experiment "
             "branches."
         )
         self.rev = rev
+        self.ref_infos = ref_infos
 
 
 class ApplyConflictError(InvalidArgumentError):

dvc/repo/experiments/diff.py

@@ -1,28 +1,35 @@
 import logging
 
+from dvc.repo.experiments.utils import fix_exp_head
 from dvc.utils.diff import diff as _diff
 from dvc.utils.diff import format_dict
 
 logger = logging.getLogger(__name__)
 
 
-def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
+def diff(repo, *args, a_rev=None, b_rev=None, param_deps=False, **kwargs):
     from dvc.repo.experiments.show import _collect_experiment_commit
 
     if repo.scm.no_commits:
         return {}
 
     if a_rev:
+        a_rev = fix_exp_head(repo.scm, a_rev)
         rev = repo.scm.resolve_rev(a_rev)
-        old = _collect_experiment_commit(repo, rev)
+        old = _collect_experiment_commit(repo, rev, param_deps=param_deps)
     else:
-        old = _collect_experiment_commit(repo, "HEAD")
+        old = _collect_experiment_commit(
+            repo, fix_exp_head(repo.scm, "HEAD"), param_deps=param_deps
+        )
 
     if b_rev:
+        b_rev = fix_exp_head(repo.scm, b_rev)
         rev = repo.scm.resolve_rev(b_rev)
-        new = _collect_experiment_commit(repo, rev)
+        new = _collect_experiment_commit(repo, rev, param_deps=param_deps)
     else:
-        new = _collect_experiment_commit(repo, "workspace")
+        new = _collect_experiment_commit(
+            repo, "workspace", param_deps=param_deps
+        )
 
     with_unchanged = kwargs.pop("all", False)
 

dvc/repo/experiments/executor/base.py

@@ -259,6 +259,8 @@ class BaseExecutor(ABC):
                 "Executor repro with force = '%s'", str(repro_force)
             )
 
+            repro_dry = kwargs.get("dry")
+
             # NOTE: checkpoint outs are handled as a special type of persist
             # out:
             #
@@ -271,19 +273,23 @@ class BaseExecutor(ABC):
             # - if run without --reset, the checkpoint out will be checked out
             #   using any hash present in dvc.lock (or removed if no entry
             #   exists in dvc.lock)
-            checkpoint_reset = kwargs.pop("reset", False)
-            dvc_checkout(
-                dvc,
-                targets=targets,
-                with_deps=targets is not None,
-                force=True,
-                quiet=True,
-                allow_missing=True,
-                checkpoint_reset=checkpoint_reset,
-            )
+            checkpoint_reset: bool = kwargs.pop("reset", False)
+            if not repro_dry:
+                dvc_checkout(
+                    dvc,
+                    targets=targets,
+                    with_deps=targets is not None,
+                    force=True,
+                    quiet=True,
+                    allow_missing=True,
+                    checkpoint_reset=checkpoint_reset,
+                )
 
             checkpoint_func = partial(
-                cls.checkpoint_callback, dvc.scm, name, repro_force
+                cls.checkpoint_callback,
+                dvc.scm,
+                name,
+                repro_force or checkpoint_reset,
             )
             stages = dvc_reproduce(
                 dvc,
@@ -294,29 +300,38 @@ class BaseExecutor(ABC):
             )
 
             exp_hash = cls.hash_exp(stages)
-            try:
-                cls.commit(
-                    dvc.scm,
-                    exp_hash,
-                    exp_name=name,
-                    force=repro_force,
-                    checkpoint=any(stage.is_checkpoint for stage in stages),
-                )
-            except UnchangedExperimentError:
-                pass
-            ref = dvc.scm.get_ref(EXEC_BRANCH, follow=False)
-            if ref:
-                exp_ref = ExpRefInfo.from_ref(ref)
-            if cls.WARN_UNTRACKED:
-                untracked = dvc.scm.untracked_files()
-                if untracked:
-                    logger.warning(
-                        "The following untracked files were present in the "
-                        "experiment directory after reproduction but will "
-                        "not be included in experiment commits:\n"
-                        "\t%s",
-                        ", ".join(untracked),
+            if not repro_dry:
+                try:
+                    is_checkpoint = any(
+                        stage.is_checkpoint for stage in stages
                     )
+                    if is_checkpoint and checkpoint_reset:
+                        # For reset checkpoint stages, we need to force
+                        # overwriting existing checkpoint refs even though
+                        # repro may not have actually been run with --force
+                        repro_force = True
+                    cls.commit(
+                        dvc.scm,
+                        exp_hash,
+                        exp_name=name,
+                        force=repro_force,
+                        checkpoint=is_checkpoint,
+                    )
+                except UnchangedExperimentError:
+                    pass
+                ref = dvc.scm.get_ref(EXEC_BRANCH, follow=False)
+                if ref:
+                    exp_ref = ExpRefInfo.from_ref(ref)
+                if cls.WARN_UNTRACKED:
+                    untracked = dvc.scm.untracked_files()
+                    if untracked:
+                        logger.warning(
+                            "The following untracked files were present in "
+                            "the experiment directory after reproduction but "
+                            "will not be included in experiment commits:\n"
+                            "\t%s",
+                            ", ".join(untracked),
+                        )
 
         # ideally we would return stages here like a normal repro() call, but
         # stages is not currently picklable and cannot be returned across

dvc/repo/experiments/show.py

@@ -5,6 +5,7 @@ from datetime import datetime
 from dvc.exceptions import InvalidArgumentError
 from dvc.repo import locked
 from dvc.repo.experiments.base import ExpRefInfo
+from dvc.repo.experiments.utils import fix_exp_head
 from dvc.repo.metrics.show import _collect_metrics, _read_metrics
 from dvc.repo.params.show import _collect_configs, _read_params
 from dvc.scm.base import SCMError
@@ -12,17 +13,23 @@ from dvc.scm.base import SCMError
 logger = logging.getLogger(__name__)
 
 
-def _collect_experiment_commit(repo, rev, stash=False, sha_only=True):
+def _collect_experiment_commit(
+    repo, exp_rev, stash=False, sha_only=True, param_deps=False
+):
     res = defaultdict(dict)
-    for rev in repo.brancher(revs=[rev]):
+    for rev in repo.brancher(revs=[exp_rev]):
         if rev == "workspace":
+            if exp_rev != "workspace":
+                continue
             res["timestamp"] = None
         else:
             commit = repo.scm.resolve_commit(rev)
             res["timestamp"] = datetime.fromtimestamp(commit.commit_time)
 
-        configs = _collect_configs(repo, rev=rev)
-        params = _read_params(repo, configs, rev)
+        params, params_path_infos = _collect_configs(repo, rev=rev)
+        params = _read_params(
+            repo, params, params_path_infos, rev, deps=param_deps
+        )
         if params:
             res["params"] = params
 
@@ -83,6 +90,7 @@ def show(
     all_commits=False,
     sha_only=False,
     num=1,
+    param_deps=False,
 ):
     res = defaultdict(OrderedDict)
 
@@ -93,7 +101,8 @@ def show(
         revs = []
         for n in range(num):
             try:
-                revs.append(repo.scm.resolve_rev(f"HEAD~{n}"))
+                head = fix_exp_head(repo.scm, f"HEAD~{n}")
+                revs.append(repo.scm.resolve_rev(head))
             except SCMError:
                 break
 
@@ -110,7 +119,7 @@ def show(
 
     for rev in revs:
         res[rev]["baseline"] = _collect_experiment_commit(
-            repo, rev, sha_only=sha_only
+            repo, rev, sha_only=sha_only, param_deps=param_deps
         )
 
         if rev == "workspace":
@@ -127,14 +136,19 @@ def show(
             ref_info = ExpRefInfo.from_ref(exp_ref)
             assert ref_info.baseline_sha == rev
             _collect_experiment_branch(
-                res[rev], repo, exp_ref, rev, sha_only=sha_only
+                res[rev],
+                repo,
+                exp_ref,
+                rev,
+                sha_only=sha_only,
+                param_deps=param_deps,
             )
 
     # collect queued (not yet reproduced) experiments
     for stash_rev, entry in repo.experiments.stash_revs.items():
         if entry.baseline_rev in revs:
             experiment = _collect_experiment_commit(
-                repo, stash_rev, stash=True
+                repo, stash_rev, stash=True, param_deps=param_deps
             )
             res[entry.baseline_rev][stash_rev] = experiment
 

dvc/repo/experiments/utils.py

@@ -1,9 +1,14 @@
-from typing import TYPE_CHECKING, Generator, Iterable, Set
+from typing import Generator, Iterable, Optional, Set
 
-from .base import EXEC_NAMESPACE, EXPS_NAMESPACE, EXPS_STASH, ExpRefInfo
+from dvc.scm.git import Git
 
-if TYPE_CHECKING:
-    from dvc.scm.git import Git
+from .base import (
+    EXEC_BASELINE,
+    EXEC_NAMESPACE,
+    EXPS_NAMESPACE,
+    EXPS_STASH,
+    ExpRefInfo,
+)
 
 
 def exp_refs(scm: "Git") -> Generator["ExpRefInfo", None, None]:
@@ -102,3 +107,11 @@ def remove_exp_refs(scm: "Git", ref_infos: Iterable["ExpRefInfo"]):
         if exec_checkpoint and exec_checkpoint == ref:
             scm.remove_ref(EXEC_CHECKPOINT)
         scm.remove_ref(str(ref_info))
+
+
+def fix_exp_head(scm: "Git", ref: Optional[str]) -> Optional[str]:
+    if ref:
+        name, tail = Git.split_ref_pattern(ref)
+        if name == "HEAD" and scm.get_ref(EXEC_BASELINE):
+            return "".join((EXEC_BASELINE, tail))
+    return ref

dvc/repo/imp_url.py

@@ -74,9 +74,8 @@ def imp_url(
     if no_exec:
         stage.ignore_outs()
     elif to_remote:
-        stage.outs[0].hash_info = self.cloud.transfer(
-            url, jobs=jobs, remote=remote, command="import-url"
-        )
+        remote = self.cloud.get_remote(remote, "import-url")
+        stage.outs[0].transfer(url, odb=remote.odb, jobs=jobs)
     else:
         stage.run(jobs=jobs)
 

dvc/repo/live.py

@@ -14,14 +14,13 @@ if TYPE_CHECKING:
 
 
 def create_summary(out):
-    from dvc.utils.html import write
-
     assert out.live and out.live["html"]
 
     metrics, plots = out.repo.live.show(str(out.path_info))
 
     html_path = out.path_info.with_suffix(".html")
-    write(html_path, plots, metrics)
+
+    out.repo.plots.write_html(html_path, plots, metrics)
     logger.info(f"\nfile://{os.path.abspath(html_path)}")
 
 

dvc/repo/metrics/diff.py

@@ -1,4 +1,5 @@
 from dvc.exceptions import MetricsError
+from dvc.repo.experiments.utils import fix_exp_head
 from dvc.utils.diff import diff as _diff
 from dvc.utils.diff import format_dict
 
@@ -18,7 +19,8 @@ def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
     with_unchanged = kwargs.pop("all", False)
 
     a_rev = a_rev or "HEAD"
-    b_rev = b_rev or "workspace"
+    a_rev = fix_exp_head(repo.scm, a_rev)
+    b_rev = fix_exp_head(repo.scm, b_rev) or "workspace"
 
     metrics = _get_metrics(repo, *args, **kwargs, revs=[a_rev, b_rev])
     old = metrics.get(a_rev, {})

dvc/repo/params/diff.py

@@ -1,3 +1,4 @@
+from dvc.repo.experiments.utils import fix_exp_head
 from dvc.utils.diff import diff as _diff
 from dvc.utils.diff import format_dict
 
@@ -19,7 +20,8 @@ def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
     with_unchanged = kwargs.pop("all", False)
 
     a_rev = a_rev or "HEAD"
-    b_rev = b_rev or "workspace"
+    a_rev = fix_exp_head(repo.scm, a_rev)
+    b_rev = fix_exp_head(repo.scm, b_rev) or "workspace"
 
     params = _get_params(repo, *args, **kwargs, revs=[a_rev, b_rev])
     old = params.get(a_rev, {})

dvc/repo/params/show.py

@@ -1,6 +1,7 @@
 import logging
 from collections import defaultdict
-from typing import TYPE_CHECKING, List
+from copy import copy
+from typing import TYPE_CHECKING, List, Tuple
 
 from dvc.dependency.param import ParamsDependency
 from dvc.exceptions import DvcException
@@ -27,7 +28,9 @@ def _is_params(dep: "BaseOutput"):
     return isinstance(dep, ParamsDependency)
 
 
-def _collect_configs(repo: "Repo", rev, targets=None) -> List["DvcPath"]:
+def _collect_configs(
+    repo: "Repo", rev, targets=None
+) -> Tuple[List["BaseOutput"], List["DvcPath"]]:
     params, path_infos = collect(
         repo,
         targets=targets or [],
@@ -35,31 +38,45 @@ def _collect_configs(repo: "Repo", rev, targets=None) -> List["DvcPath"]:
         output_filter=_is_params,
         rev=rev,
     )
-    path_infos.extend([p.path_info for p in params])
+    all_path_infos = path_infos + [p.path_info for p in params]
     if not targets:
         default_params = (
             PathInfo(repo.root_dir) / ParamsDependency.DEFAULT_PARAMS_FILE
         )
-        if default_params not in path_infos:
+        if default_params not in all_path_infos:
             path_infos.append(default_params)
-    return list(path_infos)
+    return params, path_infos
 
 
-def _read_params(repo, configs, rev):
-    res = {}
-    for config in configs:
-        if not repo.fs.exists(config):
-            continue
+def _read_path_info(fs, path_info, rev):
+    if not fs.exists(path_info):
+        return None
 
-        suffix = config.suffix.lower()
-        loader = LOADERS[suffix]
-        try:
-            res[str(config)] = loader(config, fs=repo.fs)
-        except ParseError:
-            logger.debug(
-                "failed to read '%s' on '%s'", config, rev, exc_info=True
-            )
-            continue
+    suffix = path_info.suffix.lower()
+    loader = LOADERS[suffix]
+    try:
+        return loader(path_info, fs=fs)
+    except ParseError:
+        logger.debug(
+            "failed to read '%s' on '%s'", path_info, rev, exc_info=True
+        )
+        return None
+
+
+def _read_params(repo, params, params_path_infos, rev, deps=False):
+    res = defaultdict(dict)
+    path_infos = copy(params_path_infos)
+
+    if deps:
+        for param in params:
+            res[str(param.path_info)].update(param.read_params_d())
+    else:
+        path_infos += [param.path_info for param in params]
+
+    for path_info in path_infos:
+        from_path = _read_path_info(repo.fs, path_info, rev)
+        if from_path:
+            res[str(path_info)] = from_path
 
     return res
 
@@ -79,12 +96,12 @@ def _collect_vars(repo, params):
 
 
 @locked
-def show(repo, revs=None, targets=None):
+def show(repo, revs=None, targets=None, deps=False):
     res = {}
 
     for branch in repo.brancher(revs=revs):
-        configs = _collect_configs(repo, branch, targets)
-        params = _read_params(repo, configs, branch)
+        params, params_path_infos = _collect_configs(repo, branch, targets)
+        params = _read_params(repo, params, params_path_infos, branch, deps)
         vars_params = _collect_vars(repo, params)
 
         # NOTE: only those that are not added as a ParamDependency are included

dvc/repo/plots/__init__.py

@@ -1,5 +1,6 @@
 import logging
-from typing import TYPE_CHECKING, Dict, List
+import os
+from typing import TYPE_CHECKING, Dict, List, Optional
 
 from funcy import cached_property, first, project
 
@@ -9,6 +10,7 @@ from dvc.exceptions import (
     NoMetricsFoundError,
     NoMetricsParsedError,
 )
+from dvc.types import StrPath
 from dvc.utils import relpath
 
 if TYPE_CHECKING:
@@ -219,6 +221,26 @@ class Plots:
 
         return PlotTemplates(self.repo.dvc_dir)
 
+    def write_html(
+        self,
+        path: StrPath,
+        plots: Dict[str, Dict],
+        metrics: Optional[Dict[str, Dict]] = None,
+        html_template_path: Optional[StrPath] = None,
+    ):
+        if not html_template_path:
+            html_template_path = self.repo.config.get("plots", {}).get(
+                "html_template", None
+            )
+            if html_template_path and not os.path.isabs(html_template_path):
+                html_template_path = os.path.join(
+                    self.repo.dvc_dir, html_template_path
+                )
+
+        from dvc.utils.html import write
+
+        write(path, plots, metrics, html_template_path)
+
 
 def _is_plot(out: "BaseOutput") -> bool:
     return bool(out.plot) or bool(out.live)

dvc/repo/plots/diff.py

@@ -1,3 +1,6 @@
+from dvc.repo.experiments.utils import fix_exp_head
+
+
 def _revisions(repo, revs, experiment):
     revisions = revs or []
     if experiment and len(revisions) == 1:
@@ -6,7 +9,7 @@ def _revisions(repo, revs, experiment):
             revisions.append(baseline[:7])
     if len(revisions) <= 1:
         if len(revisions) == 0 and repo.scm.is_dirty():
-            revisions.append("HEAD")
+            revisions.append(fix_exp_head(repo.scm, "HEAD"))
         revisions.append("workspace")
     return revisions
 

dvc/scm/git/__init__.py

@@ -7,7 +7,7 @@ import shlex
 from collections.abc import Mapping
 from contextlib import contextmanager
 from functools import partialmethod
-from typing import Dict, Iterable, List, Optional, Set, Type
+from typing import Dict, Iterable, List, Optional, Set, Tuple, Type
 
 from funcy import cached_property, first
 from pathspec.patterns import GitWildMatchPattern
@@ -77,6 +77,7 @@ class Git(Base):
     GIT_DIR = ".git"
     LOCAL_BRANCH_PREFIX = "refs/heads/"
     RE_HEXSHA = re.compile(r"^[0-9A-Fa-f]{4,40}$")
+    BAD_REF_CHARS_RE = re.compile("[\177\\s~^:?*\\[]")
 
     def __init__(
         self, *args, backends: Optional[Iterable[str]] = None, **kwargs
@@ -123,6 +124,11 @@ class Git(Base):
     def is_sha(cls, rev):
         return rev and cls.RE_HEXSHA.search(rev)
 
+    @classmethod
+    def split_ref_pattern(cls, ref: str) -> Tuple[str, str]:
+        name = cls.BAD_REF_CHARS_RE.split(ref, maxsplit=1)[0]
+        return name, ref[len(name) :]
+
     @staticmethod
     def _get_git_dir(root_dir):
         return os.path.join(root_dir, Git.GIT_DIR)
@@ -145,7 +151,9 @@ class Git(Base):
         gitignore = os.path.join(ignore_file_dir, self.GITIGNORE)
 
         if not path_isin(os.path.realpath(gitignore), self.root_dir):
-            raise FileNotInRepoError(path)
+            raise FileNotInRepoError(
+                f"'{path}' is outside of git repository '{self.root_dir}'"
+            )
 
         return entry, gitignore
 
@@ -418,7 +426,7 @@ class Git(Base):
             commit = self.resolve_commit(parent)
 
     @contextmanager
-    def detach_head(self, rev: Optional[str] = None):
+    def detach_head(self, rev: Optional[str] = None, force: bool = False):
         """Context manager for performing detached HEAD SCM operations.
 
         Detaches and restores HEAD similar to interactive git rebase.
@@ -432,7 +440,7 @@ class Git(Base):
             rev = "HEAD"
         orig_head = self.get_ref("HEAD", follow=False)
         logger.debug("Detaching HEAD at '%s'", rev)
-        self.checkout(rev, detach=True)
+        self.checkout(rev, detach=True, force=force)
         try:
             yield self.get_ref("HEAD")
         finally:

dvc/scm/git/backend/dulwich.py

@@ -182,11 +182,16 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
     def commit(self, msg: str, no_verify: bool = False):
         from dulwich.errors import CommitError
         from dulwich.porcelain import commit
+        from dulwich.repo import InvalidUserIdentity
 
         try:
             commit(self.root_dir, message=msg, no_verify=no_verify)
         except CommitError as exc:
             raise SCMError("Git commit failed") from exc
+        except InvalidUserIdentity as exc:
+            raise SCMError(
+                "Git username and email must be configured"
+            ) from exc
 
     def checkout(
         self,
@@ -528,6 +533,8 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
         message: Optional[str] = None,
         include_untracked: Optional[bool] = False,
     ) -> Tuple[Optional[str], bool]:
+        from dulwich.repo import InvalidUserIdentity
+
         from dvc.scm.git import Stash
 
         if include_untracked or ref == Stash.DEFAULT_STASH:
@@ -537,7 +544,12 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
 
         stash = self._get_stash(ref)
         message_b = message.encode("utf-8") if message else None
-        rev = stash.push(message=message_b)
+        try:
+            rev = stash.push(message=message_b)
+        except InvalidUserIdentity as exc:
+            raise SCMError(
+                "Git username and email must be configured"
+            ) from exc
         return os.fsdecode(rev), True
 
     def _stash_apply(self, rev: str):

dvc/scm/git/backend/pygit2.py

@@ -2,6 +2,7 @@ import locale
 import logging
 import os
 import stat
+from contextlib import contextmanager
 from io import BytesIO, StringIO
 from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union
 
@@ -80,6 +81,26 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
 
         return RefdbFsBackend(self.repo)
 
+    @property
+    def default_signature(self):
+        try:
+            return self.repo.default_signature
+        except KeyError as exc:
+            raise SCMError(
+                "Git username and email must be configured"
+            ) from exc
+
+    # Workaround to force git_backend_odb_pack to release open file handles
+    # in DVC's mixed git-backend environment.
+    # See https://github.com/iterative/dvc/issues/5641
+    @contextmanager
+    def release_odb_handles(self):
+        yield
+        # It is safe to free the libgit2 repo context multiple times - free
+        # just forces libgit/pygit to release git ODB related contexts which
+        # can be reacquired later as needed.
+        self.repo.free()
+
     @staticmethod
     def clone(
         url: str,
@@ -110,23 +131,24 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
 
         checkout_strategy = GIT_CHECKOUT_FORCE if force else None
 
-        if create_new:
-            commit = self.repo.revparse_single("HEAD")
-            new_branch = self.repo.branches.local.create(branch, commit)
-            self.repo.checkout(new_branch, strategy=checkout_strategy)
-        else:
-            if branch == "-":
-                branch = "@{-1}"
-            try:
-                commit, ref = self.repo.resolve_refish(branch)
-            except (KeyError, GitError):
-                raise RevError(f"unknown Git revision '{branch}'")
-            self.repo.checkout_tree(commit, strategy=checkout_strategy)
-            detach = kwargs.get("detach", False)
-            if ref and not detach:
-                self.repo.set_head(ref.name)
+        with self.release_odb_handles():
+            if create_new:
+                commit = self.repo.revparse_single("HEAD")
+                new_branch = self.repo.branches.local.create(branch, commit)
+                self.repo.checkout(new_branch, strategy=checkout_strategy)
             else:
-                self.repo.set_head(commit.id)
+                if branch == "-":
+                    branch = "@{-1}"
+                try:
+                    commit, ref = self.repo.resolve_refish(branch)
+                except (KeyError, GitError):
+                    raise RevError(f"unknown Git revision '{branch}'")
+                self.repo.checkout_tree(commit, strategy=checkout_strategy)
+                detach = kwargs.get("detach", False)
+                if ref and not detach:
+                    self.repo.set_head(ref.name)
+                else:
+                    self.repo.set_head(commit.id)
 
     def pull(self, **kwargs):
         raise NotImplementedError
@@ -328,7 +350,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         from dvc.scm.git import Stash
 
         oid = self.repo.stash(
-            self.repo.default_signature,
+            self.default_signature,
             message=message,
             include_untracked=include_untracked,
         )
@@ -450,24 +472,26 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
                 ]
         else:
             path_list = None
-        self.repo.checkout_index(
-            index=index, paths=path_list, strategy=strategy,
-        )
 
-        if index.conflicts and (ours or theirs):
-            for ancestor, ours_entry, theirs_entry in index.conflicts:
-                if not ancestor:
-                    continue
-                if ours:
-                    entry = ours_entry
-                    index.add(ours_entry)
-                else:
-                    entry = theirs_entry
-                path = os.path.join(self.root_dir, entry.path)
-                with open(path, "wb") as fobj:
-                    fobj.write(self.repo.get(entry.id).read_raw())
-                index.add(entry.path)
-            index.write()
+        with self.release_odb_handles():
+            self.repo.checkout_index(
+                index=index, paths=path_list, strategy=strategy,
+            )
+
+            if index.conflicts and (ours or theirs):
+                for ancestor, ours_entry, theirs_entry in index.conflicts:
+                    if not ancestor:
+                        continue
+                    if ours:
+                        entry = ours_entry
+                        index.add(ours_entry)
+                    else:
+                        entry = theirs_entry
+                    path = os.path.join(self.root_dir, entry.path)
+                    with open(path, "wb") as fobj:
+                        fobj.write(self.repo.get(entry.id).read_raw())
+                    index.add(entry.path)
+                index.write()
 
     def iter_remote_refs(self, url: str, base: Optional[str] = None):
         raise NotImplementedError
@@ -492,25 +516,26 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         if commit and not msg:
             raise SCMError("Merge commit message is required")
 
-        try:
-            self.repo.index.read(False)
-            self.repo.merge(rev)
-            self.repo.index.write()
-        except GitError as exc:
-            raise SCMError("Merge failed") from exc
+        with self.release_odb_handles():
+            try:
+                self.repo.index.read(False)
+                self.repo.merge(rev)
+                self.repo.index.write()
+            except GitError as exc:
+                raise SCMError("Merge failed") from exc
 
-        if self.repo.index.conflicts:
-            raise MergeConflictError("Merge contained conflicts")
+            if self.repo.index.conflicts:
+                raise MergeConflictError("Merge contained conflicts")
 
-        if commit:
-            user = self.repo.default_signature
-            tree = self.repo.index.write_tree()
-            merge_commit = self.repo.create_commit(
-                "HEAD", user, user, msg, tree, [self.repo.head.target, rev]
-            )
-            return str(merge_commit)
-        if squash:
-            self.repo.reset(self.repo.head.target, GIT_RESET_MIXED)
-            self.repo.state_cleanup()
-            self.repo.index.write()
+            if commit:
+                user = self.default_signature
+                tree = self.repo.index.write_tree()
+                merge_commit = self.repo.create_commit(
+                    "HEAD", user, user, msg, tree, [self.repo.head.target, rev]
+                )
+                return str(merge_commit)
+            if squash:
+                self.repo.reset(self.repo.head.target, GIT_RESET_MIXED)
+                self.repo.state_cleanup()
+                self.repo.index.write()
         return None

dvc/stage/__init__.py

@@ -433,15 +433,21 @@ class Stage(params.StageParams):
         return m
 
     def save(self, allow_missing=False):
-        self.save_deps()
+        self.save_deps(allow_missing=allow_missing)
         self.save_outs(allow_missing=allow_missing)
         self.md5 = self.compute_md5()
 
         self.repo.stage_cache.save(self)
 
-    def save_deps(self):
+    def save_deps(self, allow_missing=False):
+        from dvc.dependency.base import DependencyDoesNotExistError
+
         for dep in self.deps:
-            dep.save()
+            try:
+                dep.save()
+            except DependencyDoesNotExistError:
+                if not allow_missing:
+                    raise
 
     def save_outs(self, allow_missing=False):
         from dvc.output.base import OutputDoesNotExistError

dvc/stage/imports.py

@@ -13,9 +13,8 @@ def _update_import_on_remote(stage, remote, jobs):
         )
 
     url = stage.deps[0].def_path
-    stage.outs[0].hash_info = stage.repo.cloud.transfer(
-        url, jobs=jobs, remote=remote, command="update"
-    )
+    remote = stage.repo.cloud.get_remote(remote, "update")
+    stage.outs[0].transfer(url, odb=remote.odb, jobs=jobs, update=True)
 
 
 def update_import(stage, rev=None, to_remote=False, remote=None, jobs=None):

dvc/types.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 from dvc.path_info import PathInfo, URLInfo
 
@@ -11,3 +11,5 @@ AnyPath = Union[str, DvcPath, StrPath]
 
 OptStr = Optional[str]
 TargetType = Union[List[str], str]
+DictStrAny = Dict[str, Any]
+DictAny = Dict[Any, Any]

dvc/ui/__init__.py

@@ -1,14 +1,21 @@
 import sys
 from collections import defaultdict
-from typing import Any, Dict, Iterable, Optional, TextIO
-
-from funcy import cached_property
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterable,
+    Optional,
+    Sequence,
+    TextIO,
+    Union,
+)
 
 from dvc.progress import Tqdm
 from dvc.utils import colorize
 
-NEWLINE = "\n"
-SEP = " "
+if TYPE_CHECKING:
+    from dvc.ui.table import Headers, Styles, TableData
 
 
 class Formatter:
@@ -30,14 +37,23 @@ class Console:
         formatter: Formatter = None,
         output: TextIO = None,
         error: TextIO = None,
-        disable: bool = False,
+        enable: bool = False,
     ) -> None:
-        self._input: TextIO = sys.stdin
-        self._output: TextIO = output or sys.stdout
-        self._error: TextIO = error or sys.stderr
-
+        self._output: Optional[TextIO] = output
+        self._error: Optional[TextIO] = error
         self.formatter: Formatter = formatter or Formatter()
-        self.disabled: bool = disable
+        self._enabled: bool = enable
+
+    @property
+    def output(self) -> TextIO:
+        return self._output or sys.stdout
+
+    @property
+    def error_output(self) -> TextIO:
+        return self._error or sys.stderr
+
+    def enable(self) -> None:
+        self._enabled = True
 
     def success(self, message: str) -> None:
         self.write(message, style="success")
@@ -52,8 +68,8 @@ class Console:
         self,
         *objects: Any,
         style: str = None,
-        sep: str = SEP,
-        end: str = NEWLINE,
+        sep: str = None,
+        end: str = None,
         flush: bool = False,
     ) -> None:
         return self.write(
@@ -61,7 +77,7 @@ class Console:
             style=style,
             sep=sep,
             end=end,
-            file=self._error,
+            file=self.error_output,
             flush=flush,
         )
 
@@ -69,20 +85,21 @@ class Console:
         self,
         *objects: Any,
         style: str = None,
-        sep: str = SEP,
-        end: str = NEWLINE,
+        sep: str = None,
+        end: str = None,
         file: TextIO = None,
         flush: bool = False,
+        force: bool = False,
     ) -> None:
-        if self.disabled:
+        if not self._enabled and not force:
             return
 
-        file = file or self._output
+        file = file or self.output
         values = (self.formatter.format(obj, style=style) for obj in objects)
         return print(*values, sep=sep, end=end, file=file, flush=flush)
 
     def progress(self, *args, **kwargs) -> Tqdm:
-        kwargs.setdefault("file", self._error)
+        kwargs.setdefault("file", self.error_output)
         return Tqdm(*args, **kwargs)
 
     def prompt(
@@ -120,47 +137,64 @@ class Console:
             return False
         return answer.startswith("y")
 
-    @cached_property
+    @property
     def rich_console(self):
         """rich_console is only set to stdout for now."""
         from rich import console
 
-        return console.Console(file=self._output)
+        # FIXME: Getting IO Operation on closed file error
+        #  when testing with capsys, therefore we are creating
+        #  one instance each time as a temporary workaround.
+        return console.Console(file=self.output)
 
-    def rich_table(self, pager: bool = True):
-        pass
+    def table(
+        self,
+        data: "TableData",
+        headers: "Headers" = None,
+        markdown: bool = False,
+        rich_table: bool = False,
+        force: bool = True,
+        pager: bool = False,
+        header_styles: Union[Dict[str, "Styles"], Sequence["Styles"]] = None,
+        row_styles: Sequence["Styles"] = None,
+        borders: Union[bool, str] = False,
+    ) -> None:
+        from dvc.ui import table as t
 
-    def table(self, header, rows, markdown: bool = False):
-        from tabulate import tabulate
+        if not data and not markdown:
+            return
 
-        if not rows and not markdown:
-            return ""
+        if not markdown and rich_table:
+            if force or self._enabled:
+                return t.rich_table(
+                    self,
+                    data,
+                    headers,
+                    pager=pager,
+                    header_styles=header_styles,
+                    row_styles=row_styles,
+                    borders=borders,
+                )
 
-        ret = tabulate(
-            rows,
-            header,
-            tablefmt="github" if markdown else "plain",
-            disable_numparse=True,
-            # None will be shown as "" by default, overriding
-            missingval="—",
+            return
+
+        return t.plain_table(
+            self, data, headers, markdown=markdown, pager=pager, force=force,
         )
 
-        if markdown:
-            # NOTE: md table is incomplete without the trailing newline
-            ret += "\n"
 
-        self.write(ret)
+ui = Console()
 
 
 if __name__ == "__main__":
-    ui = Console()
+    ui.enable()
 
     ui.write("No default remote set")
     ui.success("Everything is up to date.")
     ui.warn("Run queued experiments will be removed.")
     ui.error("too few arguments.")
 
-    ui.table("keys", {"Path": ["scores.json"], "auc": ["0.5674"]})
+    ui.table([("scores.json", "0.5674")], headers=["Path", "auc"])
     ui.table(
-        "keys", {"Path": ["scores.json"], "auc": ["0.5674"]}, markdown=True
+        [("scores.json", "0.5674")], headers=["Path", "auc"], markdown=True
     )

dvc/ui/table.py

@@ -0,0 +1,124 @@
+from collections import abc
+from contextlib import contextmanager
+from itertools import zip_longest
+from typing import TYPE_CHECKING, Dict, Iterator, Sequence, Union
+
+from dvc.types import DictStrAny
+
+if TYPE_CHECKING:
+    from rich.console import Console as RichConsole
+    from rich.table import Table
+    from rich.text import Text
+
+    from dvc.ui import Console
+
+
+SHOW_MAX_WIDTH = 1024
+
+
+CellT = Union[str, "Text"]  # Text is mostly compatible with str
+Row = Sequence[CellT]
+TableData = Sequence[Row]
+Headers = Sequence[str]
+Styles = DictStrAny
+
+
+def plain_table(
+    ui: "Console",
+    data: TableData,
+    headers: Headers = None,
+    markdown: bool = False,
+    pager: bool = False,
+    force: bool = True,
+) -> None:
+    from tabulate import tabulate
+
+    text: str = tabulate(
+        data,
+        headers if headers is not None else (),
+        tablefmt="github" if markdown else "plain",
+        disable_numparse=True,
+        # None will be shown as "" by default, overriding
+        missingval="-",
+    )
+    if markdown:
+        # NOTE: md table is incomplete without the trailing newline
+        text += "\n"
+
+    if pager:
+        from dvc.utils.pager import pager as _pager
+
+        _pager(text)
+    else:
+        ui.write(text, force=force)
+
+
+@contextmanager
+def console_width(
+    table: "Table", console: "RichConsole", val: int
+) -> Iterator[None]:
+    # NOTE: rich does not have native support for unlimited width
+    # via pager. we override rich table compression by setting
+    # console width to the full width of the table
+    # pylint: disable=protected-access
+
+    console_options = console.options
+    original = console_options.max_width
+    con_width = console._width
+
+    try:
+        console_options.max_width = val
+        measurement = table.__rich_measure__(console, console_options)
+        console._width = measurement.maximum
+
+        yield
+    finally:
+        console_options.max_width = original
+        console._width = con_width
+
+
+def rich_table(
+    ui: "Console",
+    data: TableData,
+    headers: Headers = None,
+    pager: bool = False,
+    header_styles: Union[Dict[str, Styles], Sequence[Styles]] = None,
+    row_styles: Sequence[Styles] = None,
+    borders: Union[bool, str] = False,
+) -> None:
+    from rich import box
+
+    from dvc.utils.table import Table
+
+    border_style = {
+        True: box.HEAVY_HEAD,  # is a default in rich,
+        False: None,
+        "simple": box.SIMPLE,
+        "minimal": box.MINIMAL,
+    }
+
+    table = Table(box=border_style[borders])
+
+    if isinstance(header_styles, abc.Sequence):
+        hs: Dict[str, Styles] = dict(zip(headers or [], header_styles))
+    else:
+        hs = header_styles or {}
+
+    for header in headers or []:
+        table.add_column(header, **hs.get(header, {}))
+
+    rs: Sequence[Styles] = row_styles or []
+    for row, style in zip_longest(data, rs):
+        table.add_row(*row, **(style or {}))
+
+    console = ui.rich_console
+
+    if not pager:
+        console.print(table)
+        return
+
+    from dvc.utils.pager import DvcPager
+
+    with console_width(table, console, SHOW_MAX_WIDTH):
+        with console.pager(pager=DvcPager(), styles=True):
+            console.print(table)

dvc/utils/diff.py

@@ -85,28 +85,6 @@ def diff(old, new, with_unchanged=False):
     return dict(res)
 
 
-def table(header, rows, markdown=False):
-    from tabulate import tabulate
-
-    if not rows and not markdown:
-        return ""
-
-    ret = tabulate(
-        rows,
-        header,
-        tablefmt="github" if markdown else "plain",
-        disable_numparse=True,
-        # None will be shown as "" by default, overriding
-        missingval="—",
-    )
-
-    if markdown:
-        # NOTE: md table is incomplete without the trailing newline
-        ret += "\n"
-
-    return ret
-
-
 def format_dict(d):
     ret = {}
     for key, val in d.items():

dvc/utils/html.py

@@ -1,5 +1,8 @@
 from typing import Dict, List, Optional
 
+from dvc.exceptions import DvcException
+from dvc.types import StrPath
+
 PAGE_HTML = """<!DOCTYPE html>
 <html>
 <head>
@@ -9,7 +12,7 @@ PAGE_HTML = """<!DOCTYPE html>
     <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
 </head>
 <body>
-    {divs}
+    {plot_divs}
 </body>
 </html>"""
 
@@ -20,9 +23,22 @@ VEGA_DIV_HTML = """<div id = "{id}"></div>
 </script>"""
 
 
+class MissingPlaceholderError(DvcException):
+    def __init__(self, placeholder):
+        super().__init__(f"HTML template has to contain '{placeholder}'.")
+
+
 class HTML:
-    def __init__(self):
-        self.elements = []
+    PLACEHOLDER = "plot_divs"
+    PLACEHOLDER_FORMAT_STR = f"{{{PLACEHOLDER}}}"
+
+    def __init__(self, template: str = None):
+        template = template or PAGE_HTML
+        if self.PLACEHOLDER_FORMAT_STR not in template:
+            raise MissingPlaceholderError(self.PLACEHOLDER_FORMAT_STR)
+
+        self.template = template
+        self.elements: List[str] = []
 
     def with_metrics(self, metrics: Dict[str, Dict]) -> "HTML":
         import tabulate
@@ -54,13 +70,22 @@ class HTML:
         return self
 
     def embed(self) -> str:
-        return PAGE_HTML.format(divs="\n".join(self.elements))
+        kwargs = {self.PLACEHOLDER: "\n".join(self.elements)}
+        return self.template.format(**kwargs)
 
 
 def write(
-    path, plots: Dict[str, Dict], metrics: Optional[Dict[str, Dict]] = None
+    path: StrPath,
+    plots: Dict[str, Dict],
+    metrics: Optional[Dict[str, Dict]] = None,
+    template_path: Optional[StrPath] = None,
 ):
-    document = HTML()
+    page_html = None
+    if template_path:
+        with open(template_path, "r") as fobj:
+            page_html = fobj.read()
+
+    document = HTML(page_html)
     if metrics:
         document.with_metrics(metrics)
         document.with_element("<br>")

dvc/utils/pager.py

@@ -46,7 +46,7 @@ def find_pager():
     return pydoc.plainpager
 
 
-def pager(text):
+def pager(text: str) -> None:
     find_pager()(text)
 
 

dvc/utils/table.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING, List, cast
 
 from rich.style import StyleType
 from rich.table import Column as RichColumn
@@ -8,6 +8,7 @@ from rich.table import Table as RichTable
 if TYPE_CHECKING:
     from rich.console import (
         Console,
+        ConsoleOptions,
         JustifyMethod,
         OverflowMethod,
         RenderableType,
@@ -56,7 +57,7 @@ class Table(RichTable):
         self.columns.append(column)
 
     def _calculate_column_widths(
-        self, console: "Console", max_width: int
+        self, console: "Console", options: "ConsoleOptions"
     ) -> List[int]:
         """Calculate the widths of each column, including padding, not
         including borders.
@@ -64,22 +65,23 @@ class Table(RichTable):
         Adjacent collapsed columns will be removed until there is only a single
         truncated column remaining.
         """
-        widths = super()._calculate_column_widths(console, max_width)
+        widths = super()._calculate_column_widths(console, options)
         last_collapsed = -1
-        for i in range(len(self.columns) - 1, -1, -1):
-            if widths[i] == 1 and self.columns[i].collapse:
+        columns = cast(List[Column], self.columns)
+        for i in range(len(columns) - 1, -1, -1):
+            if widths[i] == 0 and columns[i].collapse:
                 if last_collapsed >= 0:
                     del widths[last_collapsed]
-                    del self.columns[last_collapsed]
+                    del columns[last_collapsed]
                     if self.box:
-                        max_width += 1
-                    for column in self.columns[last_collapsed:]:
+                        options.max_width += 1
+                    for column in columns[last_collapsed:]:
                         column._index -= 1
                 last_collapsed = i
                 padding = self._get_padding_width(i)
                 if (
-                    self.columns[i].overflow == "ellipsis"
-                    and (sum(widths) + padding) <= max_width
+                    columns[i].overflow == "ellipsis"
+                    and (sum(widths) + padding) <= options.max_width
                 ):
                     # Set content width to 1 (plus padding) if we can fit a
                     # single unicode ellipsis in this column
@@ -88,7 +90,7 @@ class Table(RichTable):
                 last_collapsed = -1
         return widths
 
-    def _collapse_widths(
+    def _collapse_widths(  # type: ignore[override]
         self, widths: List[int], wrapable: List[bool], max_width: int,
     ) -> List[int]:
         """Collapse columns right-to-left if possible to fit table into
@@ -97,7 +99,8 @@ class Table(RichTable):
         If table is still too wide after collapsing, rich's automatic overflow
         handling will be used.
         """
-        collapsible = [column.collapse for column in self.columns]
+        columns = cast(List[Column], self.columns)
+        collapsible = [column.collapse for column in columns]
         total_width = sum(widths)
         excess_width = total_width - max_width
         if any(collapsible):

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.0.7"
+_BASE_VERSION = "2.1.0"
 
 
 def _generate_version(base_version):

scripts/build-requirements.txt

@@ -1,2 +1,2 @@
-PyInstaller==3.5
+PyInstaller==4.3
 numpy==1.19.3

scripts/build.py

@@ -0,0 +1,103 @@
+import argparse
+import importlib
+import pathlib
+import textwrap
+from subprocess import STDOUT, check_call
+
+path = pathlib.Path(__file__).parent.absolute()
+dvc = path.parent / "dvc"
+pyinstaller = path / "pyinstaller"
+innosetup = path / "innosetup"
+fpm = path / "fpm"
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "pkg", choices=["deb", "rpm", "osxpkg", "exe"], help="package type"
+)
+parser.add_argument("--sign-application", default=False, action="store_true")
+parser.add_argument("--application-id")
+parser.add_argument("--sign-installer", default=False, action="store_true")
+parser.add_argument("--installer-id")
+parser.add_argument("--notarize", default=False, action="store_true")
+parser.add_argument("--apple-id-username")
+parser.add_argument("--apple-id-password")
+args = parser.parse_args()
+
+(dvc / "utils" / "build.py").write_text(f'PKG = "{args.pkg}"')
+
+# Autogenerate version, similar to what we do in setup.py
+spec = importlib.util.spec_from_file_location(
+    "dvc.version", dvc / "version.py"
+)
+dvc_version = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(dvc_version)
+version = dvc_version.__version__
+
+(dvc / "version.py").write_text(
+    textwrap.dedent(
+        f"""\
+        # AUTOGENERATED at build time by scripts/build.py
+        __version__ = "{version}"
+    """
+    )
+)
+
+check_call(
+    ["python", "build.py"], cwd=pyinstaller, stderr=STDOUT,
+)
+
+if args.sign_application:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not args.application_id:
+        print("--sign-application requires --application-id")
+        exit(1)
+    check_call(
+        ["python", "sign.py", "--application-id", args.application_id],
+        cwd=pyinstaller,
+        stderr=STDOUT,
+    )
+
+if args.pkg == "exe":
+    check_call(
+        ["python", "build.py"], cwd=innosetup, stderr=STDOUT,
+    )
+else:
+    check_call(
+        ["python", "build.py", args.pkg], cwd=fpm, stderr=STDOUT,
+    )
+
+if args.sign_installer:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not all(
+        [args.installer_id, args.apple_id_username, args.apple_id_password]
+    ):
+        print("--sign-installer requires --installer-id")
+        exit(1)
+    check_call(
+        ["python", "sign.py", "--installer-id", args.installer_id],
+        cwd=fpm,
+        stderr=STDOUT,
+    )
+
+if args.notarize:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not all([args.apple_id_username, args.apple_id_password]):
+        print(
+            "--notarize requires --apple-id-username and --apple-id-password"
+        )
+        exit(1)
+    check_call(
+        [
+            "python",
+            "notarize.py",
+            "--apple-id-username",
+            args.apple_id_username,
+            "--apple-id-password",
+            args.apple_id_password,
+        ],
+        cwd=fpm,
+        stderr=STDOUT,
+    )

scripts/build_posix.sh

@@ -1,139 +0,0 @@
-#!/bin/bash
-
-set -e
-set -x
-
-FPM_FLAGS='--description "Data Version Control - datasets, models, and experiments versioning for ML or data science projects"'
-FPM_FLAGS+=' -n dvc -s dir -f'
-FPM_FLAGS+=' --url https://dvc.org'
-FPM_FLAGS+=' --license "Apache License 2.0"'
-
-if [[ "$(uname)" == 'Linux' ]]; then
-  INSTALL_DIR=usr
-  FPM_FLAGS+=' --depends "git >= 1.7.0"'  # needed for gitpython
-else
-  INSTALL_DIR=usr/local
-  FPM_FLAGS+=' --osxpkg-identifier-prefix com.iterative'
-  FPM_FLAGS+=' --after-install scripts/fpm/after-install.sh'
-  FPM_FLAGS+=' --after-remove scripts/fpm/after-remove.sh'
-fi
-
-BUILD_DIR=build
-BIN_DIR=$BUILD_DIR/$INSTALL_DIR/bin
-LIB_DIR=$BUILD_DIR/$INSTALL_DIR/lib
-
-FPM_PACKAGE_DIRS="usr"
-ZSH_CMPLT_DIR=usr/share/zsh/site-functions/_dvc
-if [[ "$(uname)" == 'Linux' ]]; then
-  BASH_CMPLT_DIR=etc/bash_completion.d
-  FPM_PACKAGE_DIRS="$FPM_PACKAGE_DIRS etc"
-else
-  BASH_CMPLT_DIR=usr/local/etc/bash_completion.d
-fi
-
-print_error() {
-  echo -e "\e[31m$1\e[0m" >&2
-}
-
-if [ ! -d "dvc" ]; then
-  print_error "Please run this script from repository root"
-  exit 1
-fi
-
-trap 'print_error "FAIL"; exit 1' ERR
-
-print_info() {
-  echo -e "\e[32m$1\e[0m"
-}
-
-command_exists() {
-  command -v $1 >/dev/null 2>&1
-}
-
-fpm_build() {
-  print_info "Building $1..."
-  VERSION=$(python -c "import dvc; from dvc import __version__; print(str(__version__))")
-  eval "fpm -t $1 $FPM_FLAGS -v $VERSION -C $BUILD_DIR $FPM_PACKAGE_DIRS"
-}
-
-cleanup() {
-  print_info "Cleaning up..."
-  rm -rf build
-}
-
-install_dependencies() {
-  print_info "Installing fpm..."
-  if command_exists dnf; then
-    sudo dnf install ruby-devel gcc make rpm-build
-  elif command_exists yum; then
-    sudo yum install ruby-devel gcc make rpm-build
-  elif command_exists apt-get; then
-    sudo apt-get update -y
-    sudo apt-get install ruby-dev build-essential rpm python-pip python-dev
-  elif command_exists brew; then
-    brew install ruby
-  else
-    echo "Unable to install fpm dependencies" && exit 1
-  fi
-
-  gem install --no-document fpm
-
-  # https://github.com/iterative/dvc/issues/2970
-  gem uninstall -i /Users/travis/.rvm/gems/ruby-2.4.3@global rubygems-bundler
-
-  print_info "Installing requirements..."
-  pip install .[all]
-  pip install -r scripts/build-requirements.txt
-}
-
-build_dvc() {
-  print_info "Building dvc binary..."
-  pyinstaller \
-    --additional-hooks-dir $(pwd)/scripts/hooks dvc/__main__.py \
-    --name dvc \
-    --distpath $LIB_DIR \
-    --specpath $BUILD_DIR
-
-  $LIB_DIR/dvc/dvc --help
-
-  # NOTE: in osxpkg fpm replaces symlinks with actual file that it
-  # points to, so we need to use after-install hook. See FPM_FLAGS
-  # above.
-  if [[ "$(uname)" == 'Linux' ]]; then
-    mkdir -p $BIN_DIR
-    pushd $BIN_DIR
-    ln -s ../lib/dvc/dvc dvc
-    popd
-    $BIN_DIR/dvc --help
-  fi
-
-  # NOTE: temporarily not adding scripts to mac package. See [1]
-  # [1] https://github.com/iterative/dvc/issues/2585
-  if [[ "$(uname)" == 'Linux' ]]; then
-    mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR
-    $LIB_DIR/dvc/dvc completion -s bash > $BUILD_DIR/$BASH_CMPLT_DIR/dvc
-
-    mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR
-    $LIB_DIR/dvc/dvc completion -s zsh > $BUILD_DIR/$ZSH_CMPLT_DIR/_dvc
-  fi
-}
-
-build() {
-  cleanup
-  echo "PKG = \"$1\"" >dvc/utils/build.py
-  build_dvc
-  fpm_build $1
-}
-
-install_dependencies
-
-if [[ "$(uname)" == 'Linux' ]]; then
-  build rpm
-  build deb
-else
-  build osxpkg
-fi
-
-cleanup
-
-print_info "Successfully built dvc packages"

scripts/build_windows.cmd

@@ -1,48 +0,0 @@
-@echo Off
-
-echo ====== Starting to build dvc installer for Windows... ======
-
-if not exist dvc\NUL (echo Error: Please run this script from repository root && goto :error)
-
-rmdir /Q /S build
-rmdir /Q /S dist
-del /Q /S dvc.spec
-del /Q /S "dvc-*.exe"
-
-where pip
-if %errorlevel% neq 0 (echo Error: pip not found && goto :error)
-
-where choco
-if %errorlevel% neq 0 (echo Error: choco not found && goto :error)
-
-choco install InnoSetup
-call refreshenv
-where iscc
-if %errorlevel% neq 0 (echo Error: Couldn't find Inno Setup compiler. && goto :error)
-
-echo ====== Installing requirements... ======
-echo PKG = "exe" > dvc\utils\build.py
-call pip install .[all] || goto :error
-call pip install -r scripts\build-requirements.txt || goto :error
-call dvc pull || goto :error
-
-echo ====== Building dvc binary... ======
-call pyinstaller --additional-hooks-dir scripts\hooks dvc/__main__.py --name dvc --specpath build
-
-echo ====== Testing dvc binary... ======
-call dist\dvc\dvc.exe version || goto :error
-
-echo ====== Copying additional files... ======
-copy scripts\innosetup\addSymLinkPermissions.ps1 dist\ || goto :error
-
-echo ====== Building dvc installer... ======
-set PYTHONPATH=%cd%
-call python scripts\innosetup\config_gen.py || goto :error
-call iscc scripts\innosetup\setup.iss || goto :error
-
-echo ====== DONE ======
-goto :EOF
-
-:error
-echo ====== FAIL ======
-exit /b 1

scripts/fpm/.gitignore

@@ -0,0 +1,4 @@
+/build
+/*.deb
+/*.rpm
+/*.pkg

scripts/fpm/build.py

@@ -0,0 +1,96 @@
+import argparse
+import os
+import pathlib
+import shutil
+from subprocess import STDOUT, check_call, check_output
+
+path = pathlib.Path(__file__).parent.absolute()
+dvc = path.parent.parent / "dvc"
+pyinstaller = path.parent / "pyinstaller"
+
+build = path / "build"
+install = build / "usr"
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "pkg", choices=["deb", "rpm", "osxpkg"], help="package type"
+)
+args = parser.parse_args()
+
+flags = [
+    "--description",
+    '"Data Version Control | Git for Data & Models"',
+    "-n",
+    "dvc",
+    "-s",
+    "dir",
+    "-f",
+    "--license",
+    '"Apache License 2.0"',
+]
+
+if args.pkg == "osxpkg":
+    install /= "local"
+    bash_dir = install / "etc" / "bash_completion.d"
+    dirs = ["usr"]
+    flags.extend(
+        [
+            "--osxpkg-identifier-prefix",
+            "com.iterative",
+            "--after-install",
+            path / "after-install.sh",
+            "--after-remove",
+            path / "after-remove.sh",
+        ]
+    )
+else:
+    bash_dir = build / "etc" / "bash_completion.d"
+    dirs = ["usr", "etc"]
+    flags.extend(["--depends", "git >= 1.7.0"])  # needed for gitpython
+
+try:
+    shutil.rmtree(build)
+except FileNotFoundError:
+    pass
+
+lib = install / "lib"
+lib.mkdir(parents=True)
+shutil.copytree(pyinstaller / "dist" / "dvc", lib / "dvc")
+
+if args.pkg != "osxpkg":
+    # NOTE: in osxpkg fpm replaces symlinks with actual file that it
+    # points to, so we need to use after-install hook.
+    (install / "bin").mkdir()
+    os.symlink("../lib/dvc/dvc", install / "bin" / "dvc")
+
+bash_dir.mkdir(parents=True)
+bash_completion = check_output(
+    [lib / "dvc" / "dvc", "completion", "-s", "bash"], text=True
+)
+(bash_dir / "dvc").write_text(bash_completion)
+
+zsh_dir = install / "share" / "zsh" / "site-functions"
+zsh_dir.mkdir(parents=True)
+zsh_completion = check_output(
+    [lib / "dvc" / "dvc", "completion", "-s", "zsh"], text=True
+)
+(zsh_dir / "_dvc").write_text(zsh_completion)
+
+version = check_output([lib / "dvc" / "dvc", "--version"], text=True).strip()
+
+check_call(
+    [
+        "fpm",
+        "--verbose",
+        "-t",
+        args.pkg,
+        *flags,
+        "-v",
+        version,
+        "-C",
+        build,
+        *dirs,
+    ],
+    cwd=path,
+    stderr=STDOUT,
+)

scripts/fpm/notarize.py

@@ -0,0 +1,64 @@
+import argparse
+import json
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "path",
+    nargs="?",
+    help="Path to the osxpkg to notarize. If not specified - try to find one.",
+)
+parser.add_argument(
+    "--apple-id-username", required=True, help="Apple ID username."
+)
+parser.add_argument(
+    "--apple-id-password",
+    required=True,
+    help=(
+        "Apple ID app-specific password. Note that this is not a regular "
+        "Apple ID password, so you need to generate one at "
+        "https://appleid.apple.com/account/manage"
+    ),
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+
+if args.path:
+    pkg = pathlib.Path(args.path)
+else:
+    pkgs = list(path.glob("*.pkg"))
+    if not pkgs:
+        print("No pkgs found")
+        exit(1)
+
+    if len(pkgs) > 1:
+        print("Too many packages")
+        exit(1)
+
+    (pkg,) = pkgs
+
+
+config = {
+    "notarize": {
+        "path": os.fspath(pkg),
+        "bundle_id": "com.iterative.dvc",
+        "staple": True,
+    },
+    "apple_id": {
+        "username": args.apple_id_username,
+        "password": args.apple_id_password,
+    },
+}
+
+(path / "config.json").write_text(json.dumps(config))
+
+check_call(
+    ["gon", "config.json"], cwd=path, stderr=STDOUT,
+)

scripts/fpm/sign.py

@@ -0,0 +1,54 @@
+import argparse
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "path",
+    nargs="?",
+    help="Path to the osxpkg to sign. If not specified - try to find one.",
+)
+parser.add_argument(
+    "--installer-id",
+    required=True,
+    help="Certificate ID (should be added to the keychain).",
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+
+if args.path:
+    pkg = pathlib.Path(args.path)
+else:
+    pkgs = list(path.glob("*.pkg"))
+    if not pkgs:
+        print("No pkgs found")
+        exit(1)
+
+    if len(pkgs) > 1:
+        print("Too many packages")
+        exit(1)
+
+    (pkg,) = pkgs
+
+unsigned = pkg.with_suffix(".unsigned")
+os.rename(pkg, unsigned)
+check_call(
+    [
+        "productsign",
+        "--sign",
+        args.installer_id,
+        os.fspath(unsigned),
+        os.fspath(pkg),
+    ],
+    stderr=STDOUT,
+)
+
+check_call(
+    ["pkgutil", "--check-signature", os.fspath(pkg)], stderr=STDOUT,
+)

scripts/innosetup/.gitignore

@@ -1,4 +1,6 @@
-dvc.ico
-dvc_left.bmp
-dvc_up.bmp
+/dvc.ico
+/dvc_left.bmp
+/dvc_up.bmp
 /config.ini
+/build
+/dvc*.exe

scripts/innosetup/build.py

@@ -0,0 +1,42 @@
+import argparse
+import configparser
+import os
+import pathlib
+import shutil
+from subprocess import STDOUT, check_call, check_output
+
+path = pathlib.Path(__file__).parent.absolute()
+config = path / "config.ini"
+
+dvc = path.parent.parent / "dvc"
+pyinstaller = path.parent / "pyinstaller"
+
+build = path / "build"
+install = build / "usr"
+
+parser = argparse.ArgumentParser()
+args = parser.parse_args()
+
+try:
+    shutil.rmtree(build)
+except FileNotFoundError:
+    pass
+
+build.mkdir()
+shutil.copytree(pyinstaller / "dist" / "dvc", build / "dvc")
+shutil.copy(path / "addSymLinkPermissions.ps1", build)
+
+version = check_output(
+    [os.fspath(build / "dvc" / "dvc"), "--version"], text=True
+).strip()
+
+cfg = configparser.ConfigParser()
+cfg.add_section("Version")
+cfg.set("Version", "version", version)
+
+with (path / "config.ini").open("w") as fobj:
+    cfg.write(fobj)
+
+check_call(
+    ["iscc", "setup.iss"], cwd=path, stderr=STDOUT,
+)

scripts/innosetup/config_gen.py

@@ -1,11 +0,0 @@
-# This script generates config.ini for setup.iss script
-import configparser
-
-from dvc import __version__
-
-config = configparser.ConfigParser()
-config.add_section("Version")
-config.set("Version", "Version", __version__)
-
-with open("scripts/innosetup/config.ini", "w") as f:
-    config.write(f)

scripts/innosetup/setup.iss

@@ -1,8 +1,8 @@
 #define MyAppName "DVC (Data Version Control)"
-#define MyAppVersion ReadIni(".\scripts\innosetup\config.ini", "Version", "version", "unknown")
+#define MyAppVersion ReadIni(SourcePath + "\config.ini", "Version", "version", "unknown")
 #define MyAppPublisher "Dmitry Petrov"
-#define MyAppURL "https://dataversioncontrol.com/"
-#define MyAppDir "..\..\dist\dvc"
+#define MyAppURL "https://dvc.org"
+#define MyAppDir SourcePath + "\build\dvc"
 
 [Setup]
 AppId={{8258CE8A-110E-4E0D-AE60-FEE00B15F041}
@@ -19,7 +19,7 @@ LicenseFile=..\..\LICENSE
 OutputBaseFilename=dvc-{#MyAppVersion}
 Compression=lzma2/max
 SolidCompression=yes
-OutputDir=..\..\
+OutputDir=.
 ChangesEnvironment=yes
 SetupIconFile=dvc.ico
 WizardSmallImageFile=dvc_up.bmp

scripts/pyinstaller/.gitignore

@@ -0,0 +1,4 @@
+/build
+/dist
+/dvc.spec
+/dvc

scripts/pyinstaller/build.py

@@ -0,0 +1,22 @@
+import os
+import pathlib
+from subprocess import STDOUT, check_call
+
+path = pathlib.Path(__file__).parent.absolute()
+hooks = path / "hooks"
+dvc = path.parent.parent / "dvc"
+entry = dvc / "__main__.py"
+
+check_call(
+    [
+        "pyinstaller",
+        "--additional-hooks-dir",
+        os.fspath(hooks),
+        "--name",
+        "dvc",
+        "-y",
+        os.fspath(entry),
+    ],
+    cwd=path,
+    stderr=STDOUT,
+)

scripts/pyinstaller/entitlements.plist

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<!-- These are required for binaries built by PyInstaller -->
+	<key>com.apple.security.cs.allow-jit</key>
+	<true/>
+	<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+	<true/>
+</dict>
+</plist>