dvc: bump to 2.0.16

Got deleted by mistake in https://github.com/iterative/dvc/pull/5682/files

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+
+updates:
+  - directory: "/"
+    package-ecosystem: "pip"
+    schedule:
+      interval: "daily"
+
+  - directory: "/"
+    package-ecosystem: "github-actions"
+    schedule:
+      interval: "daily"

.github/workflows/packages.yaml

@@ -9,9 +9,9 @@ jobs:
   deb:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.1
       with:
         python-version: 3.8
     - name: Set up Ruby 2.4
@@ -25,14 +25,9 @@ jobs:
         gem install --no-document fpm
     - name: Build deb
       run: python scripts/build.py deb
-    - name: Upload deb artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/fpm/dvc*.deb
-        if-no-files-found: error
     - name: Publish deb
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
@@ -43,9 +38,9 @@ jobs:
   rpm:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.1
       with:
         python-version: 3.8
     - name: Set up Ruby 2.4
@@ -59,14 +54,9 @@ jobs:
         gem install --no-document fpm
     - name: Build rpm
       run: python scripts/build.py rpm
-    - name: Upload rpm artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/fpm/dvc*.rpm
-        if-no-files-found: error
     - name: Publish rpm
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
@@ -77,9 +67,9 @@ jobs:
   osxpkg:
     runs-on: macos-10.15
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.1
       with:
         python-version: 3.8
     - name: Set up Ruby 2.4
@@ -88,19 +78,17 @@ jobs:
         ruby-version: 2.4
     - name: Install deps
       run: |
-        pip install .[all]
+        # available lxml (webdav dependency) wheels are built with an old
+        # SDK, which breaks osxpkg notarization. Building from scratch
+        # instead.
+        pip install .[all] --no-binary lxml
         pip install -r scripts/build-requirements.txt
         gem install --no-document fpm
     - name: Build
       run: python scripts/build.py osxpkg
-    - name: Upload osxpkg artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/fpm/dvc*.pkg
-        if-no-files-found: error
     - name: Publish
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
@@ -111,9 +99,9 @@ jobs:
   exe:
     runs-on: windows-2019
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.1
       with:
         python-version: 3.8
     - name: Install deps
@@ -126,16 +114,11 @@ jobs:
     - name: Build
       run: python scripts\build.py exe
       shell: cmd
-    - name: Upload exe artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/innosetup/dvc*.exe
-        if-no-files-found: error
     - name: Publish
       if: github.event_name == 'release'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       with:
         upload_url: ${{ github.event.release.upload_url }}
         asset_path: scripts/innosetup/dvc-${{ github.event.release.tag_name }}.exe
@@ -144,9 +127,9 @@ jobs:
   pip:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.1
       with:
         python-version: 3.7
     - name: Install
@@ -162,7 +145,7 @@ jobs:
   snap:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Build the package
       uses: snapcore/action-build@v1
       id: snapcraft

.github/workflows/tests.yaml

@@ -21,13 +21,15 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.1
       with:
         python-version: 3.8
     - name: Install requirements
-      run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
+      run: |
+        pip install wheel
+        pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
     - name: install temporary dependencies
       run: |
         pip install git+https://github.com/isidentical/gdrivefs@service-account
@@ -51,9 +53,9 @@ jobs:
             pyv: "3.9"
 
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.1
       with:
         python-version: ${{ matrix.pyv }}
     - name: get pip cache dir
@@ -62,7 +64,7 @@ jobs:
         echo "::set-output name=dir::$(pip cache dir)"
     - name: set pip cache
       id: pip-cache
-      uses: actions/cache@v2
+      uses: actions/cache@v2.1.4
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
@@ -88,9 +90,9 @@ jobs:
         mkdir -p scripts/ci
         echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
     - name: run tests
-      run: python -m tests --all --cov-report=xml --cov-report=term --tap-combined -n=4
+      run: python -m tests --all --cov-report=xml --cov-report=term -n=4
     - name: upload coverage report
-      uses: codecov/codecov-action@v1.0.13
+      uses: codecov/codecov-action@v1.3.1
       with:
         file: ./coverage.xml
         fail_ci_if_error: false

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 5.6.4
+    rev: 5.8.0
   - hooks:
       - id: flake8
         language_version: python3
@@ -18,13 +18,13 @@ repos:
           - flake8-debugger
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 3.9.0
   - hooks:
       -  id: mypy
          language_version: python3
          files: ^dvc/
     repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.790'
+    rev: 'v0.812'
   - repo: local
     hooks:
     - id: pylint

MANIFEST.in

@@ -1,2 +1,3 @@
 include fastentrypoints.py
 include LICENSE
+include test_requirements.txt

dvc/command/experiments.py

@@ -268,7 +268,8 @@ def _extend_row(row, names, items, precision):
     from rich.text import Text
 
     if not items:
-        row.extend(["-"] * len(names))
+        for keys in names.values():
+            row.extend(["-"] * len(keys))
         return
 
     for fname, item in items:
@@ -408,7 +409,9 @@ class CmdExperimentsShow(CmdBase):
                 # NOTE: rich does not have native support for unlimited width
                 # via pager. we override rich table compression by setting
                 # console width to the full width of the table
-                measurement = table.__rich_measure__(console, SHOW_MAX_WIDTH)
+                console_options = console.options
+                console_options.max_width = SHOW_MAX_WIDTH
+                measurement = table.__rich_measure__(console, console_options)
                 console._width = (  # pylint: disable=protected-access
                     measurement.maximum
                 )

dvc/command/status.py

@@ -44,21 +44,27 @@ class CmdDataStatus(CmdDataBase):
                 self._show(value, indent + 1)
 
     def run(self):
+        from dvc.repo import lock_repo
         from dvc.ui import ui
 
         indent = 1 if self.args.cloud else 0
-        try:
-            st = self.repo.status(
-                targets=self.args.targets,
-                jobs=self.args.jobs,
-                cloud=self.args.cloud,
-                remote=self.args.remote,
-                all_branches=self.args.all_branches,
-                all_tags=self.args.all_tags,
-                all_commits=self.args.all_commits,
-                with_deps=self.args.with_deps,
-                recursive=self.args.recursive,
-            )
+
+        with lock_repo(self.repo):
+            try:
+                st = self.repo.status(
+                    targets=self.args.targets,
+                    jobs=self.args.jobs,
+                    cloud=self.args.cloud,
+                    remote=self.args.remote,
+                    all_branches=self.args.all_branches,
+                    all_tags=self.args.all_tags,
+                    all_commits=self.args.all_commits,
+                    with_deps=self.args.with_deps,
+                    recursive=self.args.recursive,
+                )
+            except DvcException:
+                logger.exception("")
+                return 1
 
             if self.args.quiet:
                 return bool(st)
@@ -67,9 +73,14 @@ class CmdDataStatus(CmdDataBase):
                 import json
 
                 logger.info(json.dumps(st))
-            elif st:
+                return 0
+
+            if st:
                 self._show(st, indent)
-            elif not self.repo.stages:
+                return 0
+
+            # additional hints for the user
+            if not self.repo.stages:
                 ui.write(self.EMPTY_PROJECT_MSG)
             elif self.args.cloud or self.args.remote:
                 remote = self.args.remote or self.repo.config["core"].get(
@@ -79,7 +90,4 @@ class CmdDataStatus(CmdDataBase):
             else:
                 ui.write(self.UP_TO_DATE_MSG)
 
-        except DvcException:
-            logger.exception("")
-            return 1
         return 0

dvc/config_schema.py

@@ -145,6 +145,7 @@ SCHEMA = {
                     "session_token": str,
                     Optional("listobjects", default=False): Bool,  # obsoleted
                     Optional("use_ssl", default=True): Bool,
+                    Optional("ssl_verify", default=True): Bool,
                     "sse": str,
                     "sse_kms_key_id": str,
                     "acl": str,

dvc/fs/base.py

@@ -1,6 +1,6 @@
 import logging
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from functools import partial
+from functools import partial, partialmethod
 from multiprocessing import cpu_count
 from typing import Any, ClassVar, Dict, FrozenSet, Optional
 
@@ -258,6 +258,7 @@ class BaseFileSystem:
         name=None,
         no_progress_bar=False,
         jobs=None,
+        _only_file=False,
         **kwargs,
     ):
         if not hasattr(self, "_download"):
@@ -273,12 +274,14 @@ class BaseFileSystem:
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        if self.isdir(from_info):
+        if not _only_file and self.isdir(from_info):
             return self._download_dir(
                 from_info, to_info, name, no_progress_bar, jobs, **kwargs,
             )
         return self._download_file(from_info, to_info, name, no_progress_bar,)
 
+    download_file = partialmethod(download, _only_file=True)
+
     def _download_dir(
         self, from_info, to_info, name, no_progress_bar, jobs, **kwargs,
     ):

dvc/fs/s3.py

@@ -38,6 +38,7 @@ class S3FileSystem(BaseFileSystem):
         self.endpoint_url = config.get("endpointurl")
 
         self.use_ssl = config.get("use_ssl", True)
+        self.ssl_verify = config.get("ssl_verify", True)
 
         self.extra_args = {}
 
@@ -136,6 +137,7 @@ class S3FileSystem(BaseFileSystem):
         return session.resource(
             "s3",
             endpoint_url=self.endpoint_url,
+            verify=self.ssl_verify,
             use_ssl=self.use_ssl,
             config=boto3.session.Config(
                 signature_version="s3v4", s3=s3_config

dvc/fs/ssh/connection.py

@@ -17,7 +17,7 @@ from ..base import RemoteCmdError
 try:
     import paramiko
 except ImportError:
-    paramiko = None
+    paramiko = None  # type: ignore
 
 
 logger = logging.getLogger(__name__)

dvc/fs/webdav.py

@@ -117,7 +117,7 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
             )
 
         # Check whether connection is valid (root should always exist)
-        if not client.check("/"):
+        if not client.check(self.path_info.path):
             raise WebDAVConnectionError(self.hostname)
 
         return client
@@ -213,7 +213,7 @@ class WebDAVFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     # Creates directories
     def makedirs(self, path_info):
         # Terminate recursion
-        if path_info.path == "/" or self.exists(path_info):
+        if path_info.path == self.path_info.path or self.exists(path_info):
             return
 
         # Recursively descent to root

dvc/ignore.py

@@ -183,7 +183,12 @@ class DvcIgnoreFilter:
     def __init__(self, fs, root_dir):
         from dvc.repo import Repo
 
-        default_ignore_patterns = [".hg/", ".git/", "{}/".format(Repo.DVC_DIR)]
+        default_ignore_patterns = [
+            ".hg/",
+            ".git/",
+            ".git",
+            "{}/".format(Repo.DVC_DIR),
+        ]
 
         self.fs = fs
         self.root_dir = root_dir

dvc/parsing/context.py

@@ -401,9 +401,9 @@ class Context(CtxDict):
     ):
         path, _, keys_str = item.partition(":")
         select_keys = lfilter(bool, keys_str.split(",")) if keys_str else None
-        path_info = wdir / path
 
-        abspath = os.path.abspath(path_info)
+        abspath = os.path.abspath(wdir / path)
+        path_info = PathInfo(abspath)
         if abspath in self.imports:
             if not select_keys and self.imports[abspath] is None:
                 return  # allow specifying complete filepath multiple times

dvc/remote/base.py

@@ -309,7 +309,7 @@ class Remote:
         )
 
         if download:
-            func = _log_exceptions(self.fs.download, "download")
+            func = _log_exceptions(self.fs.download_file, "download")
             status = STATUS_DELETED
             desc = "Downloading"
         else:

dvc/scm/git/backend/dulwich.py

@@ -182,11 +182,16 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
     def commit(self, msg: str, no_verify: bool = False):
         from dulwich.errors import CommitError
         from dulwich.porcelain import commit
+        from dulwich.repo import InvalidUserIdentity
 
         try:
             commit(self.root_dir, message=msg, no_verify=no_verify)
         except CommitError as exc:
             raise SCMError("Git commit failed") from exc
+        except InvalidUserIdentity as exc:
+            raise SCMError(
+                "Git username and email must be configured"
+            ) from exc
 
     def checkout(
         self,
@@ -528,6 +533,8 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
         message: Optional[str] = None,
         include_untracked: Optional[bool] = False,
     ) -> Tuple[Optional[str], bool]:
+        from dulwich.repo import InvalidUserIdentity
+
         from dvc.scm.git import Stash
 
         if include_untracked or ref == Stash.DEFAULT_STASH:
@@ -537,7 +544,12 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
 
         stash = self._get_stash(ref)
         message_b = message.encode("utf-8") if message else None
-        rev = stash.push(message=message_b)
+        try:
+            rev = stash.push(message=message_b)
+        except InvalidUserIdentity as exc:
+            raise SCMError(
+                "Git username and email must be configured"
+            ) from exc
         return os.fsdecode(rev), True
 
     def _stash_apply(self, rev: str):

dvc/scm/git/backend/pygit2.py

@@ -80,6 +80,15 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
 
         return RefdbFsBackend(self.repo)
 
+    @property
+    def default_signature(self):
+        try:
+            return self.repo.default_signature
+        except KeyError as exc:
+            raise SCMError(
+                "Git username and email must be configured"
+            ) from exc
+
     @staticmethod
     def clone(
         url: str,
@@ -328,7 +337,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         from dvc.scm.git import Stash
 
         oid = self.repo.stash(
-            self.repo.default_signature,
+            self.default_signature,
             message=message,
             include_untracked=include_untracked,
         )
@@ -503,7 +512,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
             raise MergeConflictError("Merge contained conflicts")
 
         if commit:
-            user = self.repo.default_signature
+            user = self.default_signature
             tree = self.repo.index.write_tree()
             merge_commit = self.repo.create_commit(
                 "HEAD", user, user, msg, tree, [self.repo.head.target, rev]

dvc/ui/__init__.py

@@ -29,13 +29,19 @@ class Console:
         error: TextIO = None,
         enable: bool = False,
     ) -> None:
-        self._input: TextIO = sys.stdin
-        self._output: TextIO = output or sys.stdout
-        self._error: TextIO = error or sys.stderr
-
+        self._output: Optional[TextIO] = output
+        self._error: Optional[TextIO] = error
         self.formatter: Formatter = formatter or Formatter()
         self._enabled: bool = enable
 
+    @property
+    def output(self) -> TextIO:
+        return self._output or sys.stdout
+
+    @property
+    def error_output(self) -> TextIO:
+        return self._error or sys.stderr
+
     def enable(self):
         self._enabled = True
 
@@ -61,7 +67,7 @@ class Console:
             style=style,
             sep=sep,
             end=end,
-            file=self._error,
+            file=self.error_output,
             flush=flush,
         )
 
@@ -77,12 +83,12 @@ class Console:
         if not self._enabled:
             return
 
-        file = file or self._output
+        file = file or self.output
         values = (self.formatter.format(obj, style=style) for obj in objects)
         return print(*values, sep=sep, end=end, file=file, flush=flush)
 
     def progress(self, *args, **kwargs) -> Tqdm:
-        kwargs.setdefault("file", self._error)
+        kwargs.setdefault("file", self.error_output)
         return Tqdm(*args, **kwargs)
 
     def prompt(
@@ -125,7 +131,7 @@ class Console:
         """rich_console is only set to stdout for now."""
         from rich import console
 
-        return console.Console(file=self._output)
+        return console.Console(file=self.output)
 
     def rich_table(self, pager: bool = True):
         pass

dvc/utils/table.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING, List, cast
 
 from rich.style import StyleType
 from rich.table import Column as RichColumn
@@ -8,6 +8,7 @@ from rich.table import Table as RichTable
 if TYPE_CHECKING:
     from rich.console import (
         Console,
+        ConsoleOptions,
         JustifyMethod,
         OverflowMethod,
         RenderableType,
@@ -56,7 +57,7 @@ class Table(RichTable):
         self.columns.append(column)
 
     def _calculate_column_widths(
-        self, console: "Console", max_width: int
+        self, console: "Console", options: "ConsoleOptions"
     ) -> List[int]:
         """Calculate the widths of each column, including padding, not
         including borders.
@@ -64,22 +65,23 @@ class Table(RichTable):
         Adjacent collapsed columns will be removed until there is only a single
         truncated column remaining.
         """
-        widths = super()._calculate_column_widths(console, max_width)
+        widths = super()._calculate_column_widths(console, options)
         last_collapsed = -1
-        for i in range(len(self.columns) - 1, -1, -1):
-            if widths[i] == 1 and self.columns[i].collapse:
+        columns = cast(List[Column], self.columns)
+        for i in range(len(columns) - 1, -1, -1):
+            if widths[i] == 0 and columns[i].collapse:
                 if last_collapsed >= 0:
                     del widths[last_collapsed]
-                    del self.columns[last_collapsed]
+                    del columns[last_collapsed]
                     if self.box:
-                        max_width += 1
-                    for column in self.columns[last_collapsed:]:
+                        options.max_width += 1
+                    for column in columns[last_collapsed:]:
                         column._index -= 1
                 last_collapsed = i
                 padding = self._get_padding_width(i)
                 if (
-                    self.columns[i].overflow == "ellipsis"
-                    and (sum(widths) + padding) <= max_width
+                    columns[i].overflow == "ellipsis"
+                    and (sum(widths) + padding) <= options.max_width
                 ):
                     # Set content width to 1 (plus padding) if we can fit a
                     # single unicode ellipsis in this column
@@ -88,7 +90,7 @@ class Table(RichTable):
                 last_collapsed = -1
         return widths
 
-    def _collapse_widths(
+    def _collapse_widths(  # type: ignore[override]
         self, widths: List[int], wrapable: List[bool], max_width: int,
     ) -> List[int]:
         """Collapse columns right-to-left if possible to fit table into
@@ -97,7 +99,8 @@ class Table(RichTable):
         If table is still too wide after collapsing, rich's automatic overflow
         handling will be used.
         """
-        collapsible = [column.collapse for column in self.columns]
+        columns = cast(List[Column], self.columns)
+        collapsible = [column.collapse for column in columns]
         total_width = sum(widths)
         excess_width = total_width - max_width
         if any(collapsible):

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.0.8"
+_BASE_VERSION = "2.0.16"
 
 
 def _generate_version(base_version):

scripts/build.py

@@ -14,6 +14,13 @@ parser = argparse.ArgumentParser()
 parser.add_argument(
     "pkg", choices=["deb", "rpm", "osxpkg", "exe"], help="package type"
 )
+parser.add_argument("--sign-application", default=False, action="store_true")
+parser.add_argument("--application-id")
+parser.add_argument("--sign-installer", default=False, action="store_true")
+parser.add_argument("--installer-id")
+parser.add_argument("--notarize", default=False, action="store_true")
+parser.add_argument("--apple-id-username")
+parser.add_argument("--apple-id-password")
 args = parser.parse_args()
 
 (dvc / "utils" / "build.py").write_text(f'PKG = "{args.pkg}"')
@@ -39,6 +46,18 @@ check_call(
     ["python", "build.py"], cwd=pyinstaller, stderr=STDOUT,
 )
 
+if args.sign_application:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not args.application_id:
+        print("--sign-application requires --application-id")
+        exit(1)
+    check_call(
+        ["python", "sign.py", "--application-id", args.application_id],
+        cwd=pyinstaller,
+        stderr=STDOUT,
+    )
+
 if args.pkg == "exe":
     check_call(
         ["python", "build.py"], cwd=innosetup, stderr=STDOUT,
@@ -47,3 +66,38 @@ else:
     check_call(
         ["python", "build.py", args.pkg], cwd=fpm, stderr=STDOUT,
     )
+
+if args.sign_installer:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not all(
+        [args.installer_id, args.apple_id_username, args.apple_id_password]
+    ):
+        print("--sign-installer requires --installer-id")
+        exit(1)
+    check_call(
+        ["python", "sign.py", "--installer-id", args.installer_id],
+        cwd=fpm,
+        stderr=STDOUT,
+    )
+
+if args.notarize:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not all([args.apple_id_username, args.apple_id_password]):
+        print(
+            "--notarize requires --apple-id-username and --apple-id-password"
+        )
+        exit(1)
+    check_call(
+        [
+            "python",
+            "notarize.py",
+            "--apple-id-username",
+            args.apple_id_username,
+            "--apple-id-password",
+            args.apple_id_password,
+        ],
+        cwd=fpm,
+        stderr=STDOUT,
+    )

scripts/fpm/build.py

@@ -69,7 +69,7 @@ bash_completion = check_output(
 )
 (bash_dir / "dvc").write_text(bash_completion)
 
-zsh_dir = build / "usr" / "share" / "zsh" / "site-funtions"
+zsh_dir = install / "share" / "zsh" / "site-functions"
 zsh_dir.mkdir(parents=True)
 zsh_completion = check_output(
     [lib / "dvc" / "dvc", "completion", "-s", "zsh"], text=True

scripts/fpm/notarize.py

@@ -0,0 +1,64 @@
+import argparse
+import json
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "path",
+    nargs="?",
+    help="Path to the osxpkg to notarize. If not specified - try to find one.",
+)
+parser.add_argument(
+    "--apple-id-username", required=True, help="Apple ID username."
+)
+parser.add_argument(
+    "--apple-id-password",
+    required=True,
+    help=(
+        "Apple ID app-specific password. Note that this is not a regular "
+        "Apple ID password, so you need to generate one at "
+        "https://appleid.apple.com/account/manage"
+    ),
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+
+if args.path:
+    pkg = pathlib.Path(args.path)
+else:
+    pkgs = list(path.glob("*.pkg"))
+    if not pkgs:
+        print("No pkgs found")
+        exit(1)
+
+    if len(pkgs) > 1:
+        print("Too many packages")
+        exit(1)
+
+    (pkg,) = pkgs
+
+
+config = {
+    "notarize": {
+        "path": os.fspath(pkg),
+        "bundle_id": "com.iterative.dvc",
+        "staple": True,
+    },
+    "apple_id": {
+        "username": args.apple_id_username,
+        "password": args.apple_id_password,
+    },
+}
+
+(path / "config.json").write_text(json.dumps(config))
+
+check_call(
+    ["gon", "config.json"], cwd=path, stderr=STDOUT,
+)

scripts/fpm/sign.py

@@ -0,0 +1,54 @@
+import argparse
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "path",
+    nargs="?",
+    help="Path to the osxpkg to sign. If not specified - try to find one.",
+)
+parser.add_argument(
+    "--installer-id",
+    required=True,
+    help="Certificate ID (should be added to the keychain).",
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+
+if args.path:
+    pkg = pathlib.Path(args.path)
+else:
+    pkgs = list(path.glob("*.pkg"))
+    if not pkgs:
+        print("No pkgs found")
+        exit(1)
+
+    if len(pkgs) > 1:
+        print("Too many packages")
+        exit(1)
+
+    (pkg,) = pkgs
+
+unsigned = pkg.with_suffix(".unsigned")
+os.rename(pkg, unsigned)
+check_call(
+    [
+        "productsign",
+        "--sign",
+        args.installer_id,
+        os.fspath(unsigned),
+        os.fspath(pkg),
+    ],
+    stderr=STDOUT,
+)
+
+check_call(
+    ["pkgutil", "--check-signature", os.fspath(pkg)], stderr=STDOUT,
+)

scripts/pyinstaller/entitlements.plist

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<!-- These are required for binaries built by PyInstaller -->
+	<key>com.apple.security.cs.allow-jit</key>
+	<true/>
+	<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+	<true/>
+</dict>
+</plist>

scripts/pyinstaller/hooks/hook-googleapiclient.model.py

@@ -0,0 +1,8 @@
+"""Temporary fix for https://github.com/iterative/dvc/issues/5618."""
+from PyInstaller.utils.hooks import (  # pylint:disable=import-error
+    collect_data_files,
+    copy_metadata,
+)
+
+datas = collect_data_files("googleapiclient.discovery")
+datas += copy_metadata("google_api_python_client")

scripts/pyinstaller/sign.py

@@ -0,0 +1,39 @@
+import argparse
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--application-id",
+    required=True,
+    help="Certificate ID (should be added to the keychain).",
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+dvc = path / "dist" / "dvc"
+for root, _, fnames in os.walk(dvc):
+    for fname in fnames:
+        fpath = os.path.join(root, fname)
+        print(f"signing {fpath}")
+        check_call(
+            [
+                "codesign",
+                "--force",
+                "--verbose",
+                "-s",
+                args.application_id,
+                "-o",
+                "runtime",
+                "--entitlements",
+                "entitlements.plist",
+                fpath,
+            ],
+            stderr=STDOUT,
+            timeout=90,
+        )

setup.py

@@ -1,5 +1,6 @@
 import importlib.util
 import os
+from pathlib import Path
 
 from setuptools import find_packages, setup
 from setuptools.command.build_py import build_py as _build_py
@@ -50,7 +51,7 @@ install_requires = [
     "colorama>=0.3.9",
     "configobj>=5.0.6",
     "gitpython>3",
-    "dulwich>=0.20.20",
+    "dulwich>=0.20.21",
     "pygit2>=1.5.0",
     "setuptools>=34.0.0",
     "nanotime>=0.5.2",
@@ -82,7 +83,7 @@ install_requires = [
     "pygtrie==2.3.2",
     "dpath>=2.0.1,<3",
     "shtab>=1.3.4,<2",
-    "rich>=9.0.0",
+    "rich>=10.0.0",
     "dictdiffer>=0.8.1",
     "python-benedict>=0.21.1",
     "pyparsing==2.4.7",
@@ -113,49 +114,9 @@ webdav = ["webdavclient3>=3.14.5"]
 ssh_gssapi = ["paramiko[invoke,gssapi]>=2.7.0"]
 all_remotes = gs + s3 + azure + ssh + oss + gdrive + hdfs + webhdfs + webdav
 
-# Extra dependecies to run tests
-tests_requirements = [
-    "wheel>=0.31.1",
-    # Test requirements:
-    "pytest>=6.0.1,<6.2.2",
-    "pytest-docker>=0.7.2",
-    "pytest-timeout>=1.3.3",
-    "pytest-cov>=2.6.1",
-    "pytest-xdist>=1.26.1",
-    "pytest-mock==1.11.2",
-    "pytest-lazy-fixture",
-    "pytest-tap",
-    "flaky>=3.5.3",
-    "mock>=3.0.0",
-    "xmltodict>=0.11.0",
-    "google-compute-engine==2.8.13",
-    "Pygments",  # required by collective.checkdocs,
-    "collective.checkdocs",
-    "psutil",
-    "pydocstyle<4.0",
-    "jaraco.windows==3.9.2",
-    "mock-ssh-server>=0.8.2",
-    "moto==1.3.16.dev122",
-    # moto's indirect dependency that is causing problems with flufl.lock's
-    # dependency (atpublic). See https://github.com/iterative/dvc/pull/4853
-    "aws-sam-translator<1.29.0",
-    # for moto's indirect dependency.
-    # See https://github.com/iterative/dvc/pull/4879
-    "urllib3<1.26.0",
-    "rangehttpserver==1.2.0",
-    "beautifulsoup4==4.4.0",
-    "pylint==2.7.2",
-    "pylint-pytest==1.0.3",
-    "pylint-plugin-utils==0.6",
-    "wget",
-    "filelock",
-    "mypy",
-    "wsgidav",
-    "crc32c",
-    "google-cloud-storage==1.19.0",
-    # pypi doesn't allow for direct dependencies
-    #    "gdrivefs @ git+https://github.com/intake/gdrivefs.git",
-]
+tests_requirements = (
+    Path("test_requirements.txt").read_text().strip().splitlines()
+)
 
 setup(
     name="dvc",

test_requirements.txt

@@ -0,0 +1,48 @@
+wheel==0.36.2
+
+# Test requirements
+pytest==6.2.2
+pytest-cov==2.11.1
+pytest-xdist==2.2.1
+pytest-mock==3.5.1
+pytest-lazy-fixture==0.6.3
+pytest-timeout==1.4.2
+pytest-docker==0.10.1
+
+flaky==3.7.0
+mock==4.0.3
+rangehttpserver==1.2.0
+mock-ssh-server==0.9.0
+# moto's indirect dependency that is causing problems with flufl.lock's
+# dependency (atpublic). See https://github.com/iterative/dvc/pull/4853
+moto==2.0.4
+beautifulsoup4==4.9.3
+wget==3.2
+filelock==3.0.12
+wsgidav==3.1.0
+crc32c==2.2
+xmltodict==0.12.0
+google-compute-engine==2.8.13
+google-cloud-storage==1.37.0
+
+urllib3==1.26.4
+
+# required by collective.checkdocs
+Pygments==2.8.1
+collective.checkdocs==0.2
+pydocstyle==6.0.0
+jaraco.windows==5.5.0
+
+# pylint requirements
+pylint==2.7.4
+# we use this to suppress pytest-related false positives in our tests.
+pylint-pytest==1.0.3
+# we use this to suppress some messages in tests, eg: foo/bar naming,
+# and, protected method calls in our tests
+pylint-plugin-utils==0.6
+
+# type-checking
+mypy==0.812
+
+# pypi doesn't allow for direct dependencies
+# gdrivefs @ git+https://github.com/intake/gdrivefs.git

tests/conftest.py

@@ -43,6 +43,13 @@ def reset_loglevel(request, caplog):
         yield
 
 
+@pytest.fixture(autouse=True)
+def enable_ui():
+    from dvc.ui import ui
+
+    ui.enable()
+
+
 @pytest.fixture(scope="session", autouse=True)
 def _close_pools():
     from dvc.fs.pool import close_pools

tests/func/parsing/test_resolver.py

@@ -1,10 +1,11 @@
+import os
 from copy import deepcopy
 
 import pytest
 
 from dvc.parsing import DEFAULT_PARAMS_FILE, DataResolver, ResolveError
 from dvc.parsing.context import Context
-from dvc.utils.serialize import dump_json, dump_yaml
+from dvc.utils.serialize import dump_json, dump_yaml, dumps_yaml
 
 from . import CONTEXT_DATA, RESOLVED_DVC_YAML_DATA, TEMPLATED_DVC_YAML_DATA
 
@@ -67,6 +68,19 @@ def test_load_vars_from_file(tmp_dir, dvc):
     assert resolver.context == expected
 
 
+def test_load_vars_with_relpath(tmp_dir, scm, dvc):
+    tmp_dir.scm_gen(DEFAULT_PARAMS_FILE, dumps_yaml(DATA), commit="add params")
+
+    subdir = tmp_dir / "subdir"
+    d = {"vars": [os.path.relpath(tmp_dir / DEFAULT_PARAMS_FILE, subdir)]}
+
+    revisions = ["HEAD", "workspace"]
+    for rev in dvc.brancher(revs=["HEAD"]):
+        assert rev == revisions.pop()
+        resolver = DataResolver(dvc, subdir, d)
+        assert resolver.context == deepcopy(DATA)
+
+
 def test_partial_vars_doesnot_exist(tmp_dir, dvc):
     dump_yaml("test_params.yaml", {"sub1": "sub1"})
 

tests/func/test_external_repo.py

@@ -52,7 +52,7 @@ def test_cache_reused(erepo_dir, mocker, local_cloud):
         erepo_dir.dvc_gen("file", "text", commit="add file")
     erepo_dir.dvc.push()
 
-    download_spy = mocker.spy(LocalFileSystem, "download")
+    download_spy = mocker.spy(LocalFileSystem, "download_file")
 
     # Use URL to prevent any fishy optimizations
     url = f"file://{erepo_dir}"

tests/func/test_ignore.py

@@ -112,7 +112,7 @@ def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
     assert (
         DvcIgnorePatterns(
             *merge_patterns(
-                _to_pattern_info_list([".hg/", ".git/", ".dvc/"]),
+                _to_pattern_info_list([".hg/", ".git/", ".git", ".dvc/"]),
                 os.fspath(tmp_dir),
                 _to_pattern_info_list([os.path.basename(dname)]),
                 top_ignore_path,
@@ -344,7 +344,7 @@ def test_pattern_trie_fs(tmp_dir, dvc):
     )
 
     base_pattern = (
-        _to_pattern_info_list([".hg/", ".git/", ".dvc/"]),
+        _to_pattern_info_list([".hg/", ".git/", ".git", ".dvc/"]),
         os.fspath(tmp_dir),
     )
     first_pattern = merge_patterns(

tests/unit/command/test_status.py

@@ -45,7 +45,7 @@ def test_cloud_status(tmp_dir, dvc, mocker):
 
 
 @pytest.mark.parametrize("status", [{}, {"a": "b", "c": [1, 2, 3]}, [1, 2, 3]])
-def test_status_show_json(mocker, caplog, status):
+def test_status_show_json(dvc, mocker, caplog, status):
     cli_args = parse_args(["status", "--show-json"])
     assert cli_args.func == CmdDataStatus
 
@@ -60,7 +60,7 @@ def test_status_show_json(mocker, caplog, status):
 @pytest.mark.parametrize(
     "status, ret", [({}, 0), ({"a": "b", "c": [1, 2, 3]}, 1), ([1, 2, 3], 1)]
 )
-def test_status_quiet(mocker, caplog, capsys, status, ret):
+def test_status_quiet(dvc, mocker, caplog, capsys, status, ret):
     cli_args = parse_args(["status", "-q"])
     assert cli_args.func == CmdDataStatus
 
@@ -71,5 +71,45 @@ def test_status_quiet(mocker, caplog, capsys, status, ret):
     assert cmd.run() == ret
     assert not caplog.messages
     captured = capsys.readouterr()
-    assert not captured.err
     assert not captured.out
+
+
+def test_status_empty(dvc, mocker, capsys):
+    cli_args = parse_args(["status"])
+    assert cli_args.func == CmdDataStatus
+
+    cmd = cli_args.func(cli_args)
+
+    spy = mocker.spy(cmd.repo.stage, "collect_repo")
+
+    assert cmd.run() == 0
+
+    captured = capsys.readouterr()
+    assert "no data or pipelines tracked" in captured.out
+    # stages should only be collected once
+    assert spy.call_count == 1
+
+
+@pytest.mark.parametrize(
+    "cloud_opts, expected_message",
+    [
+        (["--cloud"], "Cache and remote 'default' are in sync"),
+        (["--remote", "remote1"], "Cache and remote 'remote1' are in sync"),
+        ([], "Data and pipelines are up to date"),
+    ],
+)
+def test_status_up_to_date(dvc, mocker, capsys, cloud_opts, expected_message):
+    cli_args = parse_args(["status", *cloud_opts])
+    assert cli_args.func == CmdDataStatus
+
+    cmd = cli_args.func(cli_args)
+
+    mocker.patch.dict(cmd.repo.config, {"core": {"remote": "default"}})
+    mocker.patch.object(cmd.repo, "status", autospec=True, return_value={})
+    mocker.patch.object(
+        cmd.repo.stage, "collect_repo", return_value=[object()], autospec=True
+    )
+
+    assert cmd.run() == 0
+    captured = capsys.readouterr()
+    assert expected_message in captured.out

tests/unit/fs/test_s3.py

@@ -29,6 +29,22 @@ def test_init(dvc):
     assert fs.path_info == url
 
 
+def test_verify_ssl_default_param(dvc):
+    config = {
+        "url": url,
+    }
+    fs = S3FileSystem(dvc, config)
+
+    assert fs.ssl_verify
+
+
+def test_ssl_verify_bool_param(dvc):
+    config = {"url": url, "ssl_verify": False}
+    fs = S3FileSystem(dvc, config)
+
+    assert fs.ssl_verify == config["ssl_verify"]
+
+
 def test_grants(dvc):
     config = {
         "url": url,

tests/unit/test_external_repo.py

@@ -68,7 +68,7 @@ def test_subrepo_is_constructed_properly(
 
         list(repo.repo_fs.walk(repo.root_dir))  # drain
         assert spy.call_count == 1
-        subrepo = spy.return_value
+        subrepo = spy.spy_return
 
         assert repo.url == str(tmp_dir)
         assert repo.config["cache"]["dir"] == str(cache_dir)

tests/unit/ui/test_console.py

@@ -0,0 +1,28 @@
+from pytest import CaptureFixture
+
+from dvc.ui import Console
+
+
+def test_write(capsys: CaptureFixture[str]):
+    """Test that ui.write works."""
+    console = Console(enable=True)
+    message = "hello world"
+    console.write(message)
+    console.error_write(message)
+
+    captured = capsys.readouterr()
+    assert captured.out == f"{message}\n"
+    assert captured.err == f"{message}\n"
+
+
+def test_capsys_works(capsys: CaptureFixture[str]):
+    """Sanity check that capsys can capture outputs from a global ui."""
+    from dvc.ui import ui
+
+    message = "hello world"
+    ui.write(message)
+    ui.error_write(message)
+
+    captured = capsys.readouterr()
+    assert captured.out == f"{message}\n"
+    assert captured.err == f"{message}\n"