dvc: bump to 2.0.12

* update pre-commit hooks

* fix mypy errors

1. Casting `columns` to the custom Column with collapse.
2. Ignoring typedef difference for RichTable._collapse_widths.
3. Suppressing error with conditional paramiko imports
   https://github.com/python/mypy/issues/1153

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+
+updates:
+  - directory: "/"
+    package-ecosystem: "pip"
+    schedule:
+      interval: "daily"
+
+  - directory: "/"
+    package-ecosystem: "github-actions"
+    schedule:
+      interval: "daily"

.github/workflows/packages.yaml

@@ -9,9 +9,9 @@ jobs:
   deb:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.1
       with:
         python-version: 3.8
     - name: Set up Ruby 2.4
@@ -25,14 +25,9 @@ jobs:
         gem install --no-document fpm
     - name: Build deb
       run: python scripts/build.py deb
-    - name: Upload deb artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/fpm/dvc*.deb
-        if-no-files-found: error
     - name: Publish deb
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
@@ -43,9 +38,9 @@ jobs:
   rpm:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.1
       with:
         python-version: 3.8
     - name: Set up Ruby 2.4
@@ -59,14 +54,9 @@ jobs:
         gem install --no-document fpm
     - name: Build rpm
       run: python scripts/build.py rpm
-    - name: Upload rpm artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/fpm/dvc*.rpm
-        if-no-files-found: error
     - name: Publish rpm
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
@@ -77,9 +67,9 @@ jobs:
   osxpkg:
     runs-on: macos-10.15
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.1
       with:
         python-version: 3.8
     - name: Set up Ruby 2.4
@@ -88,19 +78,17 @@ jobs:
         ruby-version: 2.4
     - name: Install deps
       run: |
-        pip install .[all]
+        # available lxml (webdav dependency) wheels are built with an old
+        # SDK, which breaks osxpkg notarization. Building from scratch
+        # instead.
+        pip install .[all] --no-binary lxml
         pip install -r scripts/build-requirements.txt
         gem install --no-document fpm
     - name: Build
       run: python scripts/build.py osxpkg
-    - name: Upload osxpkg artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/fpm/dvc*.pkg
-        if-no-files-found: error
     - name: Publish
       if: github.event_name == 'release'
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
@@ -111,9 +99,9 @@ jobs:
   exe:
     runs-on: windows-2019
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.1
       with:
         python-version: 3.8
     - name: Install deps
@@ -126,16 +114,11 @@ jobs:
     - name: Build
       run: python scripts\build.py exe
       shell: cmd
-    - name: Upload exe artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/innosetup/dvc*.exe
-        if-no-files-found: error
     - name: Publish
       if: github.event_name == 'release'
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       with:
         upload_url: ${{ github.event.release.upload_url }}
         asset_path: scripts/innosetup/dvc-${{ github.event.release.tag_name }}.exe
@@ -144,9 +127,9 @@ jobs:
   pip:
     runs-on: ubuntu-16.04
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.7
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.1
       with:
         python-version: 3.7
     - name: Install
@@ -162,7 +145,7 @@ jobs:
   snap:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Build the package
       uses: snapcore/action-build@v1
       id: snapcraft

.github/workflows/tests.yaml

@@ -21,13 +21,15 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.1
       with:
         python-version: 3.8
     - name: Install requirements
-      run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
+      run: |
+        pip install wheel
+        pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
     - name: install temporary dependencies
       run: |
         pip install git+https://github.com/isidentical/gdrivefs@service-account
@@ -51,9 +53,9 @@ jobs:
             pyv: "3.9"
 
     steps:
-    - uses: actions/[email protected].2
+    - uses: actions/[email protected].4
     - name: Set up Python
-      uses: actions/setup-python@v2.1.4
+      uses: actions/setup-python@v2.2.1
       with:
         python-version: ${{ matrix.pyv }}
     - name: get pip cache dir
@@ -62,7 +64,7 @@ jobs:
         echo "::set-output name=dir::$(pip cache dir)"
     - name: set pip cache
       id: pip-cache
-      uses: actions/cache@v2
+      uses: actions/cache@v2.1.4
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
@@ -88,9 +90,9 @@ jobs:
         mkdir -p scripts/ci
         echo "$GS_CREDS_JSON" > scripts/ci/gcp-creds.json
     - name: run tests
-      run: python -m tests --all --cov-report=xml --cov-report=term --tap-combined -n=4
+      run: python -m tests --all --cov-report=xml --cov-report=term -n=4
     - name: upload coverage report
-      uses: codecov/codecov-action@v1.0.13
+      uses: codecov/codecov-action@v1.3.1
       with:
         file: ./coverage.xml
         fail_ci_if_error: false

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: isort
         language_version: python3
     repo: https://github.com/timothycrosley/isort
-    rev: 5.6.4
+    rev: 5.8.0
   - hooks:
       - id: flake8
         language_version: python3
@@ -18,13 +18,13 @@ repos:
           - flake8-debugger
           - flake8-string-format
     repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 3.9.0
   - hooks:
       -  id: mypy
          language_version: python3
          files: ^dvc/
     repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.790'
+    rev: 'v0.812'
   - repo: local
     hooks:
     - id: pylint

dvc/fs/base.py

@@ -1,6 +1,6 @@
 import logging
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from functools import partial
+from functools import partial, partialmethod
 from multiprocessing import cpu_count
 from typing import Any, ClassVar, Dict, FrozenSet, Optional
 
@@ -258,6 +258,7 @@ class BaseFileSystem:
         name=None,
         no_progress_bar=False,
         jobs=None,
+        _only_file=False,
         **kwargs,
     ):
         if not hasattr(self, "_download"):
@@ -273,12 +274,14 @@ class BaseFileSystem:
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        if self.isdir(from_info):
+        if not _only_file and self.isdir(from_info):
             return self._download_dir(
                 from_info, to_info, name, no_progress_bar, jobs, **kwargs,
             )
         return self._download_file(from_info, to_info, name, no_progress_bar,)
 
+    download_file = partialmethod(download, _only_file=True)
+
     def _download_dir(
         self, from_info, to_info, name, no_progress_bar, jobs, **kwargs,
     ):

dvc/fs/ssh/connection.py

@@ -17,7 +17,7 @@ from ..base import RemoteCmdError
 try:
     import paramiko
 except ImportError:
-    paramiko = None
+    paramiko = None  # type: ignore
 
 
 logger = logging.getLogger(__name__)

dvc/remote/base.py

@@ -309,7 +309,7 @@ class Remote:
         )
 
         if download:
-            func = _log_exceptions(self.fs.download, "download")
+            func = _log_exceptions(self.fs.download_file, "download")
             status = STATUS_DELETED
             desc = "Downloading"
         else:

dvc/utils/table.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING, List, cast
 
 from rich.style import StyleType
 from rich.table import Column as RichColumn
@@ -66,19 +66,20 @@ class Table(RichTable):
         """
         widths = super()._calculate_column_widths(console, max_width)
         last_collapsed = -1
-        for i in range(len(self.columns) - 1, -1, -1):
-            if widths[i] == 1 and self.columns[i].collapse:
+        columns = cast(List[Column], self.columns)
+        for i in range(len(columns) - 1, -1, -1):
+            if widths[i] == 1 and columns[i].collapse:
                 if last_collapsed >= 0:
                     del widths[last_collapsed]
-                    del self.columns[last_collapsed]
+                    del columns[last_collapsed]
                     if self.box:
                         max_width += 1
-                    for column in self.columns[last_collapsed:]:
+                    for column in columns[last_collapsed:]:
                         column._index -= 1
                 last_collapsed = i
                 padding = self._get_padding_width(i)
                 if (
-                    self.columns[i].overflow == "ellipsis"
+                    columns[i].overflow == "ellipsis"
                     and (sum(widths) + padding) <= max_width
                 ):
                     # Set content width to 1 (plus padding) if we can fit a
@@ -88,7 +89,7 @@ class Table(RichTable):
                 last_collapsed = -1
         return widths
 
-    def _collapse_widths(
+    def _collapse_widths(  # type: ignore[override]
         self, widths: List[int], wrapable: List[bool], max_width: int,
     ) -> List[int]:
         """Collapse columns right-to-left if possible to fit table into
@@ -97,7 +98,8 @@ class Table(RichTable):
         If table is still too wide after collapsing, rich's automatic overflow
         handling will be used.
         """
-        collapsible = [column.collapse for column in self.columns]
+        columns = cast(List[Column], self.columns)
+        collapsible = [column.collapse for column in columns]
         total_width = sum(widths)
         excess_width = total_width - max_width
         if any(collapsible):

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.0.8"
+_BASE_VERSION = "2.0.12"
 
 
 def _generate_version(base_version):

scripts/build.py

@@ -14,6 +14,13 @@ parser = argparse.ArgumentParser()
 parser.add_argument(
     "pkg", choices=["deb", "rpm", "osxpkg", "exe"], help="package type"
 )
+parser.add_argument("--sign-application", default=False, action="store_true")
+parser.add_argument("--application-id")
+parser.add_argument("--sign-installer", default=False, action="store_true")
+parser.add_argument("--installer-id")
+parser.add_argument("--notarize", default=False, action="store_true")
+parser.add_argument("--apple-id-username")
+parser.add_argument("--apple-id-password")
 args = parser.parse_args()
 
 (dvc / "utils" / "build.py").write_text(f'PKG = "{args.pkg}"')
@@ -39,6 +46,18 @@ check_call(
     ["python", "build.py"], cwd=pyinstaller, stderr=STDOUT,
 )
 
+if args.sign_application:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not args.application_id:
+        print("--sign-application requires --application-id")
+        exit(1)
+    check_call(
+        ["python", "sign.py", "--application-id", args.application_id],
+        cwd=pyinstaller,
+        stderr=STDOUT,
+    )
+
 if args.pkg == "exe":
     check_call(
         ["python", "build.py"], cwd=innosetup, stderr=STDOUT,
@@ -47,3 +66,38 @@ else:
     check_call(
         ["python", "build.py", args.pkg], cwd=fpm, stderr=STDOUT,
     )
+
+if args.sign_installer:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not all(
+        [args.installer_id, args.apple_id_username, args.apple_id_password]
+    ):
+        print("--sign-installer requires --installer-id")
+        exit(1)
+    check_call(
+        ["python", "sign.py", "--installer-id", args.installer_id],
+        cwd=fpm,
+        stderr=STDOUT,
+    )
+
+if args.notarize:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not all([args.apple_id_username, args.apple_id_password]):
+        print(
+            "--notarize requires --apple-id-username and --apple-id-password"
+        )
+        exit(1)
+    check_call(
+        [
+            "python",
+            "notarize.py",
+            "--apple-id-username",
+            args.apple_id_username,
+            "--apple-id-password",
+            args.apple_id_password,
+        ],
+        cwd=fpm,
+        stderr=STDOUT,
+    )

scripts/fpm/build.py

@@ -69,7 +69,7 @@ bash_completion = check_output(
 )
 (bash_dir / "dvc").write_text(bash_completion)
 
-zsh_dir = build / "usr" / "share" / "zsh" / "site-funtions"
+zsh_dir = install / "share" / "zsh" / "site-funtions"
 zsh_dir.mkdir(parents=True)
 zsh_completion = check_output(
     [lib / "dvc" / "dvc", "completion", "-s", "zsh"], text=True

scripts/fpm/notarize.py

@@ -0,0 +1,64 @@
+import argparse
+import json
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "Darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "path",
+    nargs="?",
+    help="Path to the osxpkg to notarize. If not specified - try to find one.",
+)
+parser.add_argument(
+    "--apple-id-username", required=True, help="Apple ID username."
+)
+parser.add_argument(
+    "--apple-id-password",
+    required=True,
+    help=(
+        "Apple ID app-specific password. Note that this is not a regular "
+        "Apple ID password, so you need to generate one at "
+        "https://appleid.apple.com/account/manage"
+    ),
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+
+if args.path:
+    pkg = pathlib.Path(args.path)
+else:
+    pkgs = list(path.glob("*.pkg"))
+    if not pkgs:
+        print("No pkgs found")
+        exit(1)
+
+    if len(pkgs) > 1:
+        print("Too many packages")
+        exit(1)
+
+    (pkg,) = pkgs
+
+
+config = {
+    "notarize": {
+        "path": os.fspath(pkg),
+        "bundle_id": "com.iterative.dvc",
+        "staple": True,
+    },
+    "apple_id": {
+        "username": args.apple_id_username,
+        "password": args.apple_id_password,
+    },
+}
+
+(path / "config.json").write_text(json.dumps(config))
+
+check_call(
+    ["gon", "config.json"], cwd=path, stderr=STDOUT,
+)

scripts/fpm/sign.py

@@ -0,0 +1,54 @@
+import argparse
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "path",
+    nargs="?",
+    help="Path to the osxpkg to sign. If not specified - try to find one.",
+)
+parser.add_argument(
+    "--installer-id",
+    required=True,
+    help="Certificate ID (should be added to the keychain).",
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+
+if args.path:
+    pkg = pathlib.Path(args.path)
+else:
+    pkgs = list(path.glob("*.pkg"))
+    if not pkgs:
+        print("No pkgs found")
+        exit(1)
+
+    if len(pkgs) > 1:
+        print("Too many packages")
+        exit(1)
+
+    (pkg,) = pkgs
+
+unsigned = pkg.with_suffix(".unsigned")
+os.rename(pkg, unsigned)
+check_call(
+    [
+        "productsign",
+        "--sign",
+        args.installer_id,
+        os.fspath(unsigned),
+        os.fspath(pkg),
+    ],
+    stderr=STDOUT,
+)
+
+check_call(
+    ["pkgutil", "--check-signature", os.fspath(pkg)], stderr=STDOUT,
+)

scripts/pyinstaller/entitlements.plist

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<!-- These are required for binaries built by PyInstaller -->
+	<key>com.apple.security.cs.allow-jit</key>
+	<true/>
+	<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+	<true/>
+</dict>
+</plist>

scripts/pyinstaller/sign.py

@@ -0,0 +1,37 @@
+import argparse
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--application-id",
+    required=True,
+    help="Certificate ID (should be added to the keychain).",
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+dvc = path / "dist" / "dvc"
+for root, _, fnames in os.walk(dvc):
+    for fname in fnames:
+        fpath = os.path.join(root, fname)
+        print(f"signing {fpath}")
+        check_call(
+            [
+                "codesign",
+                "--force",
+                "-s",
+                args.application_id,
+                "-o",
+                "runtime",
+                "--entitlements",
+                "entitlements.plist",
+                fpath,
+            ],
+            stderr=STDOUT,
+        )

setup.py

@@ -1,5 +1,6 @@
 import importlib.util
 import os
+from pathlib import Path
 
 from setuptools import find_packages, setup
 from setuptools.command.build_py import build_py as _build_py
@@ -113,49 +114,9 @@ webdav = ["webdavclient3>=3.14.5"]
 ssh_gssapi = ["paramiko[invoke,gssapi]>=2.7.0"]
 all_remotes = gs + s3 + azure + ssh + oss + gdrive + hdfs + webhdfs + webdav
 
-# Extra dependecies to run tests
-tests_requirements = [
-    "wheel>=0.31.1",
-    # Test requirements:
-    "pytest>=6.0.1,<6.2.2",
-    "pytest-docker>=0.7.2",
-    "pytest-timeout>=1.3.3",
-    "pytest-cov>=2.6.1",
-    "pytest-xdist>=1.26.1",
-    "pytest-mock==1.11.2",
-    "pytest-lazy-fixture",
-    "pytest-tap",
-    "flaky>=3.5.3",
-    "mock>=3.0.0",
-    "xmltodict>=0.11.0",
-    "google-compute-engine==2.8.13",
-    "Pygments",  # required by collective.checkdocs,
-    "collective.checkdocs",
-    "psutil",
-    "pydocstyle<4.0",
-    "jaraco.windows==3.9.2",
-    "mock-ssh-server>=0.8.2",
-    "moto==1.3.16.dev122",
-    # moto's indirect dependency that is causing problems with flufl.lock's
-    # dependency (atpublic). See https://github.com/iterative/dvc/pull/4853
-    "aws-sam-translator<1.29.0",
-    # for moto's indirect dependency.
-    # See https://github.com/iterative/dvc/pull/4879
-    "urllib3<1.26.0",
-    "rangehttpserver==1.2.0",
-    "beautifulsoup4==4.4.0",
-    "pylint==2.7.2",
-    "pylint-pytest==1.0.3",
-    "pylint-plugin-utils==0.6",
-    "wget",
-    "filelock",
-    "mypy",
-    "wsgidav",
-    "crc32c",
-    "google-cloud-storage==1.19.0",
-    # pypi doesn't allow for direct dependencies
-    #    "gdrivefs @ git+https://github.com/intake/gdrivefs.git",
-]
+tests_requirements = (
+    Path("test_requirements.txt").read_text().strip().splitlines()
+)
 
 setup(
     name="dvc",

test_requirements.txt

@@ -0,0 +1,48 @@
+wheel==0.36.2
+
+# Test requirements
+pytest==6.2.2
+pytest-cov==2.11.1
+pytest-xdist==2.2.1
+pytest-mock==3.5.1
+pytest-lazy-fixture==0.6.3
+pytest-timeout==1.4.2
+pytest-docker==0.10.1
+
+flaky==3.7.0
+mock==4.0.3
+rangehttpserver==1.2.0
+mock-ssh-server==0.9.0
+# moto's indirect dependency that is causing problems with flufl.lock's
+# dependency (atpublic). See https://github.com/iterative/dvc/pull/4853
+moto==2.0.2
+beautifulsoup4==4.9.3
+wget==3.2
+filelock==3.0.12
+wsgidav==3.1.0
+crc32c==2.2
+xmltodict==0.12.0
+google-compute-engine==2.8.13
+google-cloud-storage==1.37.0
+
+urllib3==1.26.4
+
+# required by collective.checkdocs
+Pygments==2.8.1
+collective.checkdocs==0.2
+pydocstyle==6.0.0
+jaraco.windows==5.5.0
+
+# pylint requirements
+pylint==2.7.2
+# we use this to suppress pytest-related false positives in our tests.
+pylint-pytest==1.0.3
+# we use this to suppress some messages in tests, eg: foo/bar naming,
+# and, protected method calls in our tests
+pylint-plugin-utils==0.6
+
+# type-checking
+mypy==0.812
+
+# pypi doesn't allow for direct dependencies
+# gdrivefs @ git+https://github.com/intake/gdrivefs.git

tests/func/test_external_repo.py

@@ -52,7 +52,7 @@ def test_cache_reused(erepo_dir, mocker, local_cloud):
         erepo_dir.dvc_gen("file", "text", commit="add file")
     erepo_dir.dvc.push()
 
-    download_spy = mocker.spy(LocalFileSystem, "download")
+    download_spy = mocker.spy(LocalFileSystem, "download_file")
 
     # Use URL to prevent any fishy optimizations
     url = f"file://{erepo_dir}"

tests/unit/test_external_repo.py

@@ -68,7 +68,7 @@ def test_subrepo_is_constructed_properly(
 
         list(repo.repo_fs.walk(repo.root_dir))  # drain
         assert spy.call_count == 1
-        subrepo = spy.return_value
+        subrepo = spy.spy_return
 
         assert repo.url == str(tmp_dir)
         assert repo.config["cache"]["dir"] == str(cache_dir)