gha: don't upload pkg artifacts

* build: add osx signing/notarization

* gha: temporarily enable pkg builds

.github/workflows/packages.yaml

@@ -25,11 +25,6 @@ jobs:
         gem install --no-document fpm
     - name: Build deb
       run: python scripts/build.py deb
-    - name: Upload deb artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/fpm/dvc*.deb
-        if-no-files-found: error
     - name: Publish deb
       if: github.event_name == 'release'
       uses: actions/[email protected]
@@ -59,11 +54,6 @@ jobs:
         gem install --no-document fpm
     - name: Build rpm
       run: python scripts/build.py rpm
-    - name: Upload rpm artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/fpm/dvc*.rpm
-        if-no-files-found: error
     - name: Publish rpm
       if: github.event_name == 'release'
       uses: actions/[email protected]
@@ -88,16 +78,14 @@ jobs:
         ruby-version: 2.4
     - name: Install deps
       run: |
-        pip install .[all]
+        # available lxml (webdav dependency) wheels are built with an old
+        # SDK, which breaks osxpkg notarization. Building from scratch
+        # instead.
+        pip install .[all] --no-binary lxml
         pip install -r scripts/build-requirements.txt
         gem install --no-document fpm
     - name: Build
       run: python scripts/build.py osxpkg
-    - name: Upload osxpkg artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/fpm/dvc*.pkg
-        if-no-files-found: error
     - name: Publish
       if: github.event_name == 'release'
       uses: actions/[email protected]
@@ -126,11 +114,6 @@ jobs:
     - name: Build
       run: python scripts\build.py exe
       shell: cmd
-    - name: Upload exe artifact
-      uses: actions/upload-artifact@v2
-      with:
-        path: scripts/innosetup/dvc*.exe
-        if-no-files-found: error
     - name: Publish
       if: github.event_name == 'release'
       env:

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.0.8"
+_BASE_VERSION = "2.0.11"
 
 
 def _generate_version(base_version):

scripts/build.py

@@ -14,6 +14,13 @@ parser = argparse.ArgumentParser()
 parser.add_argument(
     "pkg", choices=["deb", "rpm", "osxpkg", "exe"], help="package type"
 )
+parser.add_argument("--sign-application", default=False, action="store_true")
+parser.add_argument("--application-id")
+parser.add_argument("--sign-installer", default=False, action="store_true")
+parser.add_argument("--installer-id")
+parser.add_argument("--notarize")
+parser.add_argument("--apple-id-username")
+parser.add_argument("--apple-id-password")
 args = parser.parse_args()
 
 (dvc / "utils" / "build.py").write_text(f'PKG = "{args.pkg}"')
@@ -39,6 +46,18 @@ check_call(
     ["python", "build.py"], cwd=pyinstaller, stderr=STDOUT,
 )
 
+if args.sign_application:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not args.application_id:
+        print("--sign-application requires --application-id")
+        exit(1)
+    check_call(
+        ["python", "sign.py", "--application-id", args.application_id],
+        cwd=pyinstaller,
+        stderr=STDOUT,
+    )
+
 if args.pkg == "exe":
     check_call(
         ["python", "build.py"], cwd=innosetup, stderr=STDOUT,
@@ -47,3 +66,38 @@ else:
     check_call(
         ["python", "build.py", args.pkg], cwd=fpm, stderr=STDOUT,
     )
+
+if args.sign_installer:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not all(
+        [args.installer_id, args.apple_id_username, args.apple_id_password]
+    ):
+        print("--sign-installer requires --installer-id")
+        exit(1)
+    check_call(
+        ["python", "sign.py", "--installer-id", args.installer_id],
+        cwd=fpm,
+        stderr=STDOUT,
+    )
+
+if args.notarize:
+    if args.pkg != "osxpkg":
+        raise NotImplementedError
+    if not all([args.apple_id_username, args.apple_id_password]):
+        print(
+            "--notarize requires --apple-id-username and --apple-id-password"
+        )
+        exit(1)
+    check_call(
+        [
+            "python",
+            "notarize.py",
+            "--apple-id-username",
+            args.apple_id_username,
+            "--apple-id-password",
+            args.apple_id_password,
+        ],
+        cwd=fpm,
+        stderr=STDOUT,
+    )

scripts/fpm/build.py

@@ -69,7 +69,7 @@ bash_completion = check_output(
 )
 (bash_dir / "dvc").write_text(bash_completion)
 
-zsh_dir = build / "usr" / "share" / "zsh" / "site-funtions"
+zsh_dir = install / "share" / "zsh" / "site-funtions"
 zsh_dir.mkdir(parents=True)
 zsh_completion = check_output(
     [lib / "dvc" / "dvc", "completion", "-s", "zsh"], text=True

scripts/fpm/notarize.py

@@ -0,0 +1,64 @@
+import argparse
+import json
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "Darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "path",
+    nargs="?",
+    help="Path to the osxpkg to notarize. If not specified - try to find one.",
+)
+parser.add_argument(
+    "--apple-id-username", required=True, help="Apple ID username."
+)
+parser.add_argument(
+    "--apple-id-password",
+    required=True,
+    help=(
+        "Apple ID app-specific password. Note that this is not a regular "
+        "Apple ID password, so you need to generate one at "
+        "https://appleid.apple.com/account/manage"
+    ),
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+
+if args.path:
+    pkg = pathlib.Path(args.path)
+else:
+    pkgs = list(path.glob("*.pkg"))
+    if not pkgs:
+        print("No pkgs found")
+        exit(1)
+
+    if len(pkgs) > 1:
+        print("Too many packages")
+        exit(1)
+
+    (pkg,) = pkgs
+
+
+config = {
+    "notarize": {
+        "path": os.fspath(pkg),
+        "bundle_id": "com.iterative.dvc",
+        "staple": True,
+    },
+    "apple_id": {
+        "username": args.apple_id_username,
+        "password": args.apple_id_password,
+    },
+}
+
+(path / "config.json").write_text(json.dumps(config))
+
+check_call(
+    ["gon", "config.json"], cwd=path, stderr=STDOUT,
+)

scripts/fpm/sign.py

@@ -0,0 +1,54 @@
+import argparse
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "path",
+    nargs="?",
+    help="Path to the osxpkg to sign. If not specified - try to find one.",
+)
+parser.add_argument(
+    "--installer-id",
+    required=True,
+    help="Certificate ID (should be added to the keychain).",
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+
+if args.path:
+    pkg = pathlib.Path(args.path)
+else:
+    pkgs = list(path.glob("*.pkg"))
+    if not pkgs:
+        print("No pkgs found")
+        exit(1)
+
+    if len(pkgs) > 1:
+        print("Too many packages")
+        exit(1)
+
+    (pkg,) = pkgs
+
+unsigned = pkg.with_suffix(".unsigned")
+os.rename(pkg, unsigned)
+check_call(
+    [
+        "productsign",
+        "--sign",
+        args.installer_id,
+        os.fspath(unsigned),
+        os.fspath(pkg),
+    ],
+    stderr=STDOUT,
+)
+
+check_call(
+    ["pkgutil", "--check-signature", os.fspath(pkg)], stderr=STDOUT,
+)

scripts/pyinstaller/entitlements.plist

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<!-- These are required for binaries built by PyInstaller -->
+	<key>com.apple.security.cs.allow-jit</key>
+	<true/>
+	<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+	<true/>
+</dict>
+</plist>

scripts/pyinstaller/sign.py

@@ -0,0 +1,37 @@
+import argparse
+import os
+import pathlib
+import sys
+from subprocess import STDOUT, check_call
+
+if sys.platform != "darwin":
+    raise NotImplementedError
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--application-id",
+    required=True,
+    help="Certificate ID (should be added to the keychain).",
+)
+args = parser.parse_args()
+
+path = pathlib.Path(__file__).parent.absolute()
+dvc = path / "dist" / "dvc"
+for root, _, fnames in os.walk(dvc):
+    for fname in fnames:
+        fpath = os.path.join(root, fname)
+        print(f"signing {fpath}")
+        check_call(
+            [
+                "codesign",
+                "--force",
+                "-s",
+                args.application_id,
+                "-o",
+                "runtime",
+                "--entitlements",
+                "entitlements.plist",
+                fpath,
+            ],
+            stderr=STDOUT,
+        )