gha: build packages (#4607)

.github/workflows/check-patch.yaml

@@ -14,6 +14,8 @@ jobs:
       with:
         python-version: 3.8
     - name: Install requirements
-      run: bash ./scripts/ci/install.sh
-    - name: Check Patch
-      run: ./scripts/ci/check_patch.sh
+      run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit
+    - name: Check README
+      run: python setup.py checkdocs
+    - name: Check formatting
+      run: pre-commit run --all-files

.github/workflows/packages.yaml

@@ -0,0 +1,126 @@
+name: Build packages
+on: [push, pull_request, release]
+jobs:
+  linux:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Build
+      run: ./scripts/build_posix.sh
+    - name: Publish deb
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb
+        asset_content_type: binary/octet-stream
+    - name: Publish rpm
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm
+        asset_content_type: binary/octet-stream
+  mac:
+    runs-on: macos-10.15
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Set up Ruby 2.4
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.4
+    - name: Build
+      run: ./scripts/build_posix.sh
+    - name: Publish
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_name: dvc-${{ github.event.release.tag_name }}.pkg
+        asset_content_type: binary/octet-stream
+  windows:
+    runs-on: windows-2019
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Build
+      run: scripts\build_windows.cmd
+      shell: cmd
+    - name: Publish
+      if: github.event_name == 'release'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: dvc-${{ github.event.release.tag_name }}.exe
+        asset_name: dvc-${{ github.event.release.tag_name }}.exe
+        asset_content_type: binary/octet-stream
+  python:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Install
+      run: pip install ".[all,tests]"
+    - name: Build packages
+      run: ./scripts/build_package.sh
+    - name: Publish packages to PyPI
+      if: github.event_name == 'release'
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_TOKEN }}
+  snap:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build the package
+      uses: snapcore/action-build@v1
+      id: snapcraft
+    - name: Test the package
+      run: |
+        sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }}
+        dvc version
+        sudo snap remove dvc
+    - name: Publish to the edge channel
+      uses: snapcore/action-publish@v1
+      if: github.event_name == 'push'
+      with:
+        store_login: ${{ secrets.STORE_LOGIN }}
+        snap: ${{ steps.snapcraft.outputs.snap }}
+        release: edge
+    - name: Publish to the stable channel
+      uses: snapcore/action-publish@v1
+      if: github.event_name == 'release'
+      with:
+        store_login: ${{ secrets.STORE_LOGIN }}
+        snap: ${{ steps.snapcraft.outputs.snap }}
+        release: stable

.travis.yml

@@ -1,136 +0,0 @@
-cache:
-  directories:
-    - $HOME/.cache/pip
-    - $HOME/.pyenv
-dist: xenial
-env:
-  global:
-    - HOMEBREW_NO_AUTO_UPDATE=1
-    - PYENV_ROOT="$HOME/.pyenv"
-    - PATH="$PYENV_ROOT/bin:$PATH"
-    - PIP_CACHE_DIR="$HOME/.cache/pip"  # unify pip cache location for all platforms
-stages:
-  - check
-  - test
-  - build
-# Travis doesn't support testing python on Mac yet, so we need
-# to workaround it by installing it directly with brew.
-#
-# Note that we specify language as ruby-2.4.3 to help us satisfy
-# fpm dependencies, as travis runs el capitan with an old ruby
-# version, that causes fpm to crash.
-#
-# Each job will use the previous job's `stage` tag if not specified.
-jobs:
-  include:
-  # patch formatting checks
-  - name: "Check patch formatting"
-    stage: check
-    os: linux
-    language: python
-    python: 3.7
-    before_install: bash ./scripts/ci/install.sh
-    install:
-    script: ./scripts/ci/check_patch.sh
-  # build jobs
-  - name: "Mac pkg"
-    stage: build
-    os: osx
-    osx_image: xcode9.4
-    language: ruby
-    rvm: 2.4.3
-    install:
-    script: ./scripts/build_posix.sh
-  - name: "Windows pkg"
-    os: windows
-    language: shell
-    install:
-    script: ./scripts/build_windows.cmd
-    env:
-      - PYTHON_VERSION=3.7.5
-      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
-  - name: "Linux pkgs"
-    language: python
-    python: 3.7
-    before_install:
-    install:
-    script: ./scripts/build_posix.sh
-  - name: "PyPI pkgs"
-    language: python
-    python: 3.8
-    script: ./scripts/build_package.sh
-  - name: Snapcraft snap
-    language: python
-    python: 3.8
-    addons:
-     snaps:
-     - name: snapcraft
-       channel: stable
-       confinement: classic
-     - name: review-tools
-       channel: stable
-     - name: lxd
-       channel: stable
-    env:
-    - SNAPCRAFT_IMAGE_INFO: |
-        '{"build_url": "$TRAVIS_JOB_WEB_URL"}'
-    - SNAPCRAFT_BUILD_ENVIRONMENT: lxd
-    - SNAPCRAFT_BUILD_INFO: 1  # https://snapcraft.io/blog/introducing-developer-notifications-for-snap-security-updates
-    install:
-    - sudo usermod --append --groups lxd $USER
-    - sudo /snap/bin/lxd.migrate -yes
-    - sudo /snap/bin/lxd waitready
-    - sudo /snap/bin/lxd init --auto
-    script:
-    - ./scripts/build_snap.sh
-    after_failure:
-    - sudo journalctl -u snapd
-
-before_install:
-  - bash ./scripts/ci/before_install.sh
-  - source env.sh
-  - bash ./scripts/ci/check_python.sh
-install:
-  - bash ./scripts/ci/install.sh
-script:
-  - bash ./scripts/ci/script.sh
-deploy:
-  - provider: releases
-    api_key:
-      secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=
-    file_glob: true
-    file:
-      - dvc*.rpm
-      - dvc*.deb
-      - dvc*.pkg
-      - dvc*.exe
-    skip_cleanup: true
-    on:
-      condition: "$(./scripts/ci/deploy_condition.sh dvc*.rpm dvc*.deb dvc*.pkg dvc*.exe)"
-      tags: true
-      repo: iterative/dvc
-      stage: build
-  - provider: pypi
-    user:
-      secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
-    password:
-      secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
-    skip_existing: true
-    file_glob: true
-    file:
-      - dist/dvc*.whl
-      - dist/dvc*.tar.gz
-    skip_cleanup: true
-    on:
-      condition: "$(./scripts/ci/deploy_condition.sh dist/dvc*.whl dist/dvc*.tar.gz)"
-      tags: true
-      repo: iterative/dvc
-      stage: build
-  - provider: script
-    skip_cleanup: true
-    script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out"
-    on:
-      all_branches: true
-      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL"
-      repo: iterative/dvc
-      stage: build

scripts/build_snap.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-set -e
-set -x
-
-if [ ! -d "dvc" ]; then
-  echo "Please run this script from repository root"
-  exit 1
-fi
-
-sg lxd -c snapcraft
-
-pip uninstall -y dvc
-if which dvc; then
-  echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2
-  exit 1
-fi
-sudo snap install --dangerous --classic dvc_*.snap
-if [[ -n "$TRAVIS_TAG" ]]; then
-  # Make sure we have a correct version
-  if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then
-    echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2
-    exit 1
-  fi
-fi
-# ensure basic commands can run
-# N.B.: cannot run `dvc get` on travis (#2956)
-dvc version
-dvc.git status
-dvc.git fetch --all
-sudo snap remove dvc

scripts/ci/before_install.sh

@@ -1,53 +0,0 @@
-#!/bin/bash
-
-set -x
-set -e
-
-scriptdir="$(dirname $0)"
-
-if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-  ulimit -a
-  sudo sysctl -w kern.maxproc=2048
-  sudo sysctl -w kern.maxprocperuid=2048
-  echo '\nulimit -u 2048' >>~/.bash_profile
-  ulimit -a
-fi
-
-echo >env.sh
-
-if [[ "$TRAVIS_BUILD_STAGE_NAME" == "test" ]]; then
-  if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then
-    brew install openssl
-    $scriptdir/retry.sh brew cask install google-cloud-sdk
-  fi
-fi
-
-if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
-  $scriptdir/retry.sh choco install python --version $PYTHON_VERSION
-  echo "PATH='$PATH'" >>env.sh
-elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-  ln -s -f /usr/local/bin/python3 /usr/local/bin/python
-  ln -s -f /usr/local/bin/pip3 /usr/local/bin/pip
-fi
-
-if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-  # fetch tags for `git-describe`, since
-  # - can't rely on $TRAVIS_TAG for snapcraft `edge` (master) releases, and
-  # - `snapcraft` also uses `git-describe` for version detection
-  git fetch --tags
-  TAG_MAJOR="$(git describe --tags | sed -r 's/^v?([0-9]+)\.[0-9]+\.[0-9]+.*/\1/')"
-  [[ -n "$TAG_MAJOR" ]] || exit 1  # failed to detect major version
-
-  if [[ -n "$TRAVIS_TAG" ]]; then
-    if [[ $(echo "$TRAVIS_TAG" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$') ]]; then
-      SNAP_CHANNEL=stable
-    else
-      SNAP_CHANNEL=beta
-    fi
-  else
-    SNAP_CHANNEL=edge
-  fi
-  echo "export SNAP_CHANNEL=${SNAP_CHANNEL},v${TAG_MAJOR}/${SNAP_CHANNEL}" >>env.sh
-  # NOTE: after deprecating this major version branch, uncomment this line
-  # echo "export SNAP_CHANNEL=v${TAG_MAJOR}/${SNAP_CHANNEL}" >>env.sh
-fi

scripts/ci/check_patch.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-set -x
-set -e
-
-pip install Pygments collective.checkdocs pre-commit .
-
-# stop the build if there are any readme formatting errors
-python setup.py checkdocs
-
-# stop the build if there are any formatting errors
-pre-commit run --all-files

scripts/ci/check_python.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -e
-set -x
-
-if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
-  where python
-  where pip
-else
-  which python
-  which pip
-fi
-
-py_ver="$(python -c 'import sys; print(sys.version[0])')"
-
-if [[ "$py_ver" != 3 ]]; then
-  exit 1
-fi

scripts/ci/deploy_condition.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-set -e
-
-if [[ $TRAVIS_EVENT_TYPE = pull_request || $TRAVIS_EVENT_TPYE = cron ]]; then
-  exit 2
-fi
-
-# positional args are assumed to be file glob patterns to deploy
-if [[ ${#} -ge 1 ]]; then
-  # ensure at least one file exists
-  [[ -n "$(ls $@ 2>/dev/null)" ]] || exit 3
-fi
-
-echo true

scripts/ci/install.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-set -x
-set -e
-
-scriptdir="$(dirname $0)"
-
-# NOTE: it is not uncommon for pip to hang on travis for what seems to be
-# networking issues. Thus, let's retry a few times to see if it will eventually
-# work or not.
-$scriptdir/retry.sh pip install .[all,tests]
-
-git config --global user.email "[email protected]"
-git config --global user.name "DVC Tester"
-
-if [[ "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
-  aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
-  aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
-  aws configure set region us-west-2
-
-  openssl enc -d -aes-256-cbc -md md5 -k $GCP_CREDS -in scripts/ci/gcp-creds.json.enc -out scripts/ci/gcp-creds.json
-fi

scripts/ci/retry.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -e
-
-N_RETRIES=3
-for i in $(seq $N_RETRIES); do
-  echo "Attempt #$i"
-  $@ && exit $?
-done
-
-exit $?

scripts/ci/script.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -x
-set -e
-
-python -mtests
-
-if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
-  pip install codecov
-  codecov
-fi

snap/snapcraft.yaml

@@ -29,7 +29,6 @@ parts:
     - git
     override-pull: |
         snapcraftctl pull
-        snapcraftctl set-version $(cd $SNAPCRAFT_PART_SRC && git describe --tags)
         git diff --quiet || error_dirty_build
         echo 'PKG = "snap"' > dvc/utils/build.py
         # install all optional extras
@@ -45,6 +44,7 @@ parts:
         touch $SNAPCRAFT_PART_INSTALL/completion.sh
     override-stage: |
         snapcraftctl stage
+        snapcraftctl set-version $(usr/bin/python3 -m dvc --version)
         usr/bin/python3 -m dvc completion -s bash > completion.sh
 apps:
   dvc: