diff --git a/.github/workflows/check-patch.yaml b/.github/workflows/check-patch.yaml index 8d9f5a062..6442f6a17 100644 --- a/.github/workflows/check-patch.yaml +++ b/.github/workflows/check-patch.yaml @@ -14,6 +14,8 @@ jobs: with: python-version: 3.8 - name: Install requirements - run: bash ./scripts/ci/install.sh - - name: Check Patch - run: ./scripts/ci/check_patch.sh + run: pip install ".[all,tests]" Pygments collective.checkdocs pre-commit + - name: Check README + run: python setup.py checkdocs + - name: Check formatting + run: pre-commit run --all-files diff --git a/.github/workflows/packages.yaml b/.github/workflows/packages.yaml new file mode 100644 index 000000000..cc649b3db --- /dev/null +++ b/.github/workflows/packages.yaml @@ -0,0 +1,126 @@ +name: Build packages +on: [push, pull_request, release] +jobs: + linux: + runs-on: ubuntu-16.04 + steps: + - uses: actions/checkout@v1 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Set up Ruby 2.4 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.4 + - name: Build + run: ./scripts/build_posix.sh + - name: Publish deb + if: github.event_name == 'release' + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: dvc_${{ github.event.release.tag_name }}_amd64.deb + asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb + asset_content_type: binary/octet-stream + - name: Publish rpm + if: github.event_name == 'release' + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm + asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm + asset_content_type: binary/octet-stream + mac: + runs-on: macos-10.15 + steps: + - uses: actions/checkout@v1 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Set up Ruby 2.4 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.4 + - name: Build + run: ./scripts/build_posix.sh + - name: Publish + if: github.event_name == 'release' + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: dvc-${{ github.event.release.tag_name }}.pkg + asset_name: dvc-${{ github.event.release.tag_name }}.pkg + asset_content_type: binary/octet-stream + windows: + runs-on: windows-2019 + steps: + - uses: actions/checkout@v1 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Build + run: scripts\build_windows.cmd + shell: cmd + - name: Publish + if: github.event_name == 'release' + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: dvc-${{ github.event.release.tag_name }}.exe + asset_name: dvc-${{ github.event.release.tag_name }}.exe + asset_content_type: binary/octet-stream + python: + runs-on: ubuntu-16.04 + steps: + - uses: actions/checkout@v1 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install + run: pip install ".[all,tests]" + - name: Build packages + run: ./scripts/build_package.sh + - name: Publish packages to PyPI + if: github.event_name == 'release' + uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.PYPI_TOKEN }} + snap: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Build the package + uses: snapcore/action-build@v1 + id: snapcraft + - name: Test the package + run: | + sudo snap install --dangerous --classic ${{ steps.snapcraft.outputs.snap }} + dvc version + sudo snap remove dvc + - name: Publish to the edge channel + uses: snapcore/action-publish@v1 + if: github.event_name == 'push' + with: + store_login: ${{ secrets.STORE_LOGIN }} + snap: ${{ steps.snapcraft.outputs.snap }} + release: edge + - name: Publish to the stable channel + uses: snapcore/action-publish@v1 + if: github.event_name == 'release' + with: + store_login: ${{ secrets.STORE_LOGIN }} + snap: ${{ steps.snapcraft.outputs.snap }} + release: stable diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 052574898..000000000 --- a/.travis.yml +++ /dev/null @@ -1,136 +0,0 @@ -cache: - directories: - - $HOME/.cache/pip - - $HOME/.pyenv -dist: xenial -env: - global: - - HOMEBREW_NO_AUTO_UPDATE=1 - - PYENV_ROOT="$HOME/.pyenv" - - PATH="$PYENV_ROOT/bin:$PATH" - - PIP_CACHE_DIR="$HOME/.cache/pip" # unify pip cache location for all platforms -stages: - - check - - test - - build -# Travis doesn't support testing python on Mac yet, so we need -# to workaround it by installing it directly with brew. -# -# Note that we specify language as ruby-2.4.3 to help us satisfy -# fpm dependencies, as travis runs el capitan with an old ruby -# version, that causes fpm to crash. -# -# Each job will use the previous job's `stage` tag if not specified. -jobs: - include: - # patch formatting checks - - name: "Check patch formatting" - stage: check - os: linux - language: python - python: 3.7 - before_install: bash ./scripts/ci/install.sh - install: - script: ./scripts/ci/check_patch.sh - # build jobs - - name: "Mac pkg" - stage: build - os: osx - osx_image: xcode9.4 - language: ruby - rvm: 2.4.3 - install: - script: ./scripts/build_posix.sh - - name: "Windows pkg" - os: windows - language: shell - install: - script: ./scripts/build_windows.cmd - env: - - PYTHON_VERSION=3.7.5 - - PATH=/c/Python37:/c/Python37/Scripts:$PATH - - name: "Linux pkgs" - language: python - python: 3.7 - before_install: - install: - script: ./scripts/build_posix.sh - - name: "PyPI pkgs" - language: python - python: 3.8 - script: ./scripts/build_package.sh - - name: Snapcraft snap - language: python - python: 3.8 - addons: - snaps: - - name: snapcraft - channel: stable - confinement: classic - - name: review-tools - channel: stable - - name: lxd - channel: stable - env: - - SNAPCRAFT_IMAGE_INFO: | - '{"build_url": "$TRAVIS_JOB_WEB_URL"}' - - SNAPCRAFT_BUILD_ENVIRONMENT: lxd - - SNAPCRAFT_BUILD_INFO: 1 # https://snapcraft.io/blog/introducing-developer-notifications-for-snap-security-updates - install: - - sudo usermod --append --groups lxd $USER - - sudo /snap/bin/lxd.migrate -yes - - sudo /snap/bin/lxd waitready - - sudo /snap/bin/lxd init --auto - script: - - ./scripts/build_snap.sh - after_failure: - - sudo journalctl -u snapd - -before_install: - - bash ./scripts/ci/before_install.sh - - source env.sh - - bash ./scripts/ci/check_python.sh -install: - - bash ./scripts/ci/install.sh -script: - - bash ./scripts/ci/script.sh -deploy: - - provider: releases - api_key: - secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg= - file_glob: true - file: - - dvc*.rpm - - dvc*.deb - - dvc*.pkg - - dvc*.exe - skip_cleanup: true - on: - condition: "$(./scripts/ci/deploy_condition.sh dvc*.rpm dvc*.deb dvc*.pkg dvc*.exe)" - tags: true - repo: iterative/dvc - stage: build - - provider: pypi - user: - secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4=" - password: - secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc=" - skip_existing: true - file_glob: true - file: - - dist/dvc*.whl - - dist/dvc*.tar.gz - skip_cleanup: true - on: - condition: "$(./scripts/ci/deploy_condition.sh dist/dvc*.whl dist/dvc*.tar.gz)" - tags: true - repo: iterative/dvc - stage: build - - provider: script - skip_cleanup: true - script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out" - on: - all_branches: true - condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL" - repo: iterative/dvc - stage: build diff --git a/scripts/build_snap.sh b/scripts/build_snap.sh deleted file mode 100755 index 3d1ea8547..000000000 --- a/scripts/build_snap.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -e -set -x - -if [ ! -d "dvc" ]; then - echo "Please run this script from repository root" - exit 1 -fi - -sg lxd -c snapcraft - -pip uninstall -y dvc -if which dvc; then - echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2 - exit 1 -fi -sudo snap install --dangerous --classic dvc_*.snap -if [[ -n "$TRAVIS_TAG" ]]; then - # Make sure we have a correct version - if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then - echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2 - exit 1 - fi -fi -# ensure basic commands can run -# N.B.: cannot run `dvc get` on travis (#2956) -dvc version -dvc.git status -dvc.git fetch --all -sudo snap remove dvc diff --git a/scripts/ci/before_install.sh b/scripts/ci/before_install.sh deleted file mode 100644 index 7404832a1..000000000 --- a/scripts/ci/before_install.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -set -x -set -e - -scriptdir="$(dirname $0)" - -if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - ulimit -a - sudo sysctl -w kern.maxproc=2048 - sudo sysctl -w kern.maxprocperuid=2048 - echo '\nulimit -u 2048' >>~/.bash_profile - ulimit -a -fi - -echo >env.sh - -if [[ "$TRAVIS_BUILD_STAGE_NAME" == "test" ]]; then - if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then - brew install openssl - $scriptdir/retry.sh brew cask install google-cloud-sdk - fi -fi - -if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then - $scriptdir/retry.sh choco install python --version $PYTHON_VERSION - echo "PATH='$PATH'" >>env.sh -elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - ln -s -f /usr/local/bin/python3 /usr/local/bin/python - ln -s -f /usr/local/bin/pip3 /usr/local/bin/pip -fi - -if [ "$TRAVIS_OS_NAME" == "linux" ]; then - # fetch tags for `git-describe`, since - # - can't rely on $TRAVIS_TAG for snapcraft `edge` (master) releases, and - # - `snapcraft` also uses `git-describe` for version detection - git fetch --tags - TAG_MAJOR="$(git describe --tags | sed -r 's/^v?([0-9]+)\.[0-9]+\.[0-9]+.*/\1/')" - [[ -n "$TAG_MAJOR" ]] || exit 1 # failed to detect major version - - if [[ -n "$TRAVIS_TAG" ]]; then - if [[ $(echo "$TRAVIS_TAG" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$') ]]; then - SNAP_CHANNEL=stable - else - SNAP_CHANNEL=beta - fi - else - SNAP_CHANNEL=edge - fi - echo "export SNAP_CHANNEL=${SNAP_CHANNEL},v${TAG_MAJOR}/${SNAP_CHANNEL}" >>env.sh - # NOTE: after deprecating this major version branch, uncomment this line - # echo "export SNAP_CHANNEL=v${TAG_MAJOR}/${SNAP_CHANNEL}" >>env.sh -fi diff --git a/scripts/ci/check_patch.sh b/scripts/ci/check_patch.sh deleted file mode 100755 index 543a87b67..000000000 --- a/scripts/ci/check_patch.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -x -set -e - -pip install Pygments collective.checkdocs pre-commit . - -# stop the build if there are any readme formatting errors -python setup.py checkdocs - -# stop the build if there are any formatting errors -pre-commit run --all-files diff --git a/scripts/ci/check_python.sh b/scripts/ci/check_python.sh deleted file mode 100644 index b4ca08932..000000000 --- a/scripts/ci/check_python.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e -set -x - -if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then - where python - where pip -else - which python - which pip -fi - -py_ver="$(python -c 'import sys; print(sys.version[0])')" - -if [[ "$py_ver" != 3 ]]; then - exit 1 -fi diff --git a/scripts/ci/deploy_condition.sh b/scripts/ci/deploy_condition.sh deleted file mode 100755 index 6b9549434..000000000 --- a/scripts/ci/deploy_condition.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -e - -if [[ $TRAVIS_EVENT_TYPE = pull_request || $TRAVIS_EVENT_TPYE = cron ]]; then - exit 2 -fi - -# positional args are assumed to be file glob patterns to deploy -if [[ ${#} -ge 1 ]]; then - # ensure at least one file exists - [[ -n "$(ls $@ 2>/dev/null)" ]] || exit 3 -fi - -echo true diff --git a/scripts/ci/gcp-creds.json.enc b/scripts/ci/gcp-creds.json.enc deleted file mode 100644 index d18531f9d..000000000 Binary files a/scripts/ci/gcp-creds.json.enc and /dev/null differ diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh deleted file mode 100644 index f1a5defc0..000000000 --- a/scripts/ci/install.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -set -x -set -e - -scriptdir="$(dirname $0)" - -# NOTE: it is not uncommon for pip to hang on travis for what seems to be -# networking issues. Thus, let's retry a few times to see if it will eventually -# work or not. -$scriptdir/retry.sh pip install .[all,tests] - -git config --global user.email "dvctester@example.com" -git config --global user.name "DVC Tester" - -if [[ "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region us-west-2 - - openssl enc -d -aes-256-cbc -md md5 -k $GCP_CREDS -in scripts/ci/gcp-creds.json.enc -out scripts/ci/gcp-creds.json -fi diff --git a/scripts/ci/retry.sh b/scripts/ci/retry.sh deleted file mode 100755 index 07458c994..000000000 --- a/scripts/ci/retry.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -e - -N_RETRIES=3 -for i in $(seq $N_RETRIES); do - echo "Attempt #$i" - $@ && exit $? -done - -exit $? diff --git a/scripts/ci/script.sh b/scripts/ci/script.sh deleted file mode 100644 index 37bfa37f0..000000000 --- a/scripts/ci/script.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -x -set -e - -python -mtests - -if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then - pip install codecov - codecov -fi diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 6d3160ed9..f682672cf 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -29,7 +29,6 @@ parts: - git override-pull: | snapcraftctl pull - snapcraftctl set-version $(cd $SNAPCRAFT_PART_SRC && git describe --tags) git diff --quiet || error_dirty_build echo 'PKG = "snap"' > dvc/utils/build.py # install all optional extras @@ -45,6 +44,7 @@ parts: touch $SNAPCRAFT_PART_INSTALL/completion.sh override-stage: | snapcraftctl stage + snapcraftctl set-version $(usr/bin/python3 -m dvc --version) usr/bin/python3 -m dvc completion -s bash > completion.sh apps: dvc: