diff --git a/.travis.yml b/.travis.yml index 015910311..708101d07 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,18 @@ -language: python cache: directories: - $HOME/.cache/pip - - /opt/python - $HOME/.pyenv - - c/ProgramData/chocolatey/bin - - c/ProgramData/chocolatey/lib + - c/Users/appveyor/AppData/Local/Temp/chocolatey sudo: true dist: xenial -python: "3.7" env: global: + - HOMEBREW_NO_AUTO_UPDATE=1 - PYENV_ROOT="$HOME/.pyenv" - PATH="$PYENV_ROOT/bin:$PATH" - PIP_CACHE_DIR="$HOME/.cache/pip" # unify pip cache location for all platforms stages: + - check - test - build - name: deploy @@ -27,84 +25,80 @@ stages: # version, that causes fpm to crash. jobs: include: + # patch formatting checks + - name: "Check patch formatting" + stage: "check" + os: linux + language: python + python: 3.7 + before_install: + install: + script: ./scripts/ci/check_patch.sh + # test jobs - name: "2.7 on Windows" stage: test os: windows # Windows 10.0.17134 N/A Build 17134 language: shell # 'language: python' is an error on Travis CI Windows env: - - PYTHON_VER=2.7 + - PY2=1 - name: "3.7 on Windows" stage: test os: windows # Windows 10.0.17134 N/A Build 17134 language: shell # 'language: python' is an error on Travis CI Windows - env: - - PYTHON_VER=3.7 - - name: "2.7" + - name: "2.7 on Linux" stage: test + language: python + python: 2.7 env: - - PYTHON_VER=2.7.13 - - name: "3.5" + - PY2=1 + - name: "3.5 on Linux" stage: test - env: - - PYTHON_VER=3.5.3 - - name: "3.6" + language: python + python: 3.5 + - name: "3.6 on Linux" stage: test - env: - - PYTHON_VER=3.6.2 - - name: "3.7" + language: python + python: 3.6 + - name: "3.7 on Linux" stage: test - env: - # NOTE: using default python - - PYTHON_VER= + language: python + python: 3.7 - name: "2.7 on Mac" stage: test os: osx - osx_image: xcode7.3 - language: ruby - rvm: 2.4.0 + osx_image: xcode9.4 env: - - PYTHON_VER=2.7.10 - - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS" - - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS" - - PYTHON_CONFIGURE_OPTS="--enable-framework" - - name: "3.7 on Mac" + - PY2=1 + - name: "3.6 on Mac" stage: test os: osx - osx_image: xcode7.3 - language: ruby - rvm: 2.4.0 - env: - - PYTHON_VER=3.7.3 - - PYTHON_CONFIGURE_OPTS="--enable-framework" + osx_image: xcode9.4 # build jobs - name: "Mac pkg" stage: build os: osx - osx_image: xcode7.3 + osx_image: xcode9.4 language: ruby rvm: 2.4.0 - env: - - PYTHON_VER=3.7.3 - - PYTHON_CONFIGURE_OPTS="--enable-framework" + install: script: ./scripts/build_posix.sh - name: "Windows pkg" stage: build os: windows language: shell - env: - - PYTHON_VER=3.7 + install: script: ./scripts/build_windows.cmd - name: "Linux pkgs" stage: build - env: - # NOTE: using default python - - PYTHON_VER= + language: python + python: 3.7 + before_install: + install: script: ./scripts/build_posix.sh - name: "Pypi pkgs" stage: build - env: - # NOTE: using default python - - PYTHON_VER= + language: python + python: 3.7 script: ./scripts/build_package.sh # deploy jobs - name: "Pypi pkgs" diff --git a/scripts/build_posix.sh b/scripts/build_posix.sh index a58902e52..e09a9ef2a 100755 --- a/scripts/build_posix.sh +++ b/scripts/build_posix.sh @@ -71,7 +71,6 @@ install_dependencies() sudo apt-get update -y sudo apt-get install ruby-dev build-essential rpm python-pip python-dev elif command_exists brew; then - brew update brew install ruby else echo "Unable to install fpm dependencies" && exit 1 @@ -80,7 +79,7 @@ install_dependencies() gem install --no-document fpm print_info "Installing requirements..." - pip install -e .[all] + pip install .[all] print_info "Installing pyinstaller..." pip install pyinstaller diff --git a/scripts/ci/before_install.sh b/scripts/ci/before_install.sh index d1a09ff3f..19645bcad 100644 --- a/scripts/ci/before_install.sh +++ b/scripts/ci/before_install.sh @@ -11,70 +11,49 @@ if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo sysctl -w kern.maxprocperuid=2048 echo '\nulimit -u 2048' >> ~/.bash_profile ulimit -a - $scriptdir/retry.sh brew update - brew upgrade pyenv - # NOTE: used to install and run our style checking tools - osx_python_ver=3.6.2 - eval "$(pyenv init -)" - pyenv install --skip-existing --keep --verbose $osx_python_ver &> pyenv.log || tail -n 50 pyenv.log - pyenv shell $osx_python_ver - python --version -elif [[ "$TRAVIS_OS_NAME" == "windows" ]]; then - choco install python - # NOTE: used to install and run our style checking tools - PATH=/c/Python37:/c/Python37/Scripts:$PATH -fi - -pip install .[tests] - -# stop the build if there are any readme formatting errors -python setup.py checkdocs - -# stop the build if there are any black or flake8 errors -black ./ --check -flake8 ./ - -if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then - #NOTE: ssh keys for ssh test to be able to ssh to the localhost - ssh-keygen -t rsa -N "" -f mykey - mkdir -p ~/.ssh - cp mykey ~/.ssh/id_rsa - cp mykey.pub ~/.ssh/id_rsa.pub - cat mykey.pub >> ~/.ssh/authorized_keys - ssh-keyscan localhost >> ~/.ssh/known_hosts - ssh localhost ls &> /dev/null - ssh-keyscan 127.0.0.1 >> ~/.ssh/known_hosts - ssh 127.0.0.1 ls &> /dev/null - ssh-keyscan 0.0.0.0 >> ~/.ssh/known_hosts - ssh 0.0.0.0 ls &> /dev/null fi echo > env.sh -if [ "$TRAVIS_OS_NAME" == "linux" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ] -then - bash "$scriptdir/install_azurite.sh" - bash "$scriptdir/install_oss.sh" - bash "$scriptdir/install_hadoop.sh" -fi +if [[ "$TRAVIS_BUILD_STAGE_NAME" == "Test" ]]; then + if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then + # NOTE: ssh keys for ssh test to be able to ssh to the localhost + ssh-keygen -t rsa -N "" -f mykey + mkdir -p ~/.ssh + cp mykey ~/.ssh/id_rsa + cp mykey.pub ~/.ssh/id_rsa.pub + cat mykey.pub >> ~/.ssh/authorized_keys + ssh-keyscan localhost >> ~/.ssh/known_hosts + ssh localhost ls &> /dev/null + ssh-keyscan 127.0.0.1 >> ~/.ssh/known_hosts + ssh 127.0.0.1 ls &> /dev/null + ssh-keyscan 0.0.0.0 >> ~/.ssh/known_hosts + ssh 0.0.0.0 ls &> /dev/null + fi -if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - brew install openssl - $scriptdir/retry.sh brew cask install google-cloud-sdk -fi + if [ "$TRAVIS_OS_NAME" == "linux" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then + bash "$scriptdir/install_azurite.sh" + bash "$scriptdir/install_oss.sh" + bash "$scriptdir/install_hadoop.sh" + fi -if [[ -n "$PYTHON_VER" ]]; then - if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then - if [[ "$PYTHON_VER" == "2.7" ]]; then - choco install python2 - echo 'PATH="/c/Python27:/c/Python27/Scripts:$PATH"' >> env.sh - else - echo 'PATH="/c/Python37:/c/Python37/Scripts:$PATH"' >> env.sh - fi + if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then + brew install openssl + $scriptdir/retry.sh brew cask install google-cloud-sdk + fi +fi + +if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then + if [[ -n "$PY2" ]]; then + choco install python2 + echo 'PATH="/c/Python27:/c/Python27/Scripts:$PATH"' >> env.sh else - eval "$(pyenv init -)" - echo 'eval "$(pyenv init -)"' >> env.sh - pyenv install --skip-existing "$PYTHON_VER" - echo "pyenv global $PYTHON_VER" >> env.sh + choco install python + echo 'PATH="/c/Python37:/c/Python37/Scripts:$PATH"' >> env.sh + fi +elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + if [[ ! -n "$PY2" ]]; then + ln -s -f /usr/local/bin/python3 /usr/local/bin/python + ln -s -f /usr/local/bin/pip3 /usr/local/bin/pip fi fi diff --git a/scripts/ci/check_patch.sh b/scripts/ci/check_patch.sh new file mode 100755 index 000000000..7268ccf2a --- /dev/null +++ b/scripts/ci/check_patch.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -x +set -e + +pip install Pygments collective.checkdocs flake8 'black==19.3b0' + +# stop the build if there are any readme formatting errors +python setup.py checkdocs + +# stop the build if there are any black or flake8 errors +black ./ --check +flake8 ./ diff --git a/scripts/ci/check_python.sh b/scripts/ci/check_python.sh index 3ce7e5e5e..3ea756a2b 100644 --- a/scripts/ci/check_python.sh +++ b/scripts/ci/check_python.sh @@ -1,3 +1,5 @@ +#!/bin/bash + set -e set -x @@ -9,16 +11,13 @@ else which pip fi -if [[ -z "$PYTHON_VER" ]]; then - exit 0 -fi - -if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then - if [[ "$(python -c 'import sys; print(sys.version[:3])')" != "$PYTHON_VER" ]]; then +py_ver="$(python -c 'import sys; print(sys.version[0])')" +if [[ -n "$PY2" ]]; then + if [[ "$py_ver" != "2" ]]; then exit 1 fi else - if [[ "$(python --version 2>&1)" != "Python $PYTHON_VER" ]]; then + if [[ "$py_ver" != "3" ]]; then exit 1 fi fi