diff --git a/.github/workflows/packages.yaml b/.github/workflows/packages.yaml index 681d5b683..deaf92564 100644 --- a/.github/workflows/packages.yaml +++ b/.github/workflows/packages.yaml @@ -6,7 +6,7 @@ on: release: types: [released, prereleased] jobs: - linux: + deb: runs-on: ubuntu-16.04 steps: - uses: actions/checkout@v2.3.2 @@ -18,8 +18,18 @@ jobs: uses: actions/setup-ruby@v1 with: ruby-version: 2.4 - - name: Build - run: ./scripts/build_posix.sh + - name: Install deps + run: | + pip install .[all] + pip install -r scripts/build-requirements.txt + gem install --no-document fpm + - name: Build deb + run: python scripts/build.py deb + - name: Upload deb artifact + uses: actions/upload-artifact@v2 + with: + path: scripts/fpm/dvc*.deb + if-no-files-found: error - name: Publish deb if: github.event_name == 'release' uses: actions/upload-release-asset@v1.0.1 @@ -27,9 +37,33 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ github.event.release.upload_url }} - asset_path: dvc_${{ github.event.release.tag_name }}_amd64.deb + asset_path: scripts/fpm/dvc_${{ github.event.release.tag_name }}_amd64.deb asset_name: dvc_${{ github.event.release.tag_name }}_amd64.deb asset_content_type: binary/octet-stream + rpm: + runs-on: ubuntu-16.04 + steps: + - uses: actions/checkout@v2.3.2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2.1.4 + with: + python-version: 3.7 + - name: Set up Ruby 2.4 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.4 + - name: Install deps + run: | + pip install .[all] + pip install -r scripts/build-requirements.txt + gem install --no-document fpm + - name: Build rpm + run: python scripts/build.py rpm + - name: Upload rpm artifact + uses: actions/upload-artifact@v2 + with: + path: scripts/fpm/dvc*.rpm + if-no-files-found: error - name: Publish rpm if: github.event_name == 'release' uses: actions/upload-release-asset@v1.0.1 @@ -37,10 +71,10 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ github.event.release.upload_url }} - asset_path: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm + asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm asset_name: dvc-${{ github.event.release.tag_name }}-1.x86_64.rpm asset_content_type: binary/octet-stream - mac: + osxpkg: runs-on: macos-10.15 steps: - uses: actions/checkout@v2.3.2 @@ -52,8 +86,18 @@ jobs: uses: actions/setup-ruby@v1 with: ruby-version: 2.4 + - name: Install deps + run: | + pip install .[all] + pip install -r scripts/build-requirements.txt + gem install --no-document fpm - name: Build - run: ./scripts/build_posix.sh + run: python scripts/build.py osxpkg + - name: Upload osxpkg artifact + uses: actions/upload-artifact@v2 + with: + path: scripts/fpm/dvc*.pkg + if-no-files-found: error - name: Publish if: github.event_name == 'release' uses: actions/upload-release-asset@v1.0.1 @@ -61,10 +105,10 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ github.event.release.upload_url }} - asset_path: dvc-${{ github.event.release.tag_name }}.pkg + asset_path: scripts/fpm/dvc-${{ github.event.release.tag_name }}.pkg asset_name: dvc-${{ github.event.release.tag_name }}.pkg asset_content_type: binary/octet-stream - windows: + exe: runs-on: windows-2019 steps: - uses: actions/checkout@v2.3.2 @@ -72,9 +116,21 @@ jobs: uses: actions/setup-python@v2.1.4 with: python-version: 3.7 + - name: Install deps + run: | + pip install .[all] + pip install -r scripts/build-requirements.txt + - name: Pull images + run: | + dvc pull - name: Build - run: scripts\build_windows.cmd + run: python scripts\build.py exe shell: cmd + - name: Upload exe artifact + uses: actions/upload-artifact@v2 + with: + path: scripts/innosetup/dvc*.exe + if-no-files-found: error - name: Publish if: github.event_name == 'release' env: @@ -82,10 +138,10 @@ jobs: uses: actions/upload-release-asset@v1.0.1 with: upload_url: ${{ github.event.release.upload_url }} - asset_path: dvc-${{ github.event.release.tag_name }}.exe + asset_path: scripts/innosetup/dvc-${{ github.event.release.tag_name }}.exe asset_name: dvc-${{ github.event.release.tag_name }}.exe asset_content_type: binary/octet-stream - python: + pip: runs-on: ubuntu-16.04 steps: - uses: actions/checkout@v2.3.2 diff --git a/scripts/build.py b/scripts/build.py new file mode 100644 index 000000000..2250d0f13 --- /dev/null +++ b/scripts/build.py @@ -0,0 +1,49 @@ +import argparse +import importlib +import pathlib +import textwrap +from subprocess import STDOUT, check_call + +path = pathlib.Path(__file__).parent.absolute() +dvc = path.parent / "dvc" +pyinstaller = path / "pyinstaller" +innosetup = path / "innosetup" +fpm = path / "fpm" + +parser = argparse.ArgumentParser() +parser.add_argument( + "pkg", choices=["deb", "rpm", "osxpkg", "exe"], help="package type" +) +args = parser.parse_args() + +(dvc / "utils" / "build.py").write_text(f'PKG = "{args.pkg}"') + +# Autogenerate version, similar to what we do in setup.py +spec = importlib.util.spec_from_file_location( + "dvc.version", dvc / "version.py" +) +dvc_version = importlib.util.module_from_spec(spec) +spec.loader.exec_module(dvc_version) +version = dvc_version.__version__ + +(dvc / "version.py").write_text( + textwrap.dedent( + f"""\ + # AUTOGENERATED at build time by scripts/build.py + __version__ = "{version}" + """ + ) +) + +check_call( + ["python", "build.py"], cwd=pyinstaller, stderr=STDOUT, +) + +if args.pkg == "exe": + check_call( + ["python", "build.py"], cwd=innosetup, stderr=STDOUT, + ) +else: + check_call( + ["python", "build.py", args.pkg], cwd=fpm, stderr=STDOUT, + ) diff --git a/scripts/build_posix.sh b/scripts/build_posix.sh deleted file mode 100755 index 0c0da117d..000000000 --- a/scripts/build_posix.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/bash - -set -e -set -x - -FPM_FLAGS='--description "Data Version Control - datasets, models, and experiments versioning for ML or data science projects"' -FPM_FLAGS+=' -n dvc -s dir -f' -FPM_FLAGS+=' --url https://dvc.org' -FPM_FLAGS+=' --license "Apache License 2.0"' - -if [[ "$(uname)" == 'Linux' ]]; then - INSTALL_DIR=usr - FPM_FLAGS+=' --depends "git >= 1.7.0"' # needed for gitpython -else - INSTALL_DIR=usr/local - FPM_FLAGS+=' --osxpkg-identifier-prefix com.iterative' - FPM_FLAGS+=' --after-install scripts/fpm/after-install.sh' - FPM_FLAGS+=' --after-remove scripts/fpm/after-remove.sh' -fi - -BUILD_DIR=build -BIN_DIR=$BUILD_DIR/$INSTALL_DIR/bin -LIB_DIR=$BUILD_DIR/$INSTALL_DIR/lib - -FPM_PACKAGE_DIRS="usr" -ZSH_CMPLT_DIR=usr/share/zsh/site-functions/_dvc -if [[ "$(uname)" == 'Linux' ]]; then - BASH_CMPLT_DIR=etc/bash_completion.d - FPM_PACKAGE_DIRS="$FPM_PACKAGE_DIRS etc" -else - BASH_CMPLT_DIR=usr/local/etc/bash_completion.d -fi - -print_error() { - echo -e "\e[31m$1\e[0m" >&2 -} - -if [ ! -d "dvc" ]; then - print_error "Please run this script from repository root" - exit 1 -fi - -trap 'print_error "FAIL"; exit 1' ERR - -print_info() { - echo -e "\e[32m$1\e[0m" -} - -command_exists() { - command -v $1 >/dev/null 2>&1 -} - -fpm_build() { - print_info "Building $1..." - VERSION=$(python -c "import dvc; from dvc import __version__; print(str(__version__))") - eval "fpm -t $1 $FPM_FLAGS -v $VERSION -C $BUILD_DIR $FPM_PACKAGE_DIRS" -} - -cleanup() { - print_info "Cleaning up..." - rm -rf build -} - -install_dependencies() { - print_info "Installing fpm..." - if command_exists dnf; then - sudo dnf install ruby-devel gcc make rpm-build - elif command_exists yum; then - sudo yum install ruby-devel gcc make rpm-build - elif command_exists apt-get; then - sudo apt-get update -y - sudo apt-get install ruby-dev build-essential rpm python-pip python-dev - elif command_exists brew; then - brew install ruby - else - echo "Unable to install fpm dependencies" && exit 1 - fi - - gem install --no-document fpm - - # https://github.com/iterative/dvc/issues/2970 - gem uninstall -i /Users/travis/.rvm/gems/ruby-2.4.3@global rubygems-bundler - - print_info "Installing requirements..." - pip install .[all] - pip install -r scripts/build-requirements.txt -} - -build_dvc() { - print_info "Building dvc binary..." - pyinstaller \ - --additional-hooks-dir $(pwd)/scripts/hooks dvc/__main__.py \ - --name dvc \ - --distpath $LIB_DIR \ - --specpath $BUILD_DIR - - $LIB_DIR/dvc/dvc --help - - # NOTE: in osxpkg fpm replaces symlinks with actual file that it - # points to, so we need to use after-install hook. See FPM_FLAGS - # above. - if [[ "$(uname)" == 'Linux' ]]; then - mkdir -p $BIN_DIR - pushd $BIN_DIR - ln -s ../lib/dvc/dvc dvc - popd - $BIN_DIR/dvc --help - fi - - # NOTE: temporarily not adding scripts to mac package. See [1] - # [1] https://github.com/iterative/dvc/issues/2585 - if [[ "$(uname)" == 'Linux' ]]; then - mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR - $LIB_DIR/dvc/dvc completion -s bash > $BUILD_DIR/$BASH_CMPLT_DIR/dvc - - mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR - $LIB_DIR/dvc/dvc completion -s zsh > $BUILD_DIR/$ZSH_CMPLT_DIR/_dvc - fi -} - -build() { - cleanup - echo "PKG = \"$1\"" >dvc/utils/build.py - build_dvc - fpm_build $1 -} - -install_dependencies - -if [[ "$(uname)" == 'Linux' ]]; then - build rpm - build deb -else - build osxpkg -fi - -cleanup - -print_info "Successfully built dvc packages" diff --git a/scripts/build_windows.cmd b/scripts/build_windows.cmd deleted file mode 100755 index 8adecbd00..000000000 --- a/scripts/build_windows.cmd +++ /dev/null @@ -1,48 +0,0 @@ -@echo Off - -echo ====== Starting to build dvc installer for Windows... ====== - -if not exist dvc\NUL (echo Error: Please run this script from repository root && goto :error) - -rmdir /Q /S build -rmdir /Q /S dist -del /Q /S dvc.spec -del /Q /S "dvc-*.exe" - -where pip -if %errorlevel% neq 0 (echo Error: pip not found && goto :error) - -where choco -if %errorlevel% neq 0 (echo Error: choco not found && goto :error) - -choco install InnoSetup -call refreshenv -where iscc -if %errorlevel% neq 0 (echo Error: Couldn't find Inno Setup compiler. && goto :error) - -echo ====== Installing requirements... ====== -echo PKG = "exe" > dvc\utils\build.py -call pip install .[all] || goto :error -call pip install -r scripts\build-requirements.txt || goto :error -call dvc pull || goto :error - -echo ====== Building dvc binary... ====== -call pyinstaller --additional-hooks-dir scripts\hooks dvc/__main__.py --name dvc --specpath build - -echo ====== Testing dvc binary... ====== -call dist\dvc\dvc.exe version || goto :error - -echo ====== Copying additional files... ====== -copy scripts\innosetup\addSymLinkPermissions.ps1 dist\ || goto :error - -echo ====== Building dvc installer... ====== -set PYTHONPATH=%cd% -call python scripts\innosetup\config_gen.py || goto :error -call iscc scripts\innosetup\setup.iss || goto :error - -echo ====== DONE ====== -goto :EOF - -:error -echo ====== FAIL ====== -exit /b 1 diff --git a/scripts/fpm/.gitignore b/scripts/fpm/.gitignore new file mode 100644 index 000000000..80294d137 --- /dev/null +++ b/scripts/fpm/.gitignore @@ -0,0 +1,4 @@ +/build +/*.deb +/*.rpm +/*.pkg diff --git a/scripts/fpm/build.py b/scripts/fpm/build.py new file mode 100644 index 000000000..6933919ea --- /dev/null +++ b/scripts/fpm/build.py @@ -0,0 +1,96 @@ +import argparse +import os +import pathlib +import shutil +from subprocess import STDOUT, check_call, check_output + +path = pathlib.Path(__file__).parent.absolute() +dvc = path.parent.parent / "dvc" +pyinstaller = path.parent / "pyinstaller" + +build = path / "build" +install = build / "usr" + +parser = argparse.ArgumentParser() +parser.add_argument( + "pkg", choices=["deb", "rpm", "osxpkg"], help="package type" +) +args = parser.parse_args() + +flags = [ + "--description", + '"Data Version Control | Git for Data & Models"', + "-n", + "dvc", + "-s", + "dir", + "-f", + "--license", + '"Apache License 2.0"', +] + +if args.pkg == "osxpkg": + install /= "local" + bash_dir = install / "etc" / "bash_completion.d" + dirs = ["usr"] + flags.extend( + [ + "--osxpkg-identifier-prefix", + "com.iterative", + "--after-install", + path / "after-install.sh", + "--after-remove", + path / "after-remove.sh", + ] + ) +else: + bash_dir = build / "etc" / "bash_completion.d" + dirs = ["usr", "etc"] + flags.extend(["--depends", "git >= 1.7.0"]) # needed for gitpython + +try: + shutil.rmtree(build) +except FileNotFoundError: + pass + +lib = install / "lib" +lib.mkdir(parents=True) +shutil.copytree(pyinstaller / "dist" / "dvc", lib / "dvc") + +if args.pkg != "osxpkg": + # NOTE: in osxpkg fpm replaces symlinks with actual file that it + # points to, so we need to use after-install hook. + (install / "bin").mkdir() + os.symlink("../lib/dvc/dvc", install / "bin" / "dvc") + +bash_dir.mkdir(parents=True) +bash_completion = check_output( + [lib / "dvc" / "dvc", "completion", "-s", "bash"], text=True +) +(bash_dir / "dvc").write_text(bash_completion) + +zsh_dir = build / "usr" / "share" / "zsh" / "site-funtions" +zsh_dir.mkdir(parents=True) +zsh_completion = check_output( + [lib / "dvc" / "dvc", "completion", "-s", "zsh"], text=True +) +(zsh_dir / "_dvc").write_text(zsh_completion) + +version = check_output([lib / "dvc" / "dvc", "--version"], text=True).strip() + +check_call( + [ + "fpm", + "--verbose", + "-t", + args.pkg, + *flags, + "-v", + version, + "-C", + build, + *dirs, + ], + cwd=path, + stderr=STDOUT, +) diff --git a/scripts/innosetup/.gitignore b/scripts/innosetup/.gitignore index 9aa3f1657..2b0954e7d 100644 --- a/scripts/innosetup/.gitignore +++ b/scripts/innosetup/.gitignore @@ -1,4 +1,6 @@ -dvc.ico -dvc_left.bmp -dvc_up.bmp +/dvc.ico +/dvc_left.bmp +/dvc_up.bmp /config.ini +/build +/dvc*.exe diff --git a/scripts/innosetup/build.py b/scripts/innosetup/build.py new file mode 100644 index 000000000..58457da1f --- /dev/null +++ b/scripts/innosetup/build.py @@ -0,0 +1,42 @@ +import argparse +import configparser +import os +import pathlib +import shutil +from subprocess import STDOUT, check_call, check_output + +path = pathlib.Path(__file__).parent.absolute() +config = path / "config.ini" + +dvc = path.parent.parent / "dvc" +pyinstaller = path.parent / "pyinstaller" + +build = path / "build" +install = build / "usr" + +parser = argparse.ArgumentParser() +args = parser.parse_args() + +try: + shutil.rmtree(build) +except FileNotFoundError: + pass + +build.mkdir() +shutil.copytree(pyinstaller / "dist" / "dvc", build / "dvc") +shutil.copy(path / "addSymLinkPermissions.ps1", build) + +version = check_output( + [os.fspath(build / "dvc" / "dvc"), "--version"], text=True +).strip() + +cfg = configparser.ConfigParser() +cfg.add_section("Version") +cfg.set("Version", "version", version) + +with (path / "config.ini").open("w") as fobj: + cfg.write(fobj) + +check_call( + ["iscc", "setup.iss"], cwd=path, stderr=STDOUT, +) diff --git a/scripts/innosetup/config_gen.py b/scripts/innosetup/config_gen.py deleted file mode 100644 index 061b3cb2b..000000000 --- a/scripts/innosetup/config_gen.py +++ /dev/null @@ -1,11 +0,0 @@ -# This script generates config.ini for setup.iss script -import configparser - -from dvc import __version__ - -config = configparser.ConfigParser() -config.add_section("Version") -config.set("Version", "Version", __version__) - -with open("scripts/innosetup/config.ini", "w") as f: - config.write(f) diff --git a/scripts/innosetup/setup.iss b/scripts/innosetup/setup.iss index bf690f893..a89206063 100644 --- a/scripts/innosetup/setup.iss +++ b/scripts/innosetup/setup.iss @@ -1,8 +1,8 @@ #define MyAppName "DVC (Data Version Control)" -#define MyAppVersion ReadIni(".\scripts\innosetup\config.ini", "Version", "version", "unknown") +#define MyAppVersion ReadIni(SourcePath + "\config.ini", "Version", "version", "unknown") #define MyAppPublisher "Dmitry Petrov" -#define MyAppURL "https://dataversioncontrol.com/" -#define MyAppDir "..\..\dist\dvc" +#define MyAppURL "https://dvc.org" +#define MyAppDir SourcePath + "\build\dvc" [Setup] AppId={{8258CE8A-110E-4E0D-AE60-FEE00B15F041} @@ -19,7 +19,7 @@ LicenseFile=..\..\LICENSE OutputBaseFilename=dvc-{#MyAppVersion} Compression=lzma2/max SolidCompression=yes -OutputDir=..\..\ +OutputDir=. ChangesEnvironment=yes SetupIconFile=dvc.ico WizardSmallImageFile=dvc_up.bmp diff --git a/scripts/pyinstaller/.gitignore b/scripts/pyinstaller/.gitignore new file mode 100644 index 000000000..1f1748b55 --- /dev/null +++ b/scripts/pyinstaller/.gitignore @@ -0,0 +1,4 @@ +/build +/dist +/dvc.spec +/dvc diff --git a/scripts/pyinstaller/build.py b/scripts/pyinstaller/build.py new file mode 100644 index 000000000..c677c9fa1 --- /dev/null +++ b/scripts/pyinstaller/build.py @@ -0,0 +1,22 @@ +import os +import pathlib +from subprocess import STDOUT, check_call + +path = pathlib.Path(__file__).parent.absolute() +hooks = path / "hooks" +dvc = path.parent.parent / "dvc" +entry = dvc / "__main__.py" + +check_call( + [ + "pyinstaller", + "--additional-hooks-dir", + os.fspath(hooks), + "--name", + "dvc", + "-y", + os.fspath(entry), + ], + cwd=path, + stderr=STDOUT, +) diff --git a/scripts/hooks/hook-dvc.py b/scripts/pyinstaller/hooks/hook-dvc.py similarity index 100% rename from scripts/hooks/hook-dvc.py rename to scripts/pyinstaller/hooks/hook-dvc.py diff --git a/scripts/hooks/hook-dvc.system.py b/scripts/pyinstaller/hooks/hook-dvc.system.py similarity index 100% rename from scripts/hooks/hook-dvc.system.py rename to scripts/pyinstaller/hooks/hook-dvc.system.py diff --git a/scripts/hooks/hook-dvc.tree.gs.py b/scripts/pyinstaller/hooks/hook-dvc.tree.gs.py similarity index 100% rename from scripts/hooks/hook-dvc.tree.gs.py rename to scripts/pyinstaller/hooks/hook-dvc.tree.gs.py diff --git a/scripts/hooks/hook-dvc.utils.flatten.py b/scripts/pyinstaller/hooks/hook-dvc.utils.flatten.py similarity index 100% rename from scripts/hooks/hook-dvc.utils.flatten.py rename to scripts/pyinstaller/hooks/hook-dvc.utils.flatten.py diff --git a/scripts/hooks/hook-google_compute_engine.logger.py b/scripts/pyinstaller/hooks/hook-google_compute_engine.logger.py similarity index 100% rename from scripts/hooks/hook-google_compute_engine.logger.py rename to scripts/pyinstaller/hooks/hook-google_compute_engine.logger.py diff --git a/scripts/hooks/hook-googleapiclient.model.py b/scripts/pyinstaller/hooks/hook-googleapiclient.model.py similarity index 100% rename from scripts/hooks/hook-googleapiclient.model.py rename to scripts/pyinstaller/hooks/hook-googleapiclient.model.py diff --git a/scripts/hooks/hook-pydrive2.py b/scripts/pyinstaller/hooks/hook-pydrive2.py similarity index 100% rename from scripts/hooks/hook-pydrive2.py rename to scripts/pyinstaller/hooks/hook-pydrive2.py diff --git a/scripts/hooks/hook-win32file.py b/scripts/pyinstaller/hooks/hook-win32file.py similarity index 100% rename from scripts/hooks/hook-win32file.py rename to scripts/pyinstaller/hooks/hook-win32file.py