From 226b598dead014ff4193285a79c4a0c63877f2d9 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Mon, 11 Nov 2019 23:42:43 +0000 Subject: [PATCH 01/37] remove unneeded stage names --- .travis.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index b9bb1b8ad..388b07d7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ jobs: include: # patch formatting checks - name: "Check patch formatting" - stage: "check" + stage: check os: linux language: python python: 3.7 @@ -41,35 +41,28 @@ jobs: env: - PY2=1 - name: "3.7 on Windows" - stage: test os: windows # Windows 10.0.17134 N/A Build 17134 language: shell # 'language: python' is an error on Travis CI Windows - name: "2.7 on Linux" - stage: test language: python python: 2.7 env: - PY2=1 - name: "3.5 on Linux" - stage: test language: python python: 3.5 - name: "3.6 on Linux" - stage: test language: python python: 3.6 - name: "3.7 on Linux" - stage: test language: python python: 3.7 - name: "2.7 on Mac" - stage: test os: osx osx_image: xcode9.4 env: - PY2=1 - name: "3.6 on Mac" - stage: test os: osx osx_image: xcode9.4 # build jobs @@ -82,20 +75,17 @@ jobs: install: script: ./scripts/build_posix.sh - name: "Windows pkg" - stage: build os: windows language: shell install: script: ./scripts/build_windows.cmd - name: "Linux pkgs" - stage: build language: python python: 3.7 before_install: install: script: ./scripts/build_posix.sh - name: "Pypi pkgs" - stage: build language: python python: 3.7 script: ./scripts/build_package.sh From e18fe609cb95b5b989ee0bebeeec36908184013e Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Mon, 11 Nov 2019 23:43:05 +0000 Subject: [PATCH 02/37] remove deprecated sudo --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 388b07d7d..48f934516 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ cache: directories: - $HOME/.cache/pip - $HOME/.pyenv -sudo: true dist: xenial env: global: From 8c17ec14f977357d19badc9179b374a9e54f67be Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 12 Nov 2019 00:05:58 +0000 Subject: [PATCH 03/37] initial snap support --- .travis.yml | 25 +++++++++++++++++++++++++ snapcraft.yaml | 18 ++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 snapcraft.yaml diff --git a/.travis.yml b/.travis.yml index 48f934516..fc30fcd88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -109,6 +109,31 @@ jobs: on: tags: true repo: iterative/dvc + - name: Snapcraft snap + addons: + snaps: + - name: snapcraft + channel: stable + confinement: classic + - name: lxd + channel: stable + env: + - SNAPCRAFT_IMAGE_INFO: '{"build_url": "$TRAVIS_BUILD_URL"}' + script: + - sudo /snap/bin/lxd.migrate -yes + - sudo /snap/bin/lxd waitready + - sudo /snap/bin/lxd init --auto + - 'if [ -n "$SOURCE_TAG" ]; then sed -i "s/source-branch.*/source-tag: $SOURCE_TAG/" snap/snapcraft.yaml ; fi' + - sudo snapcraft --use-lxd + after_failure: + - sudo journalctl -u snapd + before_deploy: + - export SNAP_DEPLOY_TRACK="$SOURCE_TAG" + deploy: + - provider: snap + snap: dvc_*.snap + channel: ${SOURCE_TAG:-latest}/edge + skip_cleanup: true before_install: - bash ./scripts/ci/before_install.sh diff --git a/snapcraft.yaml b/snapcraft.yaml new file mode 100644 index 000000000..a3ccdc330 --- /dev/null +++ b/snapcraft.yaml @@ -0,0 +1,18 @@ +name: dvc +summary: Data Version Control +description: | + Git for Data & Models https://dvc.org +adopt-info: dvc +grade: stable +confinement: strict +base: core18 +license: Apache-2.0 +parts: + dvc: + plugin: python + python-version: python3 + source: . + parse-info: [setup.py] +apps: + dvc: + command: bin/dvc From 4c3bf028e9c3edcdcf43fe93fdcb3b01eabb1449 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 12 Nov 2019 01:24:28 +0000 Subject: [PATCH 04/37] fix snap build --- snapcraft.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/snapcraft.yaml b/snapcraft.yaml index a3ccdc330..1adeb74a2 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -4,15 +4,20 @@ description: | Git for Data & Models https://dvc.org adopt-info: dvc grade: stable -confinement: strict +confinement: classic # for access to home folder config base: core18 license: Apache-2.0 parts: dvc: plugin: python - python-version: python3 + python-packages: [wheel] source: . - parse-info: [setup.py] + build-packages: + - git + - libc-dev + override-pull: | + snapcraftctl pull + snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC && git describe --tags` apps: dvc: command: bin/dvc From 8212dc79920f84d25ff6ca7341a2909c67233a9e Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 12 Nov 2019 01:33:49 +0000 Subject: [PATCH 05/37] travis snap tidy --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index fc30fcd88..ed262f8ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,6 +110,7 @@ jobs: tags: true repo: iterative/dvc - name: Snapcraft snap + stage: check # just for now for travis addons: snaps: - name: snapcraft @@ -119,16 +120,15 @@ jobs: channel: stable env: - SNAPCRAFT_IMAGE_INFO: '{"build_url": "$TRAVIS_BUILD_URL"}' - script: + before_install: - sudo /snap/bin/lxd.migrate -yes - sudo /snap/bin/lxd waitready - sudo /snap/bin/lxd init --auto - - 'if [ -n "$SOURCE_TAG" ]; then sed -i "s/source-branch.*/source-tag: $SOURCE_TAG/" snap/snapcraft.yaml ; fi' + install: + script: - sudo snapcraft --use-lxd after_failure: - sudo journalctl -u snapd - before_deploy: - - export SNAP_DEPLOY_TRACK="$SOURCE_TAG" deploy: - provider: snap snap: dvc_*.snap From d2a6b6571194f43eeaae529abe975348e654e388 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 12 Nov 2019 02:45:18 +0000 Subject: [PATCH 06/37] fix permissions --- snapcraft.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/snapcraft.yaml b/snapcraft.yaml index 1adeb74a2..2f1af1a46 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -4,7 +4,7 @@ description: | Git for Data & Models https://dvc.org adopt-info: dvc grade: stable -confinement: classic # for access to home folder config +confinement: strict base: core18 license: Apache-2.0 parts: @@ -21,3 +21,4 @@ parts: apps: dvc: command: bin/dvc + plugs: [home] # for config From c2c98b007ae8608a06d23cc17f9f655597c4fb72 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 12 Nov 2019 13:47:52 +0000 Subject: [PATCH 07/37] minor tidy --- .travis.yml | 5 ++++- snapcraft.yaml | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ed262f8ee..00c8ef6db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,7 +110,7 @@ jobs: tags: true repo: iterative/dvc - name: Snapcraft snap - stage: check # just for now for travis + stage: check # TODO: remove before merge addons: snaps: - name: snapcraft @@ -134,6 +134,9 @@ jobs: snap: dvc_*.snap channel: ${SOURCE_TAG:-latest}/edge skip_cleanup: true + on: + # tags: true # TODO: uncomment before merge + repo: iterative/dvc before_install: - bash ./scripts/ci/before_install.sh diff --git a/snapcraft.yaml b/snapcraft.yaml index 2f1af1a46..8548c98d2 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -1,7 +1,6 @@ name: dvc summary: Data Version Control -description: | - Git for Data & Models https://dvc.org +description: Git for Data & Models https://dvc.org adopt-info: dvc grade: stable confinement: strict From 424bdbe2867937640f0bf2ac9ae26daee69ee818 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 12 Nov 2019 15:18:30 +0000 Subject: [PATCH 08/37] stable and edge releases --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 00c8ef6db..f57910e90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -132,10 +132,9 @@ jobs: deploy: - provider: snap snap: dvc_*.snap - channel: ${SOURCE_TAG:-latest}/edge + channel: latest/${SOURCE_TAG:+stable}${SOURCE_TAG:-edge} skip_cleanup: true on: - # tags: true # TODO: uncomment before merge repo: iterative/dvc before_install: From 8f06e76aac47690a796345cc73c9b455b0998245 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 12 Nov 2019 17:25:34 +0000 Subject: [PATCH 09/37] document and add badge --- README.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.rst b/README.rst index 2ba9c2c25..5ae8697ba 100644 --- a/README.rst +++ b/README.rst @@ -30,6 +30,10 @@ :target: https://anaconda.org/conda-forge/dvc :alt: Conda-forge +.. image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft + :target: https://snapcraft.io/dvc + :alt: Snapcraft + | **Data Version Control** or **DVC** is an **open-source** tool for data science and machine @@ -146,6 +150,16 @@ Conda (Anaconda) Currently, it supports only python version 2.7, 3.6 and 3.7. +Snap (Snapcraft) +---------------- + +.. code-block:: bash + + snap install dvc + +This corresponds to the latest tagged release. +Add ``--edge`` for the latest ``master`` version. + Package ------- From 73f9f3c9d5a00943b0080134baf167b0fcaa3ea2 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 12 Nov 2019 20:12:19 +0000 Subject: [PATCH 10/37] fix connectivity and permissions => back to --classic --- snapcraft.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/snapcraft.yaml b/snapcraft.yaml index 8548c98d2..b2fad552f 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -3,7 +3,7 @@ summary: Data Version Control description: Git for Data & Models https://dvc.org adopt-info: dvc grade: stable -confinement: strict +confinement: classic # for same reasons as snap `git-ubuntu` base: core18 license: Apache-2.0 parts: @@ -14,10 +14,16 @@ parts: build-packages: - git - libc-dev + stage-packages: + - git + - libc-dev override-pull: | snapcraftctl pull snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC && git describe --tags` apps: dvc: command: bin/dvc - plugs: [home] # for config + environment: + GIT_PYTHON_GIT_EXECUTABLE: dvc.git + git: + command: /usr/bin/git From 31b3bcc0e27a3d6c0f66747c9517b0cbd6528a50 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 12 Nov 2019 22:43:57 +0000 Subject: [PATCH 11/37] PyPI and GH on tags, Snap on master and tags --- .travis.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index f57910e90..c144a1956 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ stages: - test - build - name: deploy - if: tag is present + if: branch = master AND repo = iterative/dvc # Travis doesn't support testing python on Mac yet, so we need # to workaround it by installing it directly with brew. # @@ -90,6 +90,7 @@ jobs: script: ./scripts/build_package.sh # deploy jobs - name: "Pypi pkgs" + if: tag is present stage: deploy language: python python: 3.7 @@ -106,9 +107,6 @@ jobs: - dist/dvc*.whl - dist/dvc*.tar.gz skip_cleanup: true - on: - tags: true - repo: iterative/dvc - name: Snapcraft snap stage: check # TODO: remove before merge addons: @@ -134,8 +132,6 @@ jobs: snap: dvc_*.snap channel: latest/${SOURCE_TAG:+stable}${SOURCE_TAG:-edge} skip_cleanup: true - on: - repo: iterative/dvc before_install: - bash ./scripts/ci/before_install.sh From 9137ad5f18811f56721599688de6f07f63d13f18 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 12 Nov 2019 22:48:14 +0000 Subject: [PATCH 12/37] add minor comment abous stage tags Fixes https://github.com/iterative/dvc/pull/2778#discussion_r345424578 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index c144a1956..d2becbcf8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,8 @@ stages: # Note that we specify language as ruby-2.4.3 to help us satisfy # fpm dependencies, as travis runs el capitan with an old ruby # version, that causes fpm to crash. +# +# Each job will use the previous job's `stage` tag if not specified. jobs: include: # patch formatting checks From c23b8007c4956be35373402148b8bde573b92865 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 12 Nov 2019 23:01:32 +0000 Subject: [PATCH 13/37] potential deploy catch --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d2becbcf8..79f757698 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ stages: - test - build - name: deploy - if: branch = master AND repo = iterative/dvc + if: (branch = master OR tag is present) AND repo = iterative/dvc # Travis doesn't support testing python on Mac yet, so we need # to workaround it by installing it directly with brew. # From f944ce2ecd64703c240250994a2685cc9af71d55 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 12 Nov 2019 23:15:16 +0000 Subject: [PATCH 14/37] documnet --classic --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5ae8697ba..0bdfe3daa 100644 --- a/README.rst +++ b/README.rst @@ -155,7 +155,7 @@ Snap (Snapcraft) .. code-block:: bash - snap install dvc + snap install dvc --classic This corresponds to the latest tagged release. Add ``--edge`` for the latest ``master`` version. From 47e1b49e2aa961786d101f11abb0b96ace0ddaba Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 16 Nov 2019 16:42:25 +0000 Subject: [PATCH 15/37] PKG = "snap" Fixes https://github.com/iterative/dvc/pull/2778#issuecomment-554651335 --- scripts/build_package.sh | 2 +- snapcraft.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/build_package.sh b/scripts/build_package.sh index d64f3e6e4..1b9536697 100755 --- a/scripts/build_package.sh +++ b/scripts/build_package.sh @@ -8,7 +8,7 @@ if [ ! -d "dvc" ]; then exit 1 fi -echo "PKG = \"pip\"" > dvc/utils/build.py +echo 'PKG = "pip"' > dvc/utils/build.py python setup.py sdist python setup.py bdist_wheel --universal diff --git a/snapcraft.yaml b/snapcraft.yaml index b2fad552f..9fc6bbc10 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -19,6 +19,7 @@ parts: - libc-dev override-pull: | snapcraftctl pull + echo 'PKG = "snap"' > $SNAPCRAFT_PART_SRC/dvc/utils/build.py snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC && git describe --tags` apps: dvc: From 18eeb2cd84ed2645d4f91580894234bd98223fa7 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 21 Nov 2019 22:18:26 +0000 Subject: [PATCH 16/37] fix channel name --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 79f757698..8635a9a38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -132,7 +132,7 @@ jobs: deploy: - provider: snap snap: dvc_*.snap - channel: latest/${SOURCE_TAG:+stable}${SOURCE_TAG:-edge} + channel: ${SOURCE_TAG:+stable}${SOURCE_TAG:-edge} skip_cleanup: true before_install: From d82bb976e6a233a5c136256e2a7c38b4f653e50e Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 23 Nov 2019 23:24:01 +0000 Subject: [PATCH 17/37] full installation --- snapcraft.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/snapcraft.yaml b/snapcraft.yaml index 9fc6bbc10..41e6a573c 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -14,6 +14,7 @@ parts: build-packages: - git - libc-dev + - libkrb5-dev stage-packages: - git - libc-dev @@ -21,6 +22,7 @@ parts: snapcraftctl pull echo 'PKG = "snap"' > $SNAPCRAFT_PART_SRC/dvc/utils/build.py snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC && git describe --tags` + sed -ri 's/(=install_requires)/\1+all_remotes+ssh_gssapi+hdfs/' $SNAPCRAFT_PART_SRC/setup.py apps: dvc: command: bin/dvc From 8e58849f0be3f53cce86e4a85f272e96823984c0 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 23 Nov 2019 23:25:42 +0000 Subject: [PATCH 18/37] suppress update check, add comments --- snapcraft.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/snapcraft.yaml b/snapcraft.yaml index 41e6a573c..fcf77386e 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -14,19 +14,20 @@ parts: build-packages: - git - libc-dev - - libkrb5-dev + - libkrb5-dev # gssapi build stage-packages: - git - - libc-dev override-pull: | snapcraftctl pull echo 'PKG = "snap"' > $SNAPCRAFT_PART_SRC/dvc/utils/build.py snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC && git describe --tags` + # install all optional extras sed -ri 's/(=install_requires)/\1+all_remotes+ssh_gssapi+hdfs/' $SNAPCRAFT_PART_SRC/setup.py apps: dvc: command: bin/dvc environment: GIT_PYTHON_GIT_EXECUTABLE: dvc.git + CI: snap # suppress update checks git: command: /usr/bin/git From 5f13e2215da39ea53845fd074710a89aa38897c6 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 23 Nov 2019 23:53:52 +0000 Subject: [PATCH 19/37] bash completion --- snapcraft.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/snapcraft.yaml b/snapcraft.yaml index fcf77386e..c1a8835d0 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -23,9 +23,13 @@ parts: snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC && git describe --tags` # install all optional extras sed -ri 's/(=install_requires)/\1+all_remotes+ssh_gssapi+hdfs/' $SNAPCRAFT_PART_SRC/setup.py + override-build: | + snapcraftctl build + cp $SNAPCRAFT_PART_BUILD/scripts/completion/dvc.bash $SNAPCRAFT_PART_INSTALL/completion.sh apps: dvc: command: bin/dvc + completer: completion.sh environment: GIT_PYTHON_GIT_EXECUTABLE: dvc.git CI: snap # suppress update checks From ba12e1c4571a113c44b7756b67c453655ea80595 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 24 Nov 2019 00:20:37 +0000 Subject: [PATCH 20/37] test eveything apart from gssapi --- snapcraft.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/snapcraft.yaml b/snapcraft.yaml index c1a8835d0..49064aa17 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -14,7 +14,6 @@ parts: build-packages: - git - libc-dev - - libkrb5-dev # gssapi build stage-packages: - git override-pull: | @@ -22,7 +21,7 @@ parts: echo 'PKG = "snap"' > $SNAPCRAFT_PART_SRC/dvc/utils/build.py snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC && git describe --tags` # install all optional extras - sed -ri 's/(=install_requires)/\1+all_remotes+ssh_gssapi+hdfs/' $SNAPCRAFT_PART_SRC/setup.py + sed -ri 's/(=install_requires)/\1+all_remotes+hdfs/' $SNAPCRAFT_PART_SRC/setup.py override-build: | snapcraftctl build cp $SNAPCRAFT_PART_BUILD/scripts/completion/dvc.bash $SNAPCRAFT_PART_INSTALL/completion.sh From f616265c950751186427d7f148f24675748573f8 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 29 Nov 2019 17:38:44 +0000 Subject: [PATCH 21/37] create dedicated SNAP env var only for update skipping for now, instead of piggybacking off $CI --- dvc/updater.py | 2 +- snapcraft.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dvc/updater.py b/dvc/updater.py index 279d5d23a..6811f9983 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -48,7 +48,7 @@ class Updater(object): # pragma: no cover logger.debug(msg.format(self.lock.lockfile, action)) def check(self): - if os.getenv("CI") or env2bool("DVC_TEST"): + if os.getenv("CI") or env2bool("DVC_TEST") or env2bool("SNAP"): return self._with_lock(self._check, "checking") diff --git a/snapcraft.yaml b/snapcraft.yaml index 49064aa17..b77a7c944 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -31,6 +31,6 @@ apps: completer: completion.sh environment: GIT_PYTHON_GIT_EXECUTABLE: dvc.git - CI: snap # suppress update checks + SNAP: 1 # suppress update checks git: command: /usr/bin/git From c0058a6f660c12d0728a77ad76d1a2b6262c49ba Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 29 Nov 2019 18:46:57 +0000 Subject: [PATCH 22/37] actually use PKG to determine package type --- dvc/updater.py | 2 +- snapcraft.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/dvc/updater.py b/dvc/updater.py index 6811f9983..2bb1d604d 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -48,7 +48,7 @@ class Updater(object): # pragma: no cover logger.debug(msg.format(self.lock.lockfile, action)) def check(self): - if os.getenv("CI") or env2bool("DVC_TEST") or env2bool("SNAP"): + if os.getenv("CI") or env2bool("DVC_TEST") or PKG == "snap": return self._with_lock(self._check, "checking") diff --git a/snapcraft.yaml b/snapcraft.yaml index b77a7c944..d90856c1e 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -31,6 +31,5 @@ apps: completer: completion.sh environment: GIT_PYTHON_GIT_EXECUTABLE: dvc.git - SNAP: 1 # suppress update checks git: command: /usr/bin/git From 577ad548b019a182c2c34452dbc104a8508b404e Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 29 Nov 2019 19:41:41 +0000 Subject: [PATCH 23/37] move snapcraft to subdir --- snapcraft.yaml => snap/snapcraft.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename snapcraft.yaml => snap/snapcraft.yaml (100%) diff --git a/snapcraft.yaml b/snap/snapcraft.yaml similarity index 100% rename from snapcraft.yaml rename to snap/snapcraft.yaml From 537d51b5351926732566256369189ed7a63e4208 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 29 Nov 2019 19:43:12 +0000 Subject: [PATCH 24/37] use strict-confinement-compatible git --- snap/snapcraft.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index d90856c1e..13a4bde9c 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -30,6 +30,6 @@ apps: command: bin/dvc completer: completion.sh environment: - GIT_PYTHON_GIT_EXECUTABLE: dvc.git + GIT_PYTHON_GIT_EXECUTABLE: $SNAP/usr/bin/git git: - command: /usr/bin/git + command: usr/bin/git From b713a62be929749c855bc9e81bbd3345a2ecd008 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 29 Nov 2019 21:17:03 +0000 Subject: [PATCH 25/37] potential git discovery for removable media fix --- snap/snapcraft.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 13a4bde9c..2b1b781d2 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -33,3 +33,5 @@ apps: GIT_PYTHON_GIT_EXECUTABLE: $SNAP/usr/bin/git git: command: usr/bin/git + environment: + GIT_DISCOVERY_ACROSS_FILESYSTEM: 1 From d0cb359060211ca4af31716a2fb785040b0745ba Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 29 Nov 2019 21:19:21 +0000 Subject: [PATCH 26/37] try strict confinement --- README.rst | 2 +- snap/snapcraft.yaml | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 0bdfe3daa..5ae8697ba 100644 --- a/README.rst +++ b/README.rst @@ -155,7 +155,7 @@ Snap (Snapcraft) .. code-block:: bash - snap install dvc --classic + snap install dvc This corresponds to the latest tagged release. Add ``--edge`` for the latest ``master`` version. diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 2b1b781d2..ea27c06ff 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -3,9 +3,18 @@ summary: Data Version Control description: Git for Data & Models https://dvc.org adopt-info: dvc grade: stable -confinement: classic # for same reasons as snap `git-ubuntu` +confinement: strict base: core18 license: Apache-2.0 +plugs: + config-dvc: + interface: system-files + read: + - /etc/hosts + - /etc/host.conf + - /etc/resolv.conf + write: + - /etc/dvc.conf parts: dvc: plugin: python @@ -31,7 +40,18 @@ apps: completer: completion.sh environment: GIT_PYTHON_GIT_EXECUTABLE: $SNAP/usr/bin/git + plugs: + - config-dvc + - home + - mount-observe + - network-control + - removable-media git: command: usr/bin/git environment: GIT_DISCOVERY_ACROSS_FILESYSTEM: 1 + plugs: + - home + - mount-observe + - network-control + - removable-media From b36a49120b6a1de96806b787f8fda57a9e36aaa2 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 29 Nov 2019 21:49:53 +0000 Subject: [PATCH 27/37] attempt to fix dvc config --system --- snap/snapcraft.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index ea27c06ff..359147ff3 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -10,11 +10,13 @@ plugs: config-dvc: interface: system-files read: + - /etc/dvc/dvc.conf - /etc/hosts - /etc/host.conf - /etc/resolv.conf write: - - /etc/dvc.conf + - /etc/dvc + - /etc/dvc/dvc.conf parts: dvc: plugin: python From ebe80cd60a32fdada3c82d963d4f330d22fbc72f Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 30 Nov 2019 00:28:56 +0000 Subject: [PATCH 28/37] attempt to use layout --- snap/snapcraft.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 359147ff3..1aa359ce1 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -10,13 +10,12 @@ plugs: config-dvc: interface: system-files read: - - /etc/dvc/dvc.conf - - /etc/hosts - /etc/host.conf - /etc/resolv.conf - write: - - /etc/dvc - - /etc/dvc/dvc.conf + - /etc/hosts +layout: + /etc/dvc: + bind: $SNAP_DATA/etc/dvc parts: dvc: plugin: python @@ -42,6 +41,7 @@ apps: completer: completion.sh environment: GIT_PYTHON_GIT_EXECUTABLE: $SNAP/usr/bin/git + XDG_CONFIG_DIRS: /etc plugs: - config-dvc - home From f60d795709ced2b779dd59f602d3791c04e78787 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 30 Nov 2019 00:35:41 +0000 Subject: [PATCH 29/37] document connecting plugs before approval --- README.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.rst b/README.rst index 5ae8697ba..a16525b80 100644 --- a/README.rst +++ b/README.rst @@ -159,6 +159,14 @@ Snap (Snapcraft) This corresponds to the latest tagged release. Add ``--edge`` for the latest ``master`` version. +Additionally, grant required permissions: + +.. code-block:: bash + + snap connect dvc:config-dvc + snap connect dvc:mount-observe + snap connect dvc:network-control + snap connect dvc:removable-media Package ------- From 8fa7d7cfd754d331a973c7f2840b4952fae5719a Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 30 Nov 2019 01:01:20 +0000 Subject: [PATCH 30/37] fix git --- snap/snapcraft.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 1aa359ce1..377d47c23 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -15,7 +15,13 @@ plugs: - /etc/hosts layout: /etc/dvc: - bind: $SNAP_DATA/etc/dvc + bind: $SNAP_DATA/etc/dvc + /usr/share/git-core: + bind: $SNAP/usr/share/git-core + /usr/share/gitweb: + bind: $SNAP/usr/share/gitweb + /usr/lib/git-core: + bind: $SNAP/usr/lib/git-core parts: dvc: plugin: python From f6e92184e389f95ed39b109f40fbe41de1f3cf9f Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 30 Nov 2019 01:19:13 +0000 Subject: [PATCH 31/37] ready for merge --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8635a9a38..2819b726a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,7 +110,6 @@ jobs: - dist/dvc*.tar.gz skip_cleanup: true - name: Snapcraft snap - stage: check # TODO: remove before merge addons: snaps: - name: snapcraft From ae274cbab7105bbc3b6be2f5b741cbe833f9cec4 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 30 Nov 2019 01:46:48 +0000 Subject: [PATCH 32/37] try devmode --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 377d47c23..9cb2cb89e 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -3,7 +3,7 @@ summary: Data Version Control description: Git for Data & Models https://dvc.org adopt-info: dvc grade: stable -confinement: strict +confinement: devmode base: core18 license: Apache-2.0 plugs: From 7fcb4bff9cefbe1417e7fa0f621b4cabaa7dd4ab Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 30 Nov 2019 17:22:12 +0000 Subject: [PATCH 33/37] back to --classic --- .travis.yml | 48 ++++++++++++++++++++++++--------------------- README.rst | 10 +++++++++- snap/snapcraft.yaml | 2 +- 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2819b726a..fd959effa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -90,25 +90,6 @@ jobs: language: python python: 3.7 script: ./scripts/build_package.sh - # deploy jobs - - name: "Pypi pkgs" - if: tag is present - stage: deploy - language: python - python: 3.7 - script: ./scripts/build_package.sh - deploy: - - provider: pypi - user: - secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4=" - password: - secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc=" - skip_existing: true - file_glob: true - file: - - dist/dvc*.whl - - dist/dvc*.tar.gz - skip_cleanup: true - name: Snapcraft snap addons: snaps: @@ -128,10 +109,32 @@ jobs: - sudo snapcraft --use-lxd after_failure: - sudo journalctl -u snapd + #uncomment after classic approval + #<> + #stage: deploy + #deploy: + #- provider: snap + # snap: dvc_*.snap + # channel: ${SOURCE_TAG:+stable}${SOURCE_TAG:-edge} + # skip_cleanup: true + # deploy jobs + - name: "Pypi pkgs" + if: tag is present + stage: deploy + language: python + python: 3.7 + script: ./scripts/build_package.sh deploy: - - provider: snap - snap: dvc_*.snap - channel: ${SOURCE_TAG:+stable}${SOURCE_TAG:-edge} + - provider: pypi + user: + secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4=" + password: + secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc=" + skip_existing: true + file_glob: true + file: + - dist/dvc*.whl + - dist/dvc*.tar.gz skip_cleanup: true before_install: @@ -153,6 +156,7 @@ deploy: - dvc*.deb - dvc*.pkg - dvc*.exe + - dvc*.snap skip_cleanup: true on: tags: true diff --git a/README.rst b/README.rst index a16525b80..dc89ac5d0 100644 --- a/README.rst +++ b/README.rst @@ -153,11 +153,17 @@ Currently, it supports only python version 2.7, 3.6 and 3.7. Snap (Snapcraft) ---------------- +Download the latest ``dvc_*.snap`` from the +GitHub `releases page `_. + .. code-block:: bash - snap install dvc + snap install dvc_*.snap --dangerous --classic This corresponds to the latest tagged release. + +
+ Package ------- diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 9cb2cb89e..4812850d5 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -3,7 +3,7 @@ summary: Data Version Control description: Git for Data & Models https://dvc.org adopt-info: dvc grade: stable -confinement: devmode +confinement: classic base: core18 license: Apache-2.0 plugs: From a0ac129d5fbba4b491033d4a397a2ad02dccccfb Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 30 Nov 2019 17:23:57 +0000 Subject: [PATCH 34/37] cleanup things to revert after classic https://forum.snapcraft.io/t/classic-confinement-request-for-dvc/14124 --- .travis.yml | 8 -------- README.rst | 16 ---------------- 2 files changed, 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index fd959effa..762ee0552 100644 --- a/.travis.yml +++ b/.travis.yml @@ -109,14 +109,6 @@ jobs: - sudo snapcraft --use-lxd after_failure: - sudo journalctl -u snapd - #uncomment after classic approval - #<> - #stage: deploy - #deploy: - #- provider: snap - # snap: dvc_*.snap - # channel: ${SOURCE_TAG:+stable}${SOURCE_TAG:-edge} - # skip_cleanup: true # deploy jobs - name: "Pypi pkgs" if: tag is present diff --git a/README.rst b/README.rst index dc89ac5d0..6959db822 100644 --- a/README.rst +++ b/README.rst @@ -160,22 +160,6 @@ GitHub `releases page `_. snap install dvc_*.snap --dangerous --classic -This corresponds to the latest tagged release. - -
- Package ------- From c94acdf2a9eb7ec2a899b5ad7ec0ea1eb591594f Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 30 Nov 2019 17:26:48 +0000 Subject: [PATCH 35/37] use --devmode as it looks less scary --- README.rst | 2 +- snap/snapcraft.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 6959db822..f158712da 100644 --- a/README.rst +++ b/README.rst @@ -158,7 +158,7 @@ GitHub `releases page `_. .. code-block:: bash - snap install dvc_*.snap --dangerous --classic + snap install --devmode dvc_*.snap Package ------- diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 4812850d5..9cb2cb89e 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -3,7 +3,7 @@ summary: Data Version Control description: Git for Data & Models https://dvc.org adopt-info: dvc grade: stable -confinement: classic +confinement: devmode base: core18 license: Apache-2.0 plugs: From 0acb1e8e3e1a13c6cc5baad3d2eca52c8f38a5ac Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Mon, 2 Dec 2019 16:39:13 +0000 Subject: [PATCH 36/37] revert devmode - didn't work - almost completely reverts c94acdf2a9eb7ec2a899b5ad7ec0ea1eb591594f --- README.rst | 2 +- snap/snapcraft.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index f158712da..6959db822 100644 --- a/README.rst +++ b/README.rst @@ -158,7 +158,7 @@ GitHub `releases page `_. .. code-block:: bash - snap install --devmode dvc_*.snap + snap install dvc_*.snap --dangerous --classic Package ------- diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 9cb2cb89e..1f8a0b39f 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -3,7 +3,7 @@ summary: Data Version Control description: Git for Data & Models https://dvc.org adopt-info: dvc grade: stable -confinement: devmode +confinement: classic # for same reasons as snap `git-ubuntu` base: core18 license: Apache-2.0 plugs: From 52c48b56a2245dd194edde3f5348aa5f22adfcc9 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Mon, 2 Dec 2019 16:44:12 +0000 Subject: [PATCH 37/37] add snap approval request to readme --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 6959db822..407bc0d4e 100644 --- a/README.rst +++ b/README.rst @@ -160,6 +160,11 @@ GitHub `releases page `_. snap install dvc_*.snap --dangerous --classic +Once ``dvc`` is released on the snap store +(`pending request `_) +there will be no need to download ``dvc_*.snap`` or use ``--dangerous`` +(``snap install dvc --classic`` would suffice). + Package -------