From 1c89bb50c5babaa30b2048de2d49f3eb60af7923 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 7 Jan 2018 22:32:46 +0200 Subject: [PATCH 1/4] test: add initial tests for LOCAL and AWS data clouds Signed-off-by: Ruslan Kuprieiev --- dvc/data_cloud.py | 44 ++++++++-------- tests/test_data_cloud.py | 107 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 22 deletions(-) create mode 100644 tests/test_data_cloud.py diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 7ae5244f9..42b10bf65 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -192,38 +192,38 @@ class DataCloudLOCAL(DataCloudBase): """ Driver for local storage. """ - def push(self, item): - Logger.debug('sync to cloud ' + item.cache.dvc + " " + self.storage_path) - copyfile(item.cache.dvc, self.storage_path) - return item + def push(self, path): + Logger.debug('sync to cloud ' + path + " " + self.storage_path) + copyfile(path, self.storage_path) + return path - def _import(self, bucket, i, item): - tmp_file = self.tmp_file(item.data.dvc) + def _import(self, bucket, i, path): + inp = os.path.join(self.storage_path, i) + tmp_file = self.tmp_file(path) try: - copyfile(i, tmp_file) + copyfile(inp, tmp_file) except Exception as exc: Logger.error('Failed to copy "{}": {}'.format(i, exc)) return None - os.rename(tmp_file, item.data.dvc) - item.move_data_to_cache() + os.rename(tmp_file, path) - return item + return path - def pull(self, item): - Logger.debug('sync from cloud ' + self.storage_path + " " + item.cache.dvc) - return self._import(None, self.storage_path, item) + def pull(self, path): + Logger.debug('sync from cloud ' + path) + return self._import(None, path, path) - def remove(self, item): - Logger.debug('rm from cloud ' + item.cache.dvc) - os.remove(item.cache.dvc) + def remove(self, path): + Logger.debug('rm from cloud ' + path) + os.remove(path) - def import_data(self, path, item): - Logger.debug('import from cloud ' + path + " " + item.data.dvc) - return self._import(None, path, item) + def import_data(self, path, out): + Logger.debug('import from cloud ' + path + " " + out) + return self._import(None, path, out) - def _status(self, data_item): - local = data_item.cache.relative + def _status(self, path): + local = path remote = '{}/{}'.format(self.storage_path, os.path.basename(local)) remote_exists = os.path.exists(remote) @@ -814,7 +814,7 @@ class DataCloud(object): (F, issues) if bad """ key = 'Cloud' - if key.lower() not in self._config['Global'].keys() or len(self._config['Global'][key]) < 1: + if key.lower() not in [k.lower() for k in self._config['Global'].keys()] or len(self._config['Global'][key]) < 1: raise ConfigError('Please set %s in section Global in config file' % key) # now that a cloud is chosen, can check StoragePath diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py new file mode 100644 index 000000000..dbc292031 --- /dev/null +++ b/tests/test_data_cloud.py @@ -0,0 +1,107 @@ +import os + +from dvc.config import ConfigError +from dvc.data_cloud import DataCloud, DataCloudAWS, DataCloudLOCAL +from dvc.data_cloud import STATUS_UNKNOWN, STATUS_OK, STATUS_MODIFIED, STATUS_NEW, STATUS_DELETED +from dvc.cloud.instance_manager import CloudSettings + +from tests.basic_env import TestDvc + + +class TestDataCloud(TestDvc): + def test(self): + for k,v in DataCloud.CLOUD_MAP.items(): + config = {'Global': {'Cloud': k}, + k: {'StoragePath': 'a/b', + 'ProjectName': 'name'}} + cloud = DataCloud(config) + self.assertIsInstance(cloud._cloud, v) + + with self.assertRaises(ConfigError) as cx: + config = {'Global': {'Cloud': 'not_supported_type'}} + DataCloud(config) + + +class TestDataCloudBase(TestDvc): + def _setup_cloud(self): + self.cloud = None + + def _test_cloud(self): + self._setup_cloud() + + # Check status + status = self.cloud.status(self.FOO) + self.assertEqual(status, STATUS_NEW) + + # Push and check status + self.cloud.push(self.FOO) + self.assertTrue(os.path.exists(self.FOO)) + self.assertTrue(os.path.isfile(self.FOO)) + + status = self.cloud.status(self.FOO) + self.assertEqual(status, STATUS_OK) + + # Modify and check status + with open(self.FOO, 'a') as fd: + fd.write('addon') + status = self.cloud.status(self.FOO) + self.assertEqual(status, STATUS_MODIFIED) + + # Remove and check status + os.unlink(self.FOO) + status = self.cloud.status(self.FOO) + self.assertEqual(status, STATUS_DELETED) + + # Pull and check status + self.cloud.pull(self.FOO) + self.assertTrue(os.path.exists(self.FOO)) + self.assertTrue(os.path.isfile(self.FOO)) + with open(self.FOO, 'r') as fd: + self.assertEqual(fd.read(), self.FOO_CONTENTS) + + status = self.cloud.status(self.FOO) + self.assertEqual(status, STATUS_OK) + + +class TestDataCloudAWS(TestDataCloudBase): + TEST_REPO_S3 = 'dvc-test/myrepo' + TEST_REPO_REGION = 'us-east-2' + + def _should_test(self): + if os.getenv("TRAVIS_PULL_REQUEST") == "false" and \ + os.getenv("TRAVIS_SECURE_ENV_VARS") == "true": + return True + return False + + def _setup_cloud(self): + if not self._should_test(): + return + + # Cleanup + os.system('aws s3 rm --recursive "s3://{}/"'.format(self.TEST_REPO_S3)) + + # Setup cloud + config = {'StoragePath': self.TEST_REPO_S3, + 'Region': self.TEST_REPO_REGION} + cloud_settings = CloudSettings(None, config) + self.cloud = DataCloudAWS(cloud_settings) + + def test(self): + if os.getenv("TRAVIS_PULL_REQUEST") == "false" and \ + os.getenv("TRAVIS_SECURE_ENV_VARS") == "true": + self._test_cloud() + + +class TestDataCloudLOCAL(TestDataCloudBase): + def _setup_cloud(self): + self.dname = 'cloud' + os.mkdir(self.dname) + config = {'StoragePath': self.dname} + cloud_settings = CloudSettings(None, config) + self.cloud = DataCloudLOCAL(cloud_settings) + + def test(self): + self._test_cloud() + self.assertTrue(os.path.isdir(self.dname)) + path = os.path.join(self.dname, self.FOO) + self.assertTrue(os.path.isfile(path)) From b32d307452b88158c0acd0d14337a0a930fe9d14 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 7 Jan 2018 19:00:03 +0200 Subject: [PATCH 2/4] travis: use cc-reporter for codeclimate Signed-off-by: Ruslan Kuprieiev --- .travis.yml | 7 +++++-- ci/after_script.sh | 9 +++++++++ ci/before_script.sh | 11 +++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 ci/after_script.sh create mode 100644 ci/before_script.sh diff --git a/.travis.yml b/.travis.yml index 5ac87f4ca..68b5d4899 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,14 +43,17 @@ install: - mkdir ~/.aws - printf "[default]\naws_access_key_id = $AWS_ACCESS_KEY_ID\naws_secret_access_key = $AWS_SECRET_ACCESS_KEY\n" > ~/.aws/credentials - printf "[default]\n" > ~/.aws/config +before_script: + - bash ./ci/before_script.sh script: - ./unittests.sh - ./functests.sh - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./build_macos.sh; else ./build_linux.sh; fi - ./build_package.sh -after_sucess: - - CODECLIMATE_REPO_TOKEN=7350efa14a67c3911af96dadaabe811b43eead7b466309b47687499bf5d72287 codeclimate-test-reporter +after_script: + - bash ./ci/after_scripts.sh deploy: + condition: "[[ $(python -c 'import sys; print(sys.version_info[0])') == '2' ]]" provider: releases api_key: secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg= diff --git a/ci/after_script.sh b/ci/after_script.sh new file mode 100644 index 000000000..0afca5fa1 --- /dev/null +++ b/ci/after_script.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +if [[ "$TRAVIS_OS_NAME" == "linux" && \ + "$(python -c 'import sys; print(sys.version_info[0])')" == "2" && \ + "$TRAVIS_PULL_REQUEST" == "false" && \ + "$TRAVIS_BRANCH" == "master" && \ + "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then + ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT +fi diff --git a/ci/before_script.sh b/ci/before_script.sh new file mode 100644 index 000000000..c664cff3d --- /dev/null +++ b/ci/before_script.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [[ "$TRAVIS_OS_NAME" == "linux" && \ + "$(python -c 'import sys; print(sys.version_info[0])')" == "2" && \ + "$TRAVIS_PULL_REQUEST" == "false" && \ + "$TRAVIS_BRANCH" == "master" && \ + "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then + curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter + chmod +x ./cc-test-reporter + ./cc-test-reporter before-build +fi From ba8ca7319dbf6fee408b8545572ef7cbff604172 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 7 Jan 2018 19:11:53 +0200 Subject: [PATCH 3/4] build: move scripts to scripts dir Signed-off-by: Ruslan Kuprieiev --- .travis.yml | 8 ++++---- build_package.sh | 6 ------ build_linux.sh => scripts/build_linux.sh | 5 +++++ build_macos.sh => scripts/build_macos.sh | 5 +++++ scripts/build_package.sh | 11 +++++++++++ build_windows.cmd => scripts/build_windows.cmd | 2 ++ {ci => scripts/ci}/after_script.sh | 0 {ci => scripts/ci}/before_script.sh | 0 8 files changed, 27 insertions(+), 10 deletions(-) delete mode 100755 build_package.sh rename build_linux.sh => scripts/build_linux.sh (93%) rename build_macos.sh => scripts/build_macos.sh (92%) create mode 100755 scripts/build_package.sh rename build_windows.cmd => scripts/build_windows.cmd (91%) rename {ci => scripts/ci}/after_script.sh (100%) rename {ci => scripts/ci}/before_script.sh (100%) diff --git a/.travis.yml b/.travis.yml index 68b5d4899..f92a44859 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,14 +44,14 @@ install: - printf "[default]\naws_access_key_id = $AWS_ACCESS_KEY_ID\naws_secret_access_key = $AWS_SECRET_ACCESS_KEY\n" > ~/.aws/credentials - printf "[default]\n" > ~/.aws/config before_script: - - bash ./ci/before_script.sh + - bash ./scripts/ci/before_script.sh script: - ./unittests.sh - ./functests.sh - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./build_macos.sh; else ./build_linux.sh; fi - - ./build_package.sh + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./scripts/build_macos.sh; else ./scripts/build_linux.sh; fi + - ./scripts/build_package.sh after_script: - - bash ./ci/after_scripts.sh + - bash ./scripts/ci/after_scripts.sh deploy: condition: "[[ $(python -c 'import sys; print(sys.version_info[0])') == '2' ]]" provider: releases diff --git a/build_package.sh b/build_package.sh deleted file mode 100755 index 11662fbd8..000000000 --- a/build_package.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -e - -python setup.py sdist -python setup.py bdist_wheel --universal diff --git a/build_linux.sh b/scripts/build_linux.sh similarity index 93% rename from build_linux.sh rename to scripts/build_linux.sh index 89c8a981d..f9f339d9c 100755 --- a/build_linux.sh +++ b/scripts/build_linux.sh @@ -11,6 +11,11 @@ print_error() echo -e "\e[31m$1\e[0m" } +if [ ! -d "dvc" ]; then + print_error "Please run this script from repository root" + exit 1 +fi + trap 'print_error "FAIL"; exit 1' ERR print_info() diff --git a/build_macos.sh b/scripts/build_macos.sh similarity index 92% rename from build_macos.sh rename to scripts/build_macos.sh index f0c09ead8..556e8968f 100755 --- a/build_macos.sh +++ b/scripts/build_macos.sh @@ -11,6 +11,11 @@ print_error() echo -e "\e[31m$1\e[0m" } +if [ ! -d "dvc" ]; then + print_error "Please run this script from repository root" + exit 1 +fi + trap 'print_error "FAIL"; exit 1' ERR print_info() diff --git a/scripts/build_package.sh b/scripts/build_package.sh new file mode 100755 index 000000000..429feef39 --- /dev/null +++ b/scripts/build_package.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +if [ ! -d "dvc" ]; then + echo "Please run this script from repository root" + exit 1 +fi + +python setup.py sdist +python setup.py bdist_wheel --universal diff --git a/build_windows.cmd b/scripts/build_windows.cmd similarity index 91% rename from build_windows.cmd rename to scripts/build_windows.cmd index 9aeafadea..dc17a0bee 100644 --- a/build_windows.cmd +++ b/scripts/build_windows.cmd @@ -2,6 +2,8 @@ echo ====== Starting to build dvc installer for Windows... ====== +if not exist dvc\NUL (echo Error: Please run this script from repository root && goto :error) + rmdir /Q /S build rmdir /Q /S dist del /Q /S dvc.spec diff --git a/ci/after_script.sh b/scripts/ci/after_script.sh similarity index 100% rename from ci/after_script.sh rename to scripts/ci/after_script.sh diff --git a/ci/before_script.sh b/scripts/ci/before_script.sh similarity index 100% rename from ci/before_script.sh rename to scripts/ci/before_script.sh From c47d36baa0de1c5a0d386885acd8781076cd3f30 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 7 Jan 2018 22:39:59 +0200 Subject: [PATCH 4/4] build: fix windows build Signed-off-by: Ruslan Kuprieiev --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index b0f30e223..235047603 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -42,7 +42,7 @@ test_script: - nosetests --cover-inclusive --cover-erase --cover-package=dvc --with-coverage after_test: - - .\build_windows.cmd + - .\scripts\build_windows.cmd artifacts: - path: dvc*.exe