Merge pull request #389 from efiop/master

Misc test

.travis.yml

@@ -43,14 +43,17 @@ install:
   - mkdir ~/.aws
   - printf "[default]\naws_access_key_id = $AWS_ACCESS_KEY_ID\naws_secret_access_key = $AWS_SECRET_ACCESS_KEY\n" > ~/.aws/credentials
   - printf "[default]\n" > ~/.aws/config
+before_script:
+  - bash ./scripts/ci/before_script.sh
 script:
   - ./unittests.sh
   - ./functests.sh
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./build_macos.sh; else ./build_linux.sh; fi
-  - ./build_package.sh
-after_sucess:
-  - CODECLIMATE_REPO_TOKEN=7350efa14a67c3911af96dadaabe811b43eead7b466309b47687499bf5d72287 codeclimate-test-reporter
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./scripts/build_macos.sh; else ./scripts/build_linux.sh; fi
+  - ./scripts/build_package.sh
+after_script:
+  - bash ./scripts/ci/after_scripts.sh
 deploy:
+  condition: "[[ $(python -c 'import sys; print(sys.version_info[0])') == '2' ]]"
   provider: releases
   api_key:
     secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=

appveyor.yml

@@ -42,7 +42,7 @@ test_script:
   - nosetests --cover-inclusive --cover-erase --cover-package=dvc --with-coverage
 
 after_test:
-  - .\build_windows.cmd
+  - .\scripts\build_windows.cmd
 
 artifacts:
   - path: dvc*.exe

build_package.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-set -e
-
-python setup.py sdist
-python setup.py bdist_wheel --universal

dvc/data_cloud.py

@@ -192,38 +192,38 @@ class DataCloudLOCAL(DataCloudBase):
     """
     Driver for local storage.
     """
-    def push(self, item):
-        Logger.debug('sync to cloud ' + item.cache.dvc + " " + self.storage_path)
-        copyfile(item.cache.dvc, self.storage_path)
-        return item
+    def push(self, path):
+        Logger.debug('sync to cloud ' + path + " " + self.storage_path)
+        copyfile(path, self.storage_path)
+        return path
 
-    def _import(self, bucket, i, item):
-        tmp_file = self.tmp_file(item.data.dvc)
+    def _import(self, bucket, i, path):
+        inp = os.path.join(self.storage_path, i)
+        tmp_file = self.tmp_file(path)
         try:
-            copyfile(i, tmp_file)
+            copyfile(inp, tmp_file)
         except Exception as exc:
             Logger.error('Failed to copy "{}": {}'.format(i, exc))
             return None
 
-        os.rename(tmp_file, item.data.dvc)
-        item.move_data_to_cache()
+        os.rename(tmp_file, path)
 
-        return item
+        return path
 
-    def pull(self, item):
-        Logger.debug('sync from cloud ' + self.storage_path + " " + item.cache.dvc)
-        return self._import(None, self.storage_path, item)
+    def pull(self, path):
+        Logger.debug('sync from cloud ' + path)
+        return self._import(None, path, path)
 
-    def remove(self, item):
-        Logger.debug('rm from cloud ' + item.cache.dvc)
-        os.remove(item.cache.dvc)
+    def remove(self, path):
+        Logger.debug('rm from cloud ' + path)
+        os.remove(path)
 
-    def import_data(self, path, item):
-        Logger.debug('import from cloud ' + path + " " + item.data.dvc)
-        return self._import(None, path, item)
+    def import_data(self, path, out):
+        Logger.debug('import from cloud ' + path + " " + out)
+        return self._import(None, path, out)
 
-    def _status(self, data_item):
-        local = data_item.cache.relative
+    def _status(self, path):
+        local = path
         remote = '{}/{}'.format(self.storage_path, os.path.basename(local))
 
         remote_exists = os.path.exists(remote)
@@ -814,7 +814,7 @@ class DataCloud(object):
             (F, issues) if bad
         """
         key = 'Cloud'
-        if key.lower() not in self._config['Global'].keys() or len(self._config['Global'][key]) < 1:
+        if key.lower() not in [k.lower() for k in self._config['Global'].keys()] or len(self._config['Global'][key]) < 1:
             raise ConfigError('Please set %s in section Global in config file' % key)
 
         # now that a cloud is chosen, can check StoragePath

scripts/build_linux.sh

@@ -11,6 +11,11 @@ print_error()
 	echo -e "\e[31m$1\e[0m"
 }
 
+if [ ! -d "dvc" ]; then
+	print_error "Please run this script from repository root"
+	exit 1
+fi
+
 trap 'print_error "FAIL"; exit 1' ERR
 
 print_info()

scripts/build_macos.sh

@@ -11,6 +11,11 @@ print_error()
 	echo -e "\e[31m$1\e[0m"
 }
 
+if [ ! -d "dvc" ]; then
+	print_error "Please run this script from repository root"
+	exit 1
+fi
+
 trap 'print_error "FAIL"; exit 1' ERR
 
 print_info()

scripts/build_package.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+if [ ! -d "dvc" ]; then
+	echo "Please run this script from repository root"
+	exit 1
+fi
+
+python setup.py sdist
+python setup.py bdist_wheel --universal

scripts/build_windows.cmd

@@ -2,6 +2,8 @@
 
 echo ====== Starting to build dvc installer for Windows... ======
 
+if not exist dvc\NUL (echo Error: Please run this script from repository root && goto :error)
+
 rmdir /Q /S build
 rmdir /Q /S dist
 del /Q /S dvc.spec

scripts/ci/after_script.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [[ "$TRAVIS_OS_NAME" == "linux" && \
+      "$(python -c 'import sys; print(sys.version_info[0])')" == "2" && \
+      "$TRAVIS_PULL_REQUEST" == "false" && \
+      "$TRAVIS_BRANCH" == "master" && \
+      "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
+	./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
+fi

scripts/ci/before_script.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+if [[ "$TRAVIS_OS_NAME" == "linux" && \
+      "$(python -c 'import sys; print(sys.version_info[0])')" == "2" && \
+      "$TRAVIS_PULL_REQUEST" == "false" && \
+      "$TRAVIS_BRANCH" == "master" && \
+      "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
+	curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+	chmod +x ./cc-test-reporter
+	./cc-test-reporter before-build
+fi

tests/test_data_cloud.py

@@ -0,0 +1,107 @@
+import os
+
+from dvc.config import ConfigError
+from dvc.data_cloud import DataCloud, DataCloudAWS, DataCloudLOCAL
+from dvc.data_cloud import STATUS_UNKNOWN, STATUS_OK, STATUS_MODIFIED, STATUS_NEW, STATUS_DELETED
+from dvc.cloud.instance_manager import CloudSettings
+
+from tests.basic_env import TestDvc
+
+
+class TestDataCloud(TestDvc):
+    def test(self):
+        for k,v in DataCloud.CLOUD_MAP.items():
+            config = {'Global': {'Cloud': k},
+                      k: {'StoragePath': 'a/b',
+                          'ProjectName': 'name'}}
+            cloud = DataCloud(config)
+            self.assertIsInstance(cloud._cloud, v)
+
+        with self.assertRaises(ConfigError) as cx:
+            config = {'Global': {'Cloud': 'not_supported_type'}}
+            DataCloud(config)
+
+
+class TestDataCloudBase(TestDvc):
+    def _setup_cloud(self):
+        self.cloud = None
+
+    def _test_cloud(self):
+        self._setup_cloud()
+
+        # Check status
+        status = self.cloud.status(self.FOO)
+        self.assertEqual(status, STATUS_NEW)
+
+        # Push and check status
+        self.cloud.push(self.FOO)
+        self.assertTrue(os.path.exists(self.FOO))
+        self.assertTrue(os.path.isfile(self.FOO))
+
+        status = self.cloud.status(self.FOO)
+        self.assertEqual(status, STATUS_OK)
+
+        # Modify and check status
+        with open(self.FOO, 'a') as fd:
+            fd.write('addon')
+        status = self.cloud.status(self.FOO)
+        self.assertEqual(status, STATUS_MODIFIED)
+
+        # Remove and check status
+        os.unlink(self.FOO)
+        status = self.cloud.status(self.FOO)
+        self.assertEqual(status, STATUS_DELETED)
+
+        # Pull and check status
+        self.cloud.pull(self.FOO)
+        self.assertTrue(os.path.exists(self.FOO))
+        self.assertTrue(os.path.isfile(self.FOO))
+        with open(self.FOO, 'r') as fd:
+            self.assertEqual(fd.read(), self.FOO_CONTENTS)
+
+        status = self.cloud.status(self.FOO)
+        self.assertEqual(status, STATUS_OK)
+
+
+class TestDataCloudAWS(TestDataCloudBase):
+    TEST_REPO_S3 = 'dvc-test/myrepo'
+    TEST_REPO_REGION = 'us-east-2'
+
+    def _should_test(self):
+        if os.getenv("TRAVIS_PULL_REQUEST") == "false" and \
+           os.getenv("TRAVIS_SECURE_ENV_VARS") == "true":
+            return True
+        return False
+
+    def _setup_cloud(self):
+        if not self._should_test():
+            return
+
+        # Cleanup
+        os.system('aws s3 rm --recursive "s3://{}/"'.format(self.TEST_REPO_S3))
+
+        # Setup cloud
+        config = {'StoragePath': self.TEST_REPO_S3,
+                  'Region': self.TEST_REPO_REGION}
+        cloud_settings = CloudSettings(None, config)
+        self.cloud = DataCloudAWS(cloud_settings)
+
+    def test(self):
+        if os.getenv("TRAVIS_PULL_REQUEST") == "false" and \
+           os.getenv("TRAVIS_SECURE_ENV_VARS") == "true":
+            self._test_cloud()
+
+
+class TestDataCloudLOCAL(TestDataCloudBase):
+    def _setup_cloud(self):
+        self.dname = 'cloud'
+        os.mkdir(self.dname)
+        config = {'StoragePath': self.dname}
+        cloud_settings = CloudSettings(None, config)
+        self.cloud = DataCloudLOCAL(cloud_settings)
+
+    def test(self):
+        self._test_cloud()
+        self.assertTrue(os.path.isdir(self.dname))
+        path = os.path.join(self.dname, self.FOO)
+        self.assertTrue(os.path.isfile(path))