From 0f45afb3a43805006831f0df5be061bb8537584a Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 20 Aug 2017 08:56:30 +0300 Subject: [PATCH] functest: rework test system and introduce new tests Reworked existing tests and test-system, added a few new tests(init, reworked repro, import). Fixes #175 Signed-off-by: Ruslan Kuprieiev --- functest/clean.sh | 2 +- functest/common.sh | 132 +++++++++++++++++++++++++++++------- functest/init.sh | 17 ----- functest/run.sh | 17 +++++ functest/test-import.sh | 26 +++++++ functest/test-init.sh | 17 +++++ functest/test-repro-code.sh | 28 ++++---- 7 files changed, 182 insertions(+), 57 deletions(-) delete mode 100755 functest/init.sh create mode 100755 functest/run.sh create mode 100755 functest/test-import.sh create mode 100755 functest/test-init.sh diff --git a/functest/clean.sh b/functest/clean.sh index 8e84c920e..8d39fe6d4 100755 --- a/functest/clean.sh +++ b/functest/clean.sh @@ -2,4 +2,4 @@ source common.sh -rm -rf $RAW_DATA_LOCAL $CODE_LOCAL $TEST_REPO +rm -rf $CACHE_DIR $TEST_REPO diff --git a/functest/common.sh b/functest/common.sh index dfab8f296..094296428 100644 --- a/functest/common.sh +++ b/functest/common.sh @@ -2,37 +2,119 @@ set -e -BASE_DIR_S3=https://s3-us-west-2.amazonaws.com/dataversioncontrol/functests -RAW_DATA_S3=$BASE_DIR_S3/stackoverflow_raw_small +SDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +CACHE_DIR=$SDIR/cache +TEST_REPO=$SDIR/myrepo +CODE_CACHE=$CACHE_DIR/code +DATA_CACHE=$CACHE_DIR/data -RAW_DATA_LOCAL=sync_raw_data -CODE_LOCAL=sync_code -TEST_REPO=repo_test +DATA="Badges.xml Tags.xml PostLinks.xml" +CODE="xmltotsv.py" + +REGION='us-west-2' +BASE_DIR_S3=https://s3-$REGION.amazonaws.com/dataversioncontrol/functests +DATA_S3=$BASE_DIR_S3/stackoverflow_raw_small +CODE_S3=$BASE_DIR_S3/code + +function dvc_print() { + echo -e "$1" +} function dvc_info() { - echo "=============== DVC test: $1 ===============" + dvc_print "\e[34m$1\e[0m" } -function _init_repo() { - dvc_info 'init reporitory' - git init - dvc init - - dvc_info 'import xml files' - mkdir data/xml - dvc import ../$RAW_DATA_LOCAL/* data/xml - - dvc_info 'copy code' - mkdir code - cp ../$CODE_LOCAL/* code/ - git add code/ - git commit -m 'Add code' +function dvc_header() { + dvc_info "=============== DVC test: $1 ===============" } -function create_repo() { - dvc_info 'create reporitory' - rm -rf $TEST_REPO - mkdir $TEST_REPO - (cd $TEST_REPO && _init_repo) +function dvc_error() { + dvc_print "\e[31m$1\e[0m" } +function dvc_fail() { + dvc_error "FAIL" + exit 1 +} + +function dvc_fatal() { + dvc_error "$1" + dvc_fail +} + +function dvc_pass() { + dvc_print "\e[32mPASS\e[0m" + exit 0 +} + +trap 'dvc_fail;' ERR + +function dvc_prepare_cache() { + if [ -d $CACHE_DIR ]; then + return + fi + + mkdir -p $DATA_CACHE + pushd $DATA_CACHE + + for f in $DATA; do + wget $DATA_S3/$f + done + + popd + + mkdir -p $CODE_CACHE + pushd $CODE_CACHE + + for f in $CODE; do + wget $CODE_S3/$f + done + + popd +} + +function dvc_create_git_repo() { + dvc_info "Creating git repo" + + dvc_prepare_cache + + mkdir $TEST_REPO + cd $TEST_REPO + + git init + + git config user.name "DVC tester" + git config user.email "dvctester@dataversioncontrol.com" + + mkdir code + cp $CODE_CACHE/* code/ + git add code/ + git commit -m 'Add code' +} + +function dvc_create_repo() { + dvc_info "Creating dvc repo" + + dvc_create_git_repo + + dvc init + + mkdir data/xml + dvc import $DATA_CACHE/* data/xml +} + +function dvc_check_dirs() { + for d in $1; do + if [ ! -d "$d" ]; then + dvc_fatal "Directory $d doesn't exist" + fi + done +} + +function dvc_check_files() { + for f in $1; do + if [ ! -f "$f" ]; then + dvc_fatal "File $f doesn't exist" + fi + done +} diff --git a/functest/init.sh b/functest/init.sh deleted file mode 100755 index 0e88babad..000000000 --- a/functest/init.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -source common.sh - -./clean.sh - -rm -rf $RAW_DATA_LOCAL -mkdir $RAW_DATA_LOCAL -for file in "Badges.xml" "Tags.xml" "PostLinks.xml" -do - (cd $RAW_DATA_LOCAL; wget $RAW_DATA_S3/$file) -done - -rm -rf $CODE_LOCAL -mkdir $CODE_LOCAL -(cd $CODE_LOCAL; wget "$BASE_DIR_S3/code/xmltotsv.py") - diff --git a/functest/run.sh b/functest/run.sh new file mode 100755 index 000000000..e6d3ef6db --- /dev/null +++ b/functest/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +source common.sh + +TESTS+=" test-init.sh" +TESTS+=" test-import.sh" +TESTS+=" test-repro-code.sh" + +for t in $TESTS; do + rm -rf $TEST_REPO + dvc_header "$t" + ./$t +done + +rm -rf $TEST_REPO diff --git a/functest/test-import.sh b/functest/test-import.sh new file mode 100755 index 000000000..d57e1294a --- /dev/null +++ b/functest/test-import.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +source common.sh + +dvc_create_repo + +dvc_info "Import from local file" +dvc import $DATA_CACHE/Tags.xml data/local +dvc_check_files "data/local" + +dvc_info "Import from http url" +dvc import $DATA_S3/Tags.xml data/http +dvc_check_files "data/http" + +# Temporarily disabled for travis builds, as +# they don't have aws credentials setup yet. +if [ -z $TRAVIS ]; then + dvc_info "Import from s3 url" + dvc config aws.region $REGION + dvc import s3://dataversioncontrol/functests/stackoverflow_raw_small/Tags.xml data/s3 + dvc_check_files "data/s3" +fi + +dvc_pass diff --git a/functest/test-init.sh b/functest/test-init.sh new file mode 100755 index 000000000..e291597db --- /dev/null +++ b/functest/test-init.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +source common.sh + +dvc_create_git_repo + +dvc init + +DIRS="data .cache .state" +FILES="dvc.conf" + +dvc_check_dirs $DIRS +dvc_check_files $FILES + +dvc_pass diff --git a/functest/test-repro-code.sh b/functest/test-repro-code.sh index 18653d582..58d071f3d 100755 --- a/functest/test-repro-code.sh +++ b/functest/test-repro-code.sh @@ -4,19 +4,19 @@ set -e source common.sh -function run_test() { - mkdir data/tsv +dvc_create_repo - dvc_info 'convert Budget.xml to tsv' - dvc run --code code/xmltotsv.py python code/xmltotsv.py --extract row/@Id,row/@UserId,row/@Name,row/@Class,row/@TagBased,row/@Date data/xml/Badges.xml data/tsv/Badges.tsv - - dvc_info 'modify code' - echo " " >> code/xmltotsv.py - git commit -am 'Change code' - - dvc_info 'reproduce tsv convertion code' - dvc repro data/tsv/Badges.tsv -} +mkdir data/tsv -create_repo -(cd $TEST_REPO; run_test) +dvc_info 'Convert Budget.xml to tsv' +pip2 install xmltodict +dvc run --code code/xmltotsv.py python2 code/xmltotsv.py --extract row/@Id,row/@UserId,row/@Name,row/@Class,row/@TagBased,row/@Date data/xml/Badges.xml data/tsv/Badges.tsv + +dvc_info 'Modify code' +echo " " >> code/xmltotsv.py +git commit -am 'Change code' + +dvc_info 'Reproduce tsv convertion code' +dvc repro data/tsv/Badges.tsv + +dvc_pass