functest: rework test system and introduce new tests

Reworked existing tests and test-system, added a few new
tests(init, reworked repro, import).

Fixes #175

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2017-08-20 09:32:32 +03:00
parent 194e1ec419
commit 0f45afb3a4
7 changed files with 182 additions and 57 deletions
+1 -1
View File
@@ -2,4 +2,4 @@
source common.sh
rm -rf $RAW_DATA_LOCAL $CODE_LOCAL $TEST_REPO
rm -rf $CACHE_DIR $TEST_REPO
+107 -25
View File
@@ -2,37 +2,119 @@
set -e
BASE_DIR_S3=https://s3-us-west-2.amazonaws.com/dataversioncontrol/functests
RAW_DATA_S3=$BASE_DIR_S3/stackoverflow_raw_small
SDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
CACHE_DIR=$SDIR/cache
TEST_REPO=$SDIR/myrepo
CODE_CACHE=$CACHE_DIR/code
DATA_CACHE=$CACHE_DIR/data
RAW_DATA_LOCAL=sync_raw_data
CODE_LOCAL=sync_code
TEST_REPO=repo_test
DATA="Badges.xml Tags.xml PostLinks.xml"
CODE="xmltotsv.py"
REGION='us-west-2'
BASE_DIR_S3=https://s3-$REGION.amazonaws.com/dataversioncontrol/functests
DATA_S3=$BASE_DIR_S3/stackoverflow_raw_small
CODE_S3=$BASE_DIR_S3/code
function dvc_print() {
echo -e "$1"
}
function dvc_info() {
echo "=============== DVC test: $1 ==============="
dvc_print "\e[34m$1\e[0m"
}
function _init_repo() {
dvc_info 'init reporitory'
git init
dvc init
dvc_info 'import xml files'
mkdir data/xml
dvc import ../$RAW_DATA_LOCAL/* data/xml
dvc_info 'copy code'
mkdir code
cp ../$CODE_LOCAL/* code/
git add code/
git commit -m 'Add code'
function dvc_header() {
dvc_info "=============== DVC test: $1 ==============="
}
function create_repo() {
dvc_info 'create reporitory'
rm -rf $TEST_REPO
mkdir $TEST_REPO
(cd $TEST_REPO && _init_repo)
function dvc_error() {
dvc_print "\e[31m$1\e[0m"
}
function dvc_fail() {
dvc_error "FAIL"
exit 1
}
function dvc_fatal() {
dvc_error "$1"
dvc_fail
}
function dvc_pass() {
dvc_print "\e[32mPASS\e[0m"
exit 0
}
trap 'dvc_fail;' ERR
function dvc_prepare_cache() {
if [ -d $CACHE_DIR ]; then
return
fi
mkdir -p $DATA_CACHE
pushd $DATA_CACHE
for f in $DATA; do
wget $DATA_S3/$f
done
popd
mkdir -p $CODE_CACHE
pushd $CODE_CACHE
for f in $CODE; do
wget $CODE_S3/$f
done
popd
}
function dvc_create_git_repo() {
dvc_info "Creating git repo"
dvc_prepare_cache
mkdir $TEST_REPO
cd $TEST_REPO
git init
git config user.name "DVC tester"
git config user.email "[email protected]"
mkdir code
cp $CODE_CACHE/* code/
git add code/
git commit -m 'Add code'
}
function dvc_create_repo() {
dvc_info "Creating dvc repo"
dvc_create_git_repo
dvc init
mkdir data/xml
dvc import $DATA_CACHE/* data/xml
}
function dvc_check_dirs() {
for d in $1; do
if [ ! -d "$d" ]; then
dvc_fatal "Directory $d doesn't exist"
fi
done
}
function dvc_check_files() {
for f in $1; do
if [ ! -f "$f" ]; then
dvc_fatal "File $f doesn't exist"
fi
done
}
-17
View File
@@ -1,17 +0,0 @@
#!/bin/bash
source common.sh
./clean.sh
rm -rf $RAW_DATA_LOCAL
mkdir $RAW_DATA_LOCAL
for file in "Badges.xml" "Tags.xml" "PostLinks.xml"
do
(cd $RAW_DATA_LOCAL; wget $RAW_DATA_S3/$file)
done
rm -rf $CODE_LOCAL
mkdir $CODE_LOCAL
(cd $CODE_LOCAL; wget "$BASE_DIR_S3/code/xmltotsv.py")
+17
View File
@@ -0,0 +1,17 @@
#!/bin/bash
set -e
source common.sh
TESTS+=" test-init.sh"
TESTS+=" test-import.sh"
TESTS+=" test-repro-code.sh"
for t in $TESTS; do
rm -rf $TEST_REPO
dvc_header "$t"
./$t
done
rm -rf $TEST_REPO
+26
View File
@@ -0,0 +1,26 @@
#!/bin/bash
set -e
source common.sh
dvc_create_repo
dvc_info "Import from local file"
dvc import $DATA_CACHE/Tags.xml data/local
dvc_check_files "data/local"
dvc_info "Import from http url"
dvc import $DATA_S3/Tags.xml data/http
dvc_check_files "data/http"
# Temporarily disabled for travis builds, as
# they don't have aws credentials setup yet.
if [ -z $TRAVIS ]; then
dvc_info "Import from s3 url"
dvc config aws.region $REGION
dvc import s3://dataversioncontrol/functests/stackoverflow_raw_small/Tags.xml data/s3
dvc_check_files "data/s3"
fi
dvc_pass
+17
View File
@@ -0,0 +1,17 @@
#!/bin/bash
set -e
source common.sh
dvc_create_git_repo
dvc init
DIRS="data .cache .state"
FILES="dvc.conf"
dvc_check_dirs $DIRS
dvc_check_files $FILES
dvc_pass
+14 -14
View File
@@ -4,19 +4,19 @@ set -e
source common.sh
function run_test() {
mkdir data/tsv
dvc_create_repo
dvc_info 'convert Budget.xml to tsv'
dvc run --code code/xmltotsv.py python code/xmltotsv.py --extract row/@Id,row/@UserId,row/@Name,row/@Class,row/@TagBased,row/@Date data/xml/Badges.xml data/tsv/Badges.tsv
dvc_info 'modify code'
echo " " >> code/xmltotsv.py
git commit -am 'Change code'
dvc_info 'reproduce tsv convertion code'
dvc repro data/tsv/Badges.tsv
}
mkdir data/tsv
create_repo
(cd $TEST_REPO; run_test)
dvc_info 'Convert Budget.xml to tsv'
pip2 install xmltodict
dvc run --code code/xmltotsv.py python2 code/xmltotsv.py --extract row/@Id,row/@UserId,row/@Name,row/@Class,row/@TagBased,row/@Date data/xml/Badges.xml data/tsv/Badges.tsv
dvc_info 'Modify code'
echo " " >> code/xmltotsv.py
git commit -am 'Change code'
dvc_info 'Reproduce tsv convertion code'
dvc repro data/tsv/Badges.tsv
dvc_pass