mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 23:11:44 +01:00
functest: rework test system and introduce new tests
Reworked existing tests and test-system, added a few new tests(init, reworked repro, import). Fixes #175 Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
+1
-1
@@ -2,4 +2,4 @@
|
||||
|
||||
source common.sh
|
||||
|
||||
rm -rf $RAW_DATA_LOCAL $CODE_LOCAL $TEST_REPO
|
||||
rm -rf $CACHE_DIR $TEST_REPO
|
||||
|
||||
+107
-25
@@ -2,37 +2,119 @@
|
||||
|
||||
set -e
|
||||
|
||||
BASE_DIR_S3=https://s3-us-west-2.amazonaws.com/dataversioncontrol/functests
|
||||
RAW_DATA_S3=$BASE_DIR_S3/stackoverflow_raw_small
|
||||
SDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
CACHE_DIR=$SDIR/cache
|
||||
TEST_REPO=$SDIR/myrepo
|
||||
CODE_CACHE=$CACHE_DIR/code
|
||||
DATA_CACHE=$CACHE_DIR/data
|
||||
|
||||
RAW_DATA_LOCAL=sync_raw_data
|
||||
CODE_LOCAL=sync_code
|
||||
TEST_REPO=repo_test
|
||||
DATA="Badges.xml Tags.xml PostLinks.xml"
|
||||
CODE="xmltotsv.py"
|
||||
|
||||
REGION='us-west-2'
|
||||
BASE_DIR_S3=https://s3-$REGION.amazonaws.com/dataversioncontrol/functests
|
||||
DATA_S3=$BASE_DIR_S3/stackoverflow_raw_small
|
||||
CODE_S3=$BASE_DIR_S3/code
|
||||
|
||||
function dvc_print() {
|
||||
echo -e "$1"
|
||||
}
|
||||
|
||||
function dvc_info() {
|
||||
echo "=============== DVC test: $1 ==============="
|
||||
dvc_print "\e[34m$1\e[0m"
|
||||
}
|
||||
|
||||
function _init_repo() {
|
||||
dvc_info 'init reporitory'
|
||||
git init
|
||||
dvc init
|
||||
|
||||
dvc_info 'import xml files'
|
||||
mkdir data/xml
|
||||
dvc import ../$RAW_DATA_LOCAL/* data/xml
|
||||
|
||||
dvc_info 'copy code'
|
||||
mkdir code
|
||||
cp ../$CODE_LOCAL/* code/
|
||||
git add code/
|
||||
git commit -m 'Add code'
|
||||
function dvc_header() {
|
||||
dvc_info "=============== DVC test: $1 ==============="
|
||||
}
|
||||
|
||||
function create_repo() {
|
||||
dvc_info 'create reporitory'
|
||||
rm -rf $TEST_REPO
|
||||
mkdir $TEST_REPO
|
||||
(cd $TEST_REPO && _init_repo)
|
||||
function dvc_error() {
|
||||
dvc_print "\e[31m$1\e[0m"
|
||||
}
|
||||
|
||||
function dvc_fail() {
|
||||
dvc_error "FAIL"
|
||||
exit 1
|
||||
}
|
||||
|
||||
function dvc_fatal() {
|
||||
dvc_error "$1"
|
||||
dvc_fail
|
||||
}
|
||||
|
||||
function dvc_pass() {
|
||||
dvc_print "\e[32mPASS\e[0m"
|
||||
exit 0
|
||||
}
|
||||
|
||||
trap 'dvc_fail;' ERR
|
||||
|
||||
function dvc_prepare_cache() {
|
||||
if [ -d $CACHE_DIR ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
mkdir -p $DATA_CACHE
|
||||
pushd $DATA_CACHE
|
||||
|
||||
for f in $DATA; do
|
||||
wget $DATA_S3/$f
|
||||
done
|
||||
|
||||
popd
|
||||
|
||||
mkdir -p $CODE_CACHE
|
||||
pushd $CODE_CACHE
|
||||
|
||||
for f in $CODE; do
|
||||
wget $CODE_S3/$f
|
||||
done
|
||||
|
||||
popd
|
||||
}
|
||||
|
||||
function dvc_create_git_repo() {
|
||||
dvc_info "Creating git repo"
|
||||
|
||||
dvc_prepare_cache
|
||||
|
||||
mkdir $TEST_REPO
|
||||
cd $TEST_REPO
|
||||
|
||||
git init
|
||||
|
||||
git config user.name "DVC tester"
|
||||
git config user.email "[email protected]"
|
||||
|
||||
mkdir code
|
||||
cp $CODE_CACHE/* code/
|
||||
git add code/
|
||||
git commit -m 'Add code'
|
||||
}
|
||||
|
||||
function dvc_create_repo() {
|
||||
dvc_info "Creating dvc repo"
|
||||
|
||||
dvc_create_git_repo
|
||||
|
||||
dvc init
|
||||
|
||||
mkdir data/xml
|
||||
dvc import $DATA_CACHE/* data/xml
|
||||
}
|
||||
|
||||
function dvc_check_dirs() {
|
||||
for d in $1; do
|
||||
if [ ! -d "$d" ]; then
|
||||
dvc_fatal "Directory $d doesn't exist"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function dvc_check_files() {
|
||||
for f in $1; do
|
||||
if [ ! -f "$f" ]; then
|
||||
dvc_fatal "File $f doesn't exist"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
source common.sh
|
||||
|
||||
./clean.sh
|
||||
|
||||
rm -rf $RAW_DATA_LOCAL
|
||||
mkdir $RAW_DATA_LOCAL
|
||||
for file in "Badges.xml" "Tags.xml" "PostLinks.xml"
|
||||
do
|
||||
(cd $RAW_DATA_LOCAL; wget $RAW_DATA_S3/$file)
|
||||
done
|
||||
|
||||
rm -rf $CODE_LOCAL
|
||||
mkdir $CODE_LOCAL
|
||||
(cd $CODE_LOCAL; wget "$BASE_DIR_S3/code/xmltotsv.py")
|
||||
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
source common.sh
|
||||
|
||||
TESTS+=" test-init.sh"
|
||||
TESTS+=" test-import.sh"
|
||||
TESTS+=" test-repro-code.sh"
|
||||
|
||||
for t in $TESTS; do
|
||||
rm -rf $TEST_REPO
|
||||
dvc_header "$t"
|
||||
./$t
|
||||
done
|
||||
|
||||
rm -rf $TEST_REPO
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
source common.sh
|
||||
|
||||
dvc_create_repo
|
||||
|
||||
dvc_info "Import from local file"
|
||||
dvc import $DATA_CACHE/Tags.xml data/local
|
||||
dvc_check_files "data/local"
|
||||
|
||||
dvc_info "Import from http url"
|
||||
dvc import $DATA_S3/Tags.xml data/http
|
||||
dvc_check_files "data/http"
|
||||
|
||||
# Temporarily disabled for travis builds, as
|
||||
# they don't have aws credentials setup yet.
|
||||
if [ -z $TRAVIS ]; then
|
||||
dvc_info "Import from s3 url"
|
||||
dvc config aws.region $REGION
|
||||
dvc import s3://dataversioncontrol/functests/stackoverflow_raw_small/Tags.xml data/s3
|
||||
dvc_check_files "data/s3"
|
||||
fi
|
||||
|
||||
dvc_pass
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
source common.sh
|
||||
|
||||
dvc_create_git_repo
|
||||
|
||||
dvc init
|
||||
|
||||
DIRS="data .cache .state"
|
||||
FILES="dvc.conf"
|
||||
|
||||
dvc_check_dirs $DIRS
|
||||
dvc_check_files $FILES
|
||||
|
||||
dvc_pass
|
||||
+14
-14
@@ -4,19 +4,19 @@ set -e
|
||||
|
||||
source common.sh
|
||||
|
||||
function run_test() {
|
||||
mkdir data/tsv
|
||||
dvc_create_repo
|
||||
|
||||
dvc_info 'convert Budget.xml to tsv'
|
||||
dvc run --code code/xmltotsv.py python code/xmltotsv.py --extract row/@Id,row/@UserId,row/@Name,row/@Class,row/@TagBased,row/@Date data/xml/Badges.xml data/tsv/Badges.tsv
|
||||
|
||||
dvc_info 'modify code'
|
||||
echo " " >> code/xmltotsv.py
|
||||
git commit -am 'Change code'
|
||||
|
||||
dvc_info 'reproduce tsv convertion code'
|
||||
dvc repro data/tsv/Badges.tsv
|
||||
}
|
||||
mkdir data/tsv
|
||||
|
||||
create_repo
|
||||
(cd $TEST_REPO; run_test)
|
||||
dvc_info 'Convert Budget.xml to tsv'
|
||||
pip2 install xmltodict
|
||||
dvc run --code code/xmltotsv.py python2 code/xmltotsv.py --extract row/@Id,row/@UserId,row/@Name,row/@Class,row/@TagBased,row/@Date data/xml/Badges.xml data/tsv/Badges.tsv
|
||||
|
||||
dvc_info 'Modify code'
|
||||
echo " " >> code/xmltotsv.py
|
||||
git commit -am 'Change code'
|
||||
|
||||
dvc_info 'Reproduce tsv convertion code'
|
||||
dvc repro data/tsv/Badges.tsv
|
||||
|
||||
dvc_pass
|
||||
|
||||
Reference in New Issue
Block a user