diff --git a/functest/clean.sh b/functest/clean.sh deleted file mode 100755 index 8d39fe6d4..000000000 --- a/functest/clean.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -source common.sh - -rm -rf $CACHE_DIR $TEST_REPO diff --git a/functest/common.sh b/functest/common.sh deleted file mode 100644 index 598de835e..000000000 --- a/functest/common.sh +++ /dev/null @@ -1,161 +0,0 @@ -#!/bin/bash - -set -e - -SDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -CACHE_DIR=$SDIR/cache -TEST_REPO=$SDIR/myrepo -CODE_CACHE=$CACHE_DIR/code -DATA_CACHE=$CACHE_DIR/data - -DATA="foo bar" -CODE="code.sh" - -REGION='us-west-2' -BASE_DIR_S3=https://s3-$REGION.amazonaws.com/dataversioncontrol/functests -DATA_S3=$BASE_DIR_S3/dummy_data -CODE_S3=$BASE_DIR_S3/dummy_code - -TEST_REPO_S3="dvc-test/$(uuidgen)" -TEST_REPO_REGION='us-east-2' - -TEST_REPO_GCP='dvc-test/myrepo' -TEST_REPO_GCP_PROJECT='dvc-project' - -function is_osx() { - if [ "$(uname)" == "Darwin" ]; then - return 0 - else - return 1 - fi -} - -function dvc_md5() { - if is_osx; then - md5 -r $@ - else - md5sum $@ - fi -} - -function dvc_timestamp() { - if is_osx; then - stat -f "%m%t%Sm %N" $@ - else - stat -c %y data/foo $@ - fi -} - -function dvc_print() { - echo -e "$1" -} - -function dvc_info() { - dvc_print "\e[34m$1\e[0m" -} - -function dvc_header() { - dvc_info "=============== DVC test: $1 ===============" -} - -function dvc_error() { - dvc_print "\e[31m$1\e[0m" -} - -function dvc_fail() { - dvc_error "FAIL" - exit 1 -} - -function dvc_fatal() { - dvc_error "$1" - dvc_fail -} - -function dvc_pass() { - dvc_print "\e[32mPASS\e[0m" - exit 0 -} - -trap 'dvc_fail;' ERR - -function dvc_prepare_cache() { - if [ -d $CACHE_DIR ]; then - return - fi - - mkdir -p $DATA_CACHE - pushd $DATA_CACHE - - for f in $DATA; do - wget $DATA_S3/$f - done - - popd - - mkdir -p $CODE_CACHE - pushd $CODE_CACHE - - for f in $CODE; do - wget $CODE_S3/$f - done - - popd -} - -function dvc_create_git_repo() { - dvc_info "Creating git repo" - - dvc_prepare_cache - - rm -rf $TEST_REPO - mkdir $TEST_REPO - cd $TEST_REPO - - git init - - git config user.name "DVC tester" - git config user.email "dvctester@dataversioncontrol.com" - - mkdir code - cp $CODE_CACHE/* code/ - git add code/ - git commit -m 'Add code' -} - -function dvc_create_repo() { - dvc_info "Creating dvc repo" - - dvc_create_git_repo - - dvc init - - mkdir data - cp $DATA_CACHE/* data -} - -function dvc_clean_cloud_aws() { - dvc_info "Cleaning $TEST_REPO_S3" - aws s3 rm --recursive "s3://$TEST_REPO_S3/" -} - -function dvc_clean_cloud_gcp() { - dvc_info "Cleaning $TEST_REPO_GCP_PROJECT/$TEST_REPO_GCP" - gsutil rm -rf "gs://$TEST_REPO_GCP/" || true -} - -function dvc_check_dirs() { - for d in $1; do - if [ ! -d "$d" ]; then - dvc_fatal "Directory $d doesn't exist" - fi - done -} - -function dvc_check_files() { - for f in $1; do - if [ ! -f "$f" ]; then - dvc_fatal "File $f doesn't exist" - fi - done -} diff --git a/functest/requirements.txt b/functest/requirements.txt deleted file mode 100644 index 80a128f8e..000000000 --- a/functest/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -dvc -pandas -sklearn -scipy -configobj -networkx diff --git a/functest/run.sh b/functest/run.sh deleted file mode 100755 index 1a3fd5e10..000000000 --- a/functest/run.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -e - -source common.sh - -TESTS+=" test-init.sh" -TESTS+=" test-add-remove.sh" -TESTS+=" test-sync.sh" -TESTS+=" test-repro-code.sh" -TESTS+=" test-checkout.sh" -TESTS+=" test-gc.sh" -TESTS+=" test-lock.sh" - -for t in $TESTS; do - rm -rf $TEST_REPO - dvc_header "$t" - ./$t -done - -rm -rf $TEST_REPO diff --git a/functest/test-add-remove.sh b/functest/test-add-remove.sh deleted file mode 100755 index 72fca7729..000000000 --- a/functest/test-add-remove.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -set -e - -source common.sh - -dvc_create_repo - -dvc add data/foo -dvc_check_files data/foo.dvc data/foo - -dvc remove data/foo -if [ -f "data/foo" ]; then - echo "data/foo was not removed" - dvc_fail -fi -if [ -f "data/foo.dvc" ]; then - echo "data/foo.dvc was not removed" - dvc_fail -fi - -dvc_pass diff --git a/functest/test-checkout.sh b/functest/test-checkout.sh deleted file mode 100755 index c78a466f0..000000000 --- a/functest/test-checkout.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -e - -source common.sh - -dvc_create_repo - -dvc add data/foo - -rm -f data/foo -dvc checkout -dvc_check_files data/foo -dvc_pass diff --git a/functest/test-gc.sh b/functest/test-gc.sh deleted file mode 100755 index 6e52abaab..000000000 --- a/functest/test-gc.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -set -e - -source common.sh - -dvc_create_repo - -ORIG_CACHE=$(ls .dvc/cache) - -DUMMY_CACHE+=" .dvc/cache/1" -DUMMY_CACHE+=" .dvc/cache/2" -DUMMY_CACHE+=" .dvc/cache/3" - -dvc_info "Create dummy cache files" -touch $DUMMY_CACHE -if [ $? -ne 0 ]; then - dvc_error "Failed to create cache files" - dvc_fail -fi - -dvc_info "Run 'dvc gc'" -dvc gc - -dvc_info "Check that we didn't break good cache" -dvc_check_dirs ".dvc/cache" -pushd .dvc/cache -dvc_check_files $ORIG_CACHE -popd - -dvc_info "Check that dummy cache files were removed" -for d in $DUMMY_CACHE; do - if [ -f $d ]; then - dvc_error "Dummy cache file $d was not removed by gc" - dvc_fail - fi -done - -dvc_pass diff --git a/functest/test-init.sh b/functest/test-init.sh deleted file mode 100755 index 2fb773e54..000000000 --- a/functest/test-init.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -e - -source common.sh - -dvc_create_git_repo - -dvc init - -DIRS=".dvc/cache .dvc/state" -FILES=".dvc/config" - -dvc_check_dirs $DIRS -dvc_check_files $FILES - -dvc_pass diff --git a/functest/test-lock.sh b/functest/test-lock.sh deleted file mode 100755 index 15f6e1b2c..000000000 --- a/functest/test-lock.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -e - -source common.sh - -dvc_create_repo - -dvc add data/foo -dvc run -f Dvcfile -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 -cat Dvcfile | grep 'locked' | grep 'false' || dvc_fatal "stage locked after 'dvc run'" -dvc lock -cat Dvcfile | grep 'locked' | grep 'true' || dvc_fatal "stage not locked after 'dvc lock'" -dvc lock --unlock -cat Dvcfile | grep 'locked' | grep 'false' || dvc_fatal "stage locked after 'dvc lock --unlock'" -dvc_pass diff --git a/functest/test-repro-code.sh b/functest/test-repro-code.sh deleted file mode 100755 index fbae55e17..000000000 --- a/functest/test-repro-code.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash - -set -e - -source common.sh - -function test_generic() { - dvc_info '=== Generic repro test' - - dvc_create_repo - - dvc_info 'Copy foo into foo1' - dvc add data/foo - dvc run -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 - - dvc_info 'Modify code' - echo " " >> code/code.sh - git commit -am 'Change code' - - dvc_info 'Reproduce foo1' - dvc repro foo1.dvc - dvc_check_files data/foo1 data/foo1.dvc - if [ "$(cat data/foo1)" != "foo" ]; then - dvc_fail - fi - - dvc_info 'Modify foo' - rm -f data/foo - cp $DATA_CACHE/bar data/foo - - dvc_info 'Reproduce foo1 as default target' - dvc repro foo1.dvc - dvc_check_files data/foo1 data/foo1.dvc - if [ "$(cat data/foo1)" != "bar" ]; then - dvc_fail - fi -} - -function test_partial() { - dvc_info "=== Partial repro test" - - dvc_create_repo - - cp code/code.sh code/code1.sh - cp code/code1.sh code/code2.sh - cp code/code2.sh code/code3.sh - git add code - git commit -m "copy code" - - dvc_info "Create repro chain foo -> foo1 -> foo2 -> foo3" - dvc add data/foo - dvc run -f copy_foo_foo1.dvc -d code/code1.sh -d data/foo -o data/foo1 cp data/foo data/foo1 - dvc run -f copy_foo1_foo2.dvc -d code/code2.sh -d data/foo1 -o data/foo2 cp data/foo1 data/foo2 - dvc run -f copy_foo2_foo3.dvc -d code/code3.sh -d data/foo2 -o data/foo3 cp data/foo2 data/foo3 - - dvc_info "Save original timestamps" - FOO_TS=$(dvc_timestamp data/foo) - FOO1_TS=$(dvc_timestamp data/foo1) - FOO2_TS=$(dvc_timestamp data/foo2) - FOO3_TS=$(dvc_timestamp data/foo3) - - dvc_info "Modify dependency for data/foo1, that will not result in data/foo1 contents being changed" - echo " " >> code/code1.sh - git add code - git commit -m "modify code1.sh" - - dvc_info "Reproduce data/foo3" - dvc repro copy_foo2_foo3.dvc - - dvc_info "Check timestamps" - if [ "$FOO_TS" != "$(dvc_timestamp data/foo)" ]; then - dvc_error "data/foo timestamp changed" - dvc_fail - fi - - if [ "$FOO1_TS" != "$(dvc_timestamp data/foo1)" ]; then - dvc_error "data/foo1 timestamp changed" - dvc_fail - fi - - if [ "$FOO2_TS" != "$(dvc_timestamp data/foo2)" ]; then - dvc_error "data/foo2 timestamp changed" - dvc_fail - fi - - if [ "$FOO3_TS" != "$(dvc_timestamp data/foo3)" ]; then - dvc_error "data/foo3 timestamp changed" - dvc_fail - fi -} - -test_generic -test_partial - -dvc_pass diff --git a/functest/test-sync.sh b/functest/test-sync.sh deleted file mode 100755 index 3b40651ea..000000000 --- a/functest/test-sync.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -set -e - -source common.sh - -function test_sync() { - dvc add data/foo - dvc run -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 - - dvc_info "Checking status" - dvc status | grep "new file" || dvc_fail - - dvc_info "Pushing data" - dvc push - - dvc_info "Checking status" - dvc status - - dvc_info "Removing all cache" - rm -rf .dvc/cache/* - rm -rf data/foo data/bar - - dvc_info "Checking status" - dvc status | grep "deleted" || dvc_fail - - dvc_info "Pulling data" - dvc pull - dvc_check_files data/foo data/bar - - dvc_info "Checking status" - dvc status -} - -function test_aws() { - dvc_create_repo - - dvc_info "Setting up AWS cloud" - dvc_clean_cloud_aws - dvc config aws.storagepath $TEST_REPO_S3 - dvc config aws.region $TEST_REPO_REGION - dvc config global.cloud AWS - - test_sync - - dvc_clean_cloud_aws -} - -function test_gcp() { - dvc_create_repo - - dvc_info "Setting up GCP cloud" - dvc_clean_cloud_gcp - dvc config gcp.storagepath $TEST_REPO_GCP - dvc config gcp.projectname $TEST_REPO_GCP_PROJECT - dvc config global.cloud GCP - - test_sync - - dvc_clean_cloud_gcp -} - -if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then - test_aws - # Temporarily disabled for travis - if [[ "$TRAVIS" != "true" ]]; then - test_gcp - fi -fi - -dvc_pass diff --git a/functest/tutorial_branches.sh b/functest/tutorial_branches.sh deleted file mode 100755 index 48c9cf290..000000000 --- a/functest/tutorial_branches.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/bin/bash - -set -e - -# 1. First ML model - -mkdir myrepo -cd myrepo - -git init -echo "This is an empty readme" > README.md -git add README.md -git commit -m 'add readme' - -mkdir code -wget -nv -P code/ https://s3-us-west-2.amazonaws.com/dvc-share/so/code/featurization.py \ - https://s3-us-west-2.amazonaws.com/dvc-share/so/code/evaluate.py \ - https://s3-us-west-2.amazonaws.com/dvc-share/so/code/train_model.py \ - https://s3-us-west-2.amazonaws.com/dvc-share/so/code/split_train_test.py \ - https://s3-us-west-2.amazonaws.com/dvc-share/so/code/xml_to_tsv.py \ - https://s3-us-west-2.amazonaws.com/dvc-share/so/code/requirements.txt -#pip install -U -r code/requirements.txt -# Edit eval file format -git add code/ -git commit -m 'Download code' - -dvc init - -git checkout -b input_100K -mkdir data -wget https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz -O data/Posts.xml.tgz -#cp ../stackoverflow_small_xml/100K/Posts.xml.tgz data/ -dvc add data/Posts.xml.tgz -dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/ -dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python -git add . -git commit -m 'Extract TSV for 100K dataset' - -git checkout master -dvc checkout -git checkout -b input_25K -dvc checkout -mkdir data -wget https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz -O data/Posts.xml.tgz -#cp ../stackoverflow_small_xml/25K/Posts.xml.tgz data/ -dvc add data/Posts.xml.tgz -dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/ - -dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python -dvc run -d code/split_train_test.py -d data/Posts.tsv -o data/Posts-train.tsv -o data/Posts-test.tsv python code/split_train_test.py data/Posts.tsv 0.33 20170426 data/Posts-train.tsv data/Posts-test.tsv -dvc run -d code/featurization.py -d data/Posts-train.tsv -d data/Posts-test.tsv -o data/matrix-train.p -o data/matrix-test.p python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p - -dvc run -d code/train_model.py -d data/matrix-train.p -o data/model.p python code/train_model.py data/matrix-train.p 20170426 data/model.p - -dvc run -d code/evaluate.py -d data/model.p -d data/matrix-test.p -O data/eval_auc.txt python code/evaluate.py data/model.p data/matrix-test.p data/eval_auc.txt - -#dvc config Global.Target data/eval_auc.txt -#git commit -am 'Set Target' - -cat data/eval_auc.txt -# AUC: 0.596182 - -git add . -git commit -m 'Full pipeline' - - -git checkout input_100K -dvc checkout -git merge -X theirs input_25K # git merge -X theirs input_25K -## Resolve conflicts by hands!!! -#git add . -#git commit -m 'Merge conflicts' -dvc merge -dvc repro # <-- Error. Reproducng Posts.xml and Posts.tsv - - -# Improve the model: -git checkout input_25K -dvc checkout -vi code/train_model.py # Change: estimators=500 -git commit -am 'estimators=500' -vi code/train_model.py # Change: n_jobs=6 -git commit -am 'n_jobs=6' -dvc repro -cat data/eval_auc.txt -# 0.619262 -vi code/featurization.py # Change: add ngram_range=(1, 2) to CountVectorizer -git commit -am 'Add bigrams' -dvc repro -cat data/eval_auc.txt -# 0.628989 -vi code/featurization.py -git commit -am 'Add three-grams' # Change: ngram_range=(1, 3) -dvc repro -cat data/eval_auc.txt -# 0.630682 -vi code/featurization.py -git commit -am 'Add 4-grams' # Change: ngram_range=(1, 4) -dvc repro -cat data/eval_auc.txt -# 0.621002 - -# Create a branch for unsuccessful commit -git checkout -b four_grams -dvc checkout -# The last successful commit which is the second last humman-created commit -git log --pretty=oneline | grep -v "DVC repro-run: " | head -n 2 | tail -n 1 -# Move the current HEAD to the last successful commit -git branch -f input_25K `git log --pretty=oneline | grep -v "DVC repro-run: " | head -n 2 | tail -n 1 | cut -d" " -f1` -git checkout input_25K -dvc checkout - -# Continue to optimize the target metrics -vi code/train_model.py # Change: min_samples_split=5 -git commit -am 'min_samples_split=5' -dvc repro - -git checkout input_100K -dvc checkout -git merge -X theirs input_25K -git add data/ -git commit -am "After merge" -dvc repro - -## - -vi code/featurization.py -# 0.578447 - -# 2. Reproduce: change input dataset - -dvc remove data/Posts.xml.tgz -#dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz data/ -dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz data/ -dvc repro data/evaluation.txt -cat data/evaluation.txt -# AUC: 0.639861 - -# 3. Share your research - -# Improve features -echo " " >> code/featurization.py -git add code/featurization.py -git commit -m 'Include bigram' -dvc repro data/evaluation.txt -cat data/evaluation.txt diff --git a/functest/tutorial_end_to_end.sh b/functest/tutorial_end_to_end.sh deleted file mode 100755 index 531be5f18..000000000 --- a/functest/tutorial_end_to_end.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -set -e - -# 1. First ML model - -rm -rf myrepo -mkdir myrepo -cd myrepo -mkdir data -mkdir code -wget -nv -P code/ https://s3-us-west-2.amazonaws.com/dvc-share/so/code/featurization.py \ - https://s3-us-west-2.amazonaws.com/dvc-share/so/code/evaluate.py \ - https://s3-us-west-2.amazonaws.com/dvc-share/so/code/train_model.py \ - https://s3-us-west-2.amazonaws.com/dvc-share/so/code/split_train_test.py \ - https://s3-us-west-2.amazonaws.com/dvc-share/so/code/xml_to_tsv.py \ - https://s3-us-west-2.amazonaws.com/dvc-share/so/code/requirements.txt -pip install -U -r code/requirements.txt - -git init -git add code/ -git commit -m 'Download code' - - -dvc init -wget -P data/ https://s3-us-west-2.amazonaws.com/dvc-share/so/10K/Posts.xml.tgz -dvc add data/Posts.xml.tgz -dvc run -d data/Posts.xml.tgz -o data/Posts.xml tar zxf data/Posts.xml.tgz -C data/ - -dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python -dvc run -d code/split_train_test.py -d data/Posts.tsv -o data/Posts-test.tsv python code/split_train_test.py data/Posts.tsv 0.33 20170426 data/Posts-train.tsv data/Posts-test.tsv -dvc run -d code/featurization.py -d data/Posts-train.tsv -d data/Posts-test.tsv -o data/matrix-train.p -o data/matrix-test.p python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p - -dvc run -d code/train_model.py -d data/matrix-train.p -o data/model.p python code/train_model.py data/matrix-train.p 20170426 data/model.p - -dvc run -d code/evaluate.py -d data/model.p -d data/matrix-test.p -o data/evaluation.txt python code/evaluate.py data/model.p data/matrix-test.p data/evaluation.txt - -cat data/evaluation.txt -# AUC: 0.552980 - -exit 0 - -# 2. Reproduce: change input dataset - -dvc remove data/Posts.xml.tgz -#dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz data/ - -wget -P data/ https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz -dvc add data/Posts.xml.tgz -dvc repro -v data/evaluation.txt -cat data/evaluation.txt -# AUC: 0.639861 - -# 3. Share your research - -# Improve features -echo " " >> code/featurization.py -git add code/featurization.py -git commit -m 'Include bigram' -dvc repro data/evaluation.txt -cat data/evaluation.txt diff --git a/functests.sh b/functests.sh deleted file mode 100755 index 4c55eee8c..000000000 --- a/functests.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -BIN=$DIR/bin -DVC_HOME=$DIR - -export PATH=$BIN:$PATH -export DVC_HOME=$DIR - -cd functest -./run.sh diff --git a/scripts/ci/script.sh b/scripts/ci/script.sh index 3b45b2614..8fcfc970a 100644 --- a/scripts/ci/script.sh +++ b/scripts/ci/script.sh @@ -4,7 +4,6 @@ set -x set -e ./unittests.sh -./functests.sh if [[ ! -z "$TRAVIS_TAG" ]]; then if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then