From 9cd0ddb30c0c8601821885159883cbcddd555ca3 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Thu, 18 Jan 2018 00:20:05 -0800 Subject: [PATCH 1/2] Tutorial is back --- docs/tutorial/branches.sh | 119 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100755 docs/tutorial/branches.sh diff --git a/docs/tutorial/branches.sh b/docs/tutorial/branches.sh new file mode 100755 index 000000000..cbb9dd353 --- /dev/null +++ b/docs/tutorial/branches.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +set -e + +# 1. First ML model + +mkdir myrepo +cd myrepo + +git init +echo "This is an empty readme" > README.md +git add README.md +git commit -m 'add readme' + +mkdir code +wget -nv -P code/ https://s3-us-west-2.amazonaws.com/dvc-share/so/code/featurization.py \ + https://s3-us-west-2.amazonaws.com/dvc-share/so/code/evaluate.py \ + https://s3-us-west-2.amazonaws.com/dvc-share/so/code/train_model.py \ + https://s3-us-west-2.amazonaws.com/dvc-share/so/code/split_train_test.py \ + https://s3-us-west-2.amazonaws.com/dvc-share/so/code/xml_to_tsv.py \ + https://s3-us-west-2.amazonaws.com/dvc-share/so/code/requirements.txt +#pip install -U -r code/requirements.txt +# Edit eval file format +git add code/ +git commit -m 'Download code' +dvc init +git commit -m 'Init DVC' + +git checkout -b input_100K +mkdir data +#wget https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz -O data/Posts.xml.tgz +cp ../stackoverflow_small_xml/100K/Posts.xml.tgz data/ +dvc add data/Posts.xml.tgz +dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/ +dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python +git add . +git commit -m 'Extract TSV for 100K dataset' + +git checkout master +dvc checkout +git checkout -b input_25K +dvc checkout +mkdir data +#wget https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz -O data/Posts.xml.tgz +cp ../stackoverflow_small_xml/25K/Posts.xml.tgz data/ +dvc add data/Posts.xml.tgz +dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/ +dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python + +dvc run -d code/split_train_test.py -d data/Posts.tsv -o data/Posts-train.tsv -o data/Posts-test.tsv python code/split_train_test.py data/Posts.tsv 0.33 20170426 data/Posts-train.tsv data/Posts-test.tsv +dvc run -d code/featurization.py -d data/Posts-train.tsv -d data/Posts-test.tsv -o data/matrix-train.p -o data/matrix-test.p python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p +dvc run -d code/train_model.py -d data/matrix-train.p -o data/model.p python code/train_model.py data/matrix-train.p 20170426 data/model.p +dvc run -d code/evaluate.py -d data/model.p -d data/matrix-test.p -O data/eval_auc.txt python code/evaluate.py data/model.p data/matrix-test.p data/eval_auc.txt +dvc run -d data/eval_auc.txt +cat data/eval_auc.txt +# AUC: 0.596182 + +git status +git add . +git commit -m 'Full pipeline' + + +git checkout input_100K +dvc checkout +git merge -X ours input_25K # You can also merge and resolve conflicts by hands!!! + +dvc repro +git add . +git commit -m 'Repro' + +# Improve the model: +git checkout input_25K +dvc checkout +vi code/train_model.py # Change: estimators=500 +git commit -am 'estimators=500' +vi code/train_model.py # Change: n_jobs=6 +git commit -am 'n_jobs=6' +dvc repro +cat data/eval_auc.txt +# 0.619262 + +vi code/featurization.py # Change: add ngram_range=(1, 2) to CountVectorizer +dvc repro +git commit -am 'Add bigrams' +cat data/eval_auc.txt +# 0.628989 + +vi code/featurization.py +dvc repro +git commit -am 'Add three-grams' # Change: ngram_range=(1, 3) +cat data/eval_auc.txt +# 0.630682 + +vi code/featurization.py +dvc repro +git commit -am 'Add 4-grams' # Change: ngram_range=(1, 4) +cat data/eval_auc.txt +# 0.621002 + +# Create a branch for unsuccessful commit +git checkout -b four_grams +dvc checkout +# The last successful commit which is the second last humman-created commit +git log --pretty=oneline | grep -v "DVC repro-run: " | head -n 2 | tail -n 1 +# Move the current HEAD to the last successful commit +git branch -f input_25K `git log --pretty=oneline | grep -v "DVC repro-run: " | head -n 2 | tail -n 1 | cut -d" " -f1` +git checkout input_25K +dvc checkout + +# Continue to optimize the target metrics +vi code/train_model.py # Change: min_samples_split=5 +dvc repro +git commit -am 'min_samples_split=5' + +git checkout input_100K +dvc checkout +git merge -X theirs input_25K +dvc repro + From 3349ce85392a7bb0a306352d215a8b1deea2a5b6 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Thu, 18 Jan 2018 01:44:55 -0800 Subject: [PATCH 2/2] Clean up the branches tutorial --- docs/tutorial/branches.sh | 44 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/docs/tutorial/branches.sh b/docs/tutorial/branches.sh index cbb9dd353..7f6671604 100755 --- a/docs/tutorial/branches.sh +++ b/docs/tutorial/branches.sh @@ -2,11 +2,10 @@ set -e -# 1. First ML model - mkdir myrepo cd myrepo +# 1. Set up environment and get code git init echo "This is an empty readme" > README.md git add README.md @@ -19,30 +18,31 @@ wget -nv -P code/ https://s3-us-west-2.amazonaws.com/dvc-share/so/code/featuriza https://s3-us-west-2.amazonaws.com/dvc-share/so/code/split_train_test.py \ https://s3-us-west-2.amazonaws.com/dvc-share/so/code/xml_to_tsv.py \ https://s3-us-west-2.amazonaws.com/dvc-share/so/code/requirements.txt -#pip install -U -r code/requirements.txt -# Edit eval file format +pip install -U -r code/requirements.txt git add code/ git commit -m 'Download code' dvc init git commit -m 'Init DVC' +# 2. First steps in 100K dataset git checkout -b input_100K mkdir data -#wget https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz -O data/Posts.xml.tgz -cp ../stackoverflow_small_xml/100K/Posts.xml.tgz data/ +wget https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz -O data/Posts.xml.tgz +#cp ../stackoverflow_small_xml/100K/Posts.xml.tgz data/ dvc add data/Posts.xml.tgz dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/ dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python git add . git commit -m 'Extract TSV for 100K dataset' +# 3. Define full pipeline for 25K dataset git checkout master dvc checkout git checkout -b input_25K dvc checkout mkdir data -#wget https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz -O data/Posts.xml.tgz -cp ../stackoverflow_small_xml/25K/Posts.xml.tgz data/ +wget https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz -O data/Posts.xml.tgz +#cp ../stackoverflow_small_xml/25K/Posts.xml.tgz data/ dvc add data/Posts.xml.tgz dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/ dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python @@ -59,7 +59,7 @@ git status git add . git commit -m 'Full pipeline' - +# 4. Integrate the 25K dataset pipeline into 100K branch git checkout input_100K dvc checkout git merge -X ours input_25K # You can also merge and resolve conflicts by hands!!! @@ -68,14 +68,12 @@ dvc repro git add . git commit -m 'Repro' -# Improve the model: +# 5. Improve the 25K model git checkout input_25K dvc checkout -vi code/train_model.py # Change: estimators=500 -git commit -am 'estimators=500' -vi code/train_model.py # Change: n_jobs=6 -git commit -am 'n_jobs=6' +vi code/train_model.py # Change: estimators=500 and n_jobs=6 dvc repro +git commit -am 'estimators=500' cat data/eval_auc.txt # 0.619262 @@ -85,18 +83,19 @@ git commit -am 'Add bigrams' cat data/eval_auc.txt # 0.628989 -vi code/featurization.py +vi code/featurization.py # Change: ngram_range=(1, 3) dvc repro -git commit -am 'Add three-grams' # Change: ngram_range=(1, 3) +git commit -am 'Add three-grams' cat data/eval_auc.txt # 0.630682 -vi code/featurization.py +vi code/featurization.py # Change: ngram_range=(1, 4) dvc repro -git commit -am 'Add 4-grams' # Change: ngram_range=(1, 4) +git commit -am 'Add 4-grams' cat data/eval_auc.txt # 0.621002 +# 6. Revert the last step back # Create a branch for unsuccessful commit git checkout -b four_grams dvc checkout @@ -107,13 +106,12 @@ git branch -f input_25K `git log --pretty=oneline | grep -v "DVC repro-run: " | git checkout input_25K dvc checkout -# Continue to optimize the target metrics -vi code/train_model.py # Change: min_samples_split=5 -dvc repro -git commit -am 'min_samples_split=5' - +# 7. Integrate the 25K model improvements into the 100K model git checkout input_100K dvc checkout git merge -X theirs input_25K dvc repro +git commit -am 'Final model for 100K dataset' +cat data/eval_auc.txt +# 0.652154