mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 07:18:27 +01:00
Merge pull request #415 from efiop/master
Travis and removing 'dvc lock'
This commit is contained in:
-23
@@ -12,7 +12,6 @@ from dvc.command.remove import CmdRemove
|
||||
from dvc.command.run import CmdRun
|
||||
from dvc.command.repro import CmdRepro
|
||||
from dvc.command.data_sync import CmdDataPush, CmdDataPull, CmdDataStatus
|
||||
from dvc.command.lock import CmdLock
|
||||
from dvc.command.gc import CmdGC
|
||||
from dvc.command.add import CmdAdd
|
||||
from dvc.command.show_workflow import CmdShowWorkflow
|
||||
@@ -100,11 +99,6 @@ def parse_args(argv=None):
|
||||
action='append',
|
||||
default=[],
|
||||
help='Declare output regular file (sync to Git) for reproducible cmd.')
|
||||
run_parser.add_argument('-l',
|
||||
'--lock',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Lock data item - disable reproduction.')
|
||||
run_parser.add_argument('-f',
|
||||
'--file',
|
||||
help='Specify name of the state file')
|
||||
@@ -197,23 +191,6 @@ def parse_args(argv=None):
|
||||
help='Input files/directories')
|
||||
import_parser.set_defaults(func=CmdAdd)
|
||||
|
||||
# Lock
|
||||
lock_parser = subparsers.add_parser(
|
||||
'lock',
|
||||
parents=[parent_parser],
|
||||
help='Lock')
|
||||
lock_parser.add_argument('-u',
|
||||
'--unlock',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Unlock stage - enable reproduction.')
|
||||
lock_parser.add_argument(
|
||||
'files',
|
||||
nargs='*',
|
||||
default=[Stage.STAGE_FILE],
|
||||
help='Stages to lock or unlock.')
|
||||
lock_parser.set_defaults(func=CmdLock)
|
||||
|
||||
# Garbage collector
|
||||
gc_parser = subparsers.add_parser(
|
||||
'gc',
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
from dvc.command.common.base import CmdBase
|
||||
from dvc.stage import Stage
|
||||
|
||||
|
||||
class CmdLock(CmdBase):
|
||||
def run(self):
|
||||
lock = not self.args.unlock
|
||||
cmd = 'lock' if lock else 'unlock'
|
||||
ret = 0
|
||||
for file in self.args.files:
|
||||
try:
|
||||
stage = Stage.load(self.project, file)
|
||||
|
||||
if stage.locked and lock:
|
||||
self.project.logger.warn('Stage {} is already locked'.format(file))
|
||||
elif not stage.locked and not lock:
|
||||
self.project.logger.warn('Stage {} is already unlocked'.format(file))
|
||||
else:
|
||||
stage.locked = lock
|
||||
self.project.logger.debug('Saving stage file {}'.format(file))
|
||||
stage.dump()
|
||||
self.project.logger.info('Stage {} was {}ed'.format(file, cmd))
|
||||
except Exception as ex:
|
||||
ret = 1
|
||||
self.project.logger.error('Unable to {} {}: {}'.format(cmd, file, ex))
|
||||
|
||||
return ret
|
||||
@@ -15,7 +15,6 @@ class CmdRun(CmdBase):
|
||||
outs=self.args.outs,
|
||||
outs_no_cache=self.args.outs_no_cache,
|
||||
deps=self.args.deps,
|
||||
locked=self.args.lock,
|
||||
fname=fname,
|
||||
cwd=self.args.cwd,
|
||||
no_exec=self.args.no_exec)
|
||||
|
||||
+2
-5
@@ -125,8 +125,7 @@ class Project(object):
|
||||
cmd=None,
|
||||
cwd=cwd,
|
||||
outs=outputs,
|
||||
deps=[],
|
||||
locked=True)
|
||||
deps=[])
|
||||
stage.save()
|
||||
stage.dump()
|
||||
return stage
|
||||
@@ -152,7 +151,6 @@ class Project(object):
|
||||
deps=[],
|
||||
outs=[],
|
||||
outs_no_cache=[],
|
||||
locked=False,
|
||||
fname=Stage.STAGE_FILE,
|
||||
cwd=os.curdir,
|
||||
no_exec=False):
|
||||
@@ -167,8 +165,7 @@ class Project(object):
|
||||
cmd=cmd,
|
||||
cwd=cwd,
|
||||
outs=outputs,
|
||||
deps=deps,
|
||||
locked=locked)
|
||||
deps=deps)
|
||||
if not no_exec:
|
||||
stage.run()
|
||||
stage.dump()
|
||||
|
||||
+2
-7
@@ -250,16 +250,14 @@ class Stage(object):
|
||||
PARAM_CMD = 'cmd'
|
||||
PARAM_DEPS = 'deps'
|
||||
PARAM_OUTS = 'outs'
|
||||
PARAM_LOCKED = 'locked'
|
||||
|
||||
def __init__(self, project, path=None, cmd=None, cwd=None, deps=[], outs=[], locked=False):
|
||||
def __init__(self, project, path=None, cmd=None, cwd=None, deps=[], outs=[]):
|
||||
self.project = project
|
||||
self.path = path
|
||||
self.cmd = cmd
|
||||
self.cwd = cwd
|
||||
self.outs = outs
|
||||
self.deps = deps
|
||||
self.locked = locked
|
||||
|
||||
@property
|
||||
def relpath(self):
|
||||
@@ -315,15 +313,13 @@ class Stage(object):
|
||||
cmd = d[Stage.PARAM_CMD]
|
||||
deps = Dependency.loadd_from(project, d[Stage.PARAM_DEPS], cwd=cwd)
|
||||
outs = Output.loadd_from(project, d[Stage.PARAM_OUTS], cwd=cwd)
|
||||
locked = d[Stage.PARAM_LOCKED]
|
||||
|
||||
return Stage(project=project,
|
||||
path=path,
|
||||
cmd=cmd,
|
||||
cwd=cwd,
|
||||
deps=deps,
|
||||
outs=outs,
|
||||
locked=locked)
|
||||
outs=outs)
|
||||
|
||||
@staticmethod
|
||||
def load(project, fname):
|
||||
@@ -338,7 +334,6 @@ class Stage(object):
|
||||
Stage.PARAM_CMD: self.cmd,
|
||||
Stage.PARAM_DEPS: deps,
|
||||
Stage.PARAM_OUTS: outs,
|
||||
Stage.PARAM_LOCKED: self.locked
|
||||
}
|
||||
|
||||
def dump(self, fname=None):
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
source common.sh
|
||||
|
||||
rm -rf $CACHE_DIR $TEST_REPO
|
||||
@@ -1,161 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
SDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
CACHE_DIR=$SDIR/cache
|
||||
TEST_REPO=$SDIR/myrepo
|
||||
CODE_CACHE=$CACHE_DIR/code
|
||||
DATA_CACHE=$CACHE_DIR/data
|
||||
|
||||
DATA="foo bar"
|
||||
CODE="code.sh"
|
||||
|
||||
REGION='us-west-2'
|
||||
BASE_DIR_S3=https://s3-$REGION.amazonaws.com/dataversioncontrol/functests
|
||||
DATA_S3=$BASE_DIR_S3/dummy_data
|
||||
CODE_S3=$BASE_DIR_S3/dummy_code
|
||||
|
||||
TEST_REPO_S3="dvc-test/$(uuidgen)"
|
||||
TEST_REPO_REGION='us-east-2'
|
||||
|
||||
TEST_REPO_GCP='dvc-test/myrepo'
|
||||
TEST_REPO_GCP_PROJECT='dvc-project'
|
||||
|
||||
function is_osx() {
|
||||
if [ "$(uname)" == "Darwin" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function dvc_md5() {
|
||||
if is_osx; then
|
||||
md5 -r $@
|
||||
else
|
||||
md5sum $@
|
||||
fi
|
||||
}
|
||||
|
||||
function dvc_timestamp() {
|
||||
if is_osx; then
|
||||
stat -f "%m%t%Sm %N" $@
|
||||
else
|
||||
stat -c %y data/foo $@
|
||||
fi
|
||||
}
|
||||
|
||||
function dvc_print() {
|
||||
echo -e "$1"
|
||||
}
|
||||
|
||||
function dvc_info() {
|
||||
dvc_print "\e[34m$1\e[0m"
|
||||
}
|
||||
|
||||
function dvc_header() {
|
||||
dvc_info "=============== DVC test: $1 ==============="
|
||||
}
|
||||
|
||||
function dvc_error() {
|
||||
dvc_print "\e[31m$1\e[0m"
|
||||
}
|
||||
|
||||
function dvc_fail() {
|
||||
dvc_error "FAIL"
|
||||
exit 1
|
||||
}
|
||||
|
||||
function dvc_fatal() {
|
||||
dvc_error "$1"
|
||||
dvc_fail
|
||||
}
|
||||
|
||||
function dvc_pass() {
|
||||
dvc_print "\e[32mPASS\e[0m"
|
||||
exit 0
|
||||
}
|
||||
|
||||
trap 'dvc_fail;' ERR
|
||||
|
||||
function dvc_prepare_cache() {
|
||||
if [ -d $CACHE_DIR ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
mkdir -p $DATA_CACHE
|
||||
pushd $DATA_CACHE
|
||||
|
||||
for f in $DATA; do
|
||||
wget $DATA_S3/$f
|
||||
done
|
||||
|
||||
popd
|
||||
|
||||
mkdir -p $CODE_CACHE
|
||||
pushd $CODE_CACHE
|
||||
|
||||
for f in $CODE; do
|
||||
wget $CODE_S3/$f
|
||||
done
|
||||
|
||||
popd
|
||||
}
|
||||
|
||||
function dvc_create_git_repo() {
|
||||
dvc_info "Creating git repo"
|
||||
|
||||
dvc_prepare_cache
|
||||
|
||||
rm -rf $TEST_REPO
|
||||
mkdir $TEST_REPO
|
||||
cd $TEST_REPO
|
||||
|
||||
git init
|
||||
|
||||
git config user.name "DVC tester"
|
||||
git config user.email "[email protected]"
|
||||
|
||||
mkdir code
|
||||
cp $CODE_CACHE/* code/
|
||||
git add code/
|
||||
git commit -m 'Add code'
|
||||
}
|
||||
|
||||
function dvc_create_repo() {
|
||||
dvc_info "Creating dvc repo"
|
||||
|
||||
dvc_create_git_repo
|
||||
|
||||
dvc init
|
||||
|
||||
mkdir data
|
||||
cp $DATA_CACHE/* data
|
||||
}
|
||||
|
||||
function dvc_clean_cloud_aws() {
|
||||
dvc_info "Cleaning $TEST_REPO_S3"
|
||||
aws s3 rm --recursive "s3://$TEST_REPO_S3/"
|
||||
}
|
||||
|
||||
function dvc_clean_cloud_gcp() {
|
||||
dvc_info "Cleaning $TEST_REPO_GCP_PROJECT/$TEST_REPO_GCP"
|
||||
gsutil rm -rf "gs://$TEST_REPO_GCP/" || true
|
||||
}
|
||||
|
||||
function dvc_check_dirs() {
|
||||
for d in $1; do
|
||||
if [ ! -d "$d" ]; then
|
||||
dvc_fatal "Directory $d doesn't exist"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function dvc_check_files() {
|
||||
for f in $1; do
|
||||
if [ ! -f "$f" ]; then
|
||||
dvc_fatal "File $f doesn't exist"
|
||||
fi
|
||||
done
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
dvc
|
||||
pandas
|
||||
sklearn
|
||||
scipy
|
||||
configobj
|
||||
networkx
|
||||
@@ -1,21 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
source common.sh
|
||||
|
||||
TESTS+=" test-init.sh"
|
||||
TESTS+=" test-add-remove.sh"
|
||||
TESTS+=" test-sync.sh"
|
||||
TESTS+=" test-repro-code.sh"
|
||||
TESTS+=" test-checkout.sh"
|
||||
TESTS+=" test-gc.sh"
|
||||
TESTS+=" test-lock.sh"
|
||||
|
||||
for t in $TESTS; do
|
||||
rm -rf $TEST_REPO
|
||||
dvc_header "$t"
|
||||
./$t
|
||||
done
|
||||
|
||||
rm -rf $TEST_REPO
|
||||
@@ -1,22 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
source common.sh
|
||||
|
||||
dvc_create_repo
|
||||
|
||||
dvc add data/foo
|
||||
dvc_check_files data/foo.dvc data/foo
|
||||
|
||||
dvc remove data/foo
|
||||
if [ -f "data/foo" ]; then
|
||||
echo "data/foo was not removed"
|
||||
dvc_fail
|
||||
fi
|
||||
if [ -f "data/foo.dvc" ]; then
|
||||
echo "data/foo.dvc was not removed"
|
||||
dvc_fail
|
||||
fi
|
||||
|
||||
dvc_pass
|
||||
@@ -1,14 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
source common.sh
|
||||
|
||||
dvc_create_repo
|
||||
|
||||
dvc add data/foo
|
||||
|
||||
rm -f data/foo
|
||||
dvc checkout
|
||||
dvc_check_files data/foo
|
||||
dvc_pass
|
||||
@@ -1,39 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
source common.sh
|
||||
|
||||
dvc_create_repo
|
||||
|
||||
ORIG_CACHE=$(ls .dvc/cache)
|
||||
|
||||
DUMMY_CACHE+=" .dvc/cache/1"
|
||||
DUMMY_CACHE+=" .dvc/cache/2"
|
||||
DUMMY_CACHE+=" .dvc/cache/3"
|
||||
|
||||
dvc_info "Create dummy cache files"
|
||||
touch $DUMMY_CACHE
|
||||
if [ $? -ne 0 ]; then
|
||||
dvc_error "Failed to create cache files"
|
||||
dvc_fail
|
||||
fi
|
||||
|
||||
dvc_info "Run 'dvc gc'"
|
||||
dvc gc
|
||||
|
||||
dvc_info "Check that we didn't break good cache"
|
||||
dvc_check_dirs ".dvc/cache"
|
||||
pushd .dvc/cache
|
||||
dvc_check_files $ORIG_CACHE
|
||||
popd
|
||||
|
||||
dvc_info "Check that dummy cache files were removed"
|
||||
for d in $DUMMY_CACHE; do
|
||||
if [ -f $d ]; then
|
||||
dvc_error "Dummy cache file $d was not removed by gc"
|
||||
dvc_fail
|
||||
fi
|
||||
done
|
||||
|
||||
dvc_pass
|
||||
@@ -1,17 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
source common.sh
|
||||
|
||||
dvc_create_git_repo
|
||||
|
||||
dvc init
|
||||
|
||||
DIRS=".dvc/cache .dvc/state"
|
||||
FILES=".dvc/config"
|
||||
|
||||
dvc_check_dirs $DIRS
|
||||
dvc_check_files $FILES
|
||||
|
||||
dvc_pass
|
||||
@@ -1,16 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
source common.sh
|
||||
|
||||
dvc_create_repo
|
||||
|
||||
dvc add data/foo
|
||||
dvc run -f Dvcfile -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
|
||||
cat Dvcfile | grep 'locked' | grep 'false' || dvc_fatal "stage locked after 'dvc run'"
|
||||
dvc lock
|
||||
cat Dvcfile | grep 'locked' | grep 'true' || dvc_fatal "stage not locked after 'dvc lock'"
|
||||
dvc lock --unlock
|
||||
cat Dvcfile | grep 'locked' | grep 'false' || dvc_fatal "stage locked after 'dvc lock --unlock'"
|
||||
dvc_pass
|
||||
@@ -1,95 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
source common.sh
|
||||
|
||||
function test_generic() {
|
||||
dvc_info '=== Generic repro test'
|
||||
|
||||
dvc_create_repo
|
||||
|
||||
dvc_info 'Copy foo into foo1'
|
||||
dvc add data/foo
|
||||
dvc run -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
|
||||
|
||||
dvc_info 'Modify code'
|
||||
echo " " >> code/code.sh
|
||||
git commit -am 'Change code'
|
||||
|
||||
dvc_info 'Reproduce foo1'
|
||||
dvc repro foo1.dvc
|
||||
dvc_check_files data/foo1 data/foo1.dvc
|
||||
if [ "$(cat data/foo1)" != "foo" ]; then
|
||||
dvc_fail
|
||||
fi
|
||||
|
||||
dvc_info 'Modify foo'
|
||||
rm -f data/foo
|
||||
cp $DATA_CACHE/bar data/foo
|
||||
|
||||
dvc_info 'Reproduce foo1 as default target'
|
||||
dvc repro foo1.dvc
|
||||
dvc_check_files data/foo1 data/foo1.dvc
|
||||
if [ "$(cat data/foo1)" != "bar" ]; then
|
||||
dvc_fail
|
||||
fi
|
||||
}
|
||||
|
||||
function test_partial() {
|
||||
dvc_info "=== Partial repro test"
|
||||
|
||||
dvc_create_repo
|
||||
|
||||
cp code/code.sh code/code1.sh
|
||||
cp code/code1.sh code/code2.sh
|
||||
cp code/code2.sh code/code3.sh
|
||||
git add code
|
||||
git commit -m "copy code"
|
||||
|
||||
dvc_info "Create repro chain foo -> foo1 -> foo2 -> foo3"
|
||||
dvc add data/foo
|
||||
dvc run -f copy_foo_foo1.dvc -d code/code1.sh -d data/foo -o data/foo1 cp data/foo data/foo1
|
||||
dvc run -f copy_foo1_foo2.dvc -d code/code2.sh -d data/foo1 -o data/foo2 cp data/foo1 data/foo2
|
||||
dvc run -f copy_foo2_foo3.dvc -d code/code3.sh -d data/foo2 -o data/foo3 cp data/foo2 data/foo3
|
||||
|
||||
dvc_info "Save original timestamps"
|
||||
FOO_TS=$(dvc_timestamp data/foo)
|
||||
FOO1_TS=$(dvc_timestamp data/foo1)
|
||||
FOO2_TS=$(dvc_timestamp data/foo2)
|
||||
FOO3_TS=$(dvc_timestamp data/foo3)
|
||||
|
||||
dvc_info "Modify dependency for data/foo1, that will not result in data/foo1 contents being changed"
|
||||
echo " " >> code/code1.sh
|
||||
git add code
|
||||
git commit -m "modify code1.sh"
|
||||
|
||||
dvc_info "Reproduce data/foo3"
|
||||
dvc repro copy_foo2_foo3.dvc
|
||||
|
||||
dvc_info "Check timestamps"
|
||||
if [ "$FOO_TS" != "$(dvc_timestamp data/foo)" ]; then
|
||||
dvc_error "data/foo timestamp changed"
|
||||
dvc_fail
|
||||
fi
|
||||
|
||||
if [ "$FOO1_TS" != "$(dvc_timestamp data/foo1)" ]; then
|
||||
dvc_error "data/foo1 timestamp changed"
|
||||
dvc_fail
|
||||
fi
|
||||
|
||||
if [ "$FOO2_TS" != "$(dvc_timestamp data/foo2)" ]; then
|
||||
dvc_error "data/foo2 timestamp changed"
|
||||
dvc_fail
|
||||
fi
|
||||
|
||||
if [ "$FOO3_TS" != "$(dvc_timestamp data/foo3)" ]; then
|
||||
dvc_error "data/foo3 timestamp changed"
|
||||
dvc_fail
|
||||
fi
|
||||
}
|
||||
|
||||
test_generic
|
||||
test_partial
|
||||
|
||||
dvc_pass
|
||||
@@ -1,71 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
source common.sh
|
||||
|
||||
function test_sync() {
|
||||
dvc add data/foo
|
||||
dvc run -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
|
||||
|
||||
dvc_info "Checking status"
|
||||
dvc status | grep "new file" || dvc_fail
|
||||
|
||||
dvc_info "Pushing data"
|
||||
dvc push
|
||||
|
||||
dvc_info "Checking status"
|
||||
dvc status
|
||||
|
||||
dvc_info "Removing all cache"
|
||||
rm -rf .dvc/cache/*
|
||||
rm -rf data/foo data/bar
|
||||
|
||||
dvc_info "Checking status"
|
||||
dvc status | grep "deleted" || dvc_fail
|
||||
|
||||
dvc_info "Pulling data"
|
||||
dvc pull
|
||||
dvc_check_files data/foo data/bar
|
||||
|
||||
dvc_info "Checking status"
|
||||
dvc status
|
||||
}
|
||||
|
||||
function test_aws() {
|
||||
dvc_create_repo
|
||||
|
||||
dvc_info "Setting up AWS cloud"
|
||||
dvc_clean_cloud_aws
|
||||
dvc config aws.storagepath $TEST_REPO_S3
|
||||
dvc config aws.region $TEST_REPO_REGION
|
||||
dvc config global.cloud AWS
|
||||
|
||||
test_sync
|
||||
|
||||
dvc_clean_cloud_aws
|
||||
}
|
||||
|
||||
function test_gcp() {
|
||||
dvc_create_repo
|
||||
|
||||
dvc_info "Setting up GCP cloud"
|
||||
dvc_clean_cloud_gcp
|
||||
dvc config gcp.storagepath $TEST_REPO_GCP
|
||||
dvc config gcp.projectname $TEST_REPO_GCP_PROJECT
|
||||
dvc config global.cloud GCP
|
||||
|
||||
test_sync
|
||||
|
||||
dvc_clean_cloud_gcp
|
||||
}
|
||||
|
||||
if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then
|
||||
test_aws
|
||||
# Temporarily disabled for travis
|
||||
if [[ "$TRAVIS" != "true" ]]; then
|
||||
test_gcp
|
||||
fi
|
||||
fi
|
||||
|
||||
dvc_pass
|
||||
@@ -1,146 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
# 1. First ML model
|
||||
|
||||
mkdir myrepo
|
||||
cd myrepo
|
||||
|
||||
git init
|
||||
echo "This is an empty readme" > README.md
|
||||
git add README.md
|
||||
git commit -m 'add readme'
|
||||
|
||||
mkdir code
|
||||
wget -nv -P code/ https://s3-us-west-2.amazonaws.com/dvc-share/so/code/featurization.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/evaluate.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/train_model.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/split_train_test.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/xml_to_tsv.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/requirements.txt
|
||||
#pip install -U -r code/requirements.txt
|
||||
# Edit eval file format
|
||||
git add code/
|
||||
git commit -m 'Download code'
|
||||
|
||||
dvc init
|
||||
|
||||
git checkout -b input_100K
|
||||
mkdir data
|
||||
wget https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz -O data/Posts.xml.tgz
|
||||
#cp ../stackoverflow_small_xml/100K/Posts.xml.tgz data/
|
||||
dvc add data/Posts.xml.tgz
|
||||
dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/
|
||||
dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python
|
||||
git add .
|
||||
git commit -m 'Extract TSV for 100K dataset'
|
||||
|
||||
git checkout master
|
||||
dvc checkout
|
||||
git checkout -b input_25K
|
||||
dvc checkout
|
||||
mkdir data
|
||||
wget https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz -O data/Posts.xml.tgz
|
||||
#cp ../stackoverflow_small_xml/25K/Posts.xml.tgz data/
|
||||
dvc add data/Posts.xml.tgz
|
||||
dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/
|
||||
|
||||
dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python
|
||||
dvc run -d code/split_train_test.py -d data/Posts.tsv -o data/Posts-train.tsv -o data/Posts-test.tsv python code/split_train_test.py data/Posts.tsv 0.33 20170426 data/Posts-train.tsv data/Posts-test.tsv
|
||||
dvc run -d code/featurization.py -d data/Posts-train.tsv -d data/Posts-test.tsv -o data/matrix-train.p -o data/matrix-test.p python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p
|
||||
|
||||
dvc run -d code/train_model.py -d data/matrix-train.p -o data/model.p python code/train_model.py data/matrix-train.p 20170426 data/model.p
|
||||
|
||||
dvc run -d code/evaluate.py -d data/model.p -d data/matrix-test.p -O data/eval_auc.txt python code/evaluate.py data/model.p data/matrix-test.p data/eval_auc.txt
|
||||
|
||||
#dvc config Global.Target data/eval_auc.txt
|
||||
#git commit -am 'Set Target'
|
||||
|
||||
cat data/eval_auc.txt
|
||||
# AUC: 0.596182
|
||||
|
||||
git add .
|
||||
git commit -m 'Full pipeline'
|
||||
|
||||
|
||||
git checkout input_100K
|
||||
dvc checkout
|
||||
git merge -X theirs input_25K # git merge -X theirs input_25K
|
||||
## Resolve conflicts by hands!!!
|
||||
#git add .
|
||||
#git commit -m 'Merge conflicts'
|
||||
dvc merge
|
||||
dvc repro # <-- Error. Reproducng Posts.xml and Posts.tsv
|
||||
|
||||
|
||||
# Improve the model:
|
||||
git checkout input_25K
|
||||
dvc checkout
|
||||
vi code/train_model.py # Change: estimators=500
|
||||
git commit -am 'estimators=500'
|
||||
vi code/train_model.py # Change: n_jobs=6
|
||||
git commit -am 'n_jobs=6'
|
||||
dvc repro
|
||||
cat data/eval_auc.txt
|
||||
# 0.619262
|
||||
vi code/featurization.py # Change: add ngram_range=(1, 2) to CountVectorizer
|
||||
git commit -am 'Add bigrams'
|
||||
dvc repro
|
||||
cat data/eval_auc.txt
|
||||
# 0.628989
|
||||
vi code/featurization.py
|
||||
git commit -am 'Add three-grams' # Change: ngram_range=(1, 3)
|
||||
dvc repro
|
||||
cat data/eval_auc.txt
|
||||
# 0.630682
|
||||
vi code/featurization.py
|
||||
git commit -am 'Add 4-grams' # Change: ngram_range=(1, 4)
|
||||
dvc repro
|
||||
cat data/eval_auc.txt
|
||||
# 0.621002
|
||||
|
||||
# Create a branch for unsuccessful commit
|
||||
git checkout -b four_grams
|
||||
dvc checkout
|
||||
# The last successful commit which is the second last humman-created commit
|
||||
git log --pretty=oneline | grep -v "DVC repro-run: " | head -n 2 | tail -n 1
|
||||
# Move the current HEAD to the last successful commit
|
||||
git branch -f input_25K `git log --pretty=oneline | grep -v "DVC repro-run: " | head -n 2 | tail -n 1 | cut -d" " -f1`
|
||||
git checkout input_25K
|
||||
dvc checkout
|
||||
|
||||
# Continue to optimize the target metrics
|
||||
vi code/train_model.py # Change: min_samples_split=5
|
||||
git commit -am 'min_samples_split=5'
|
||||
dvc repro
|
||||
|
||||
git checkout input_100K
|
||||
dvc checkout
|
||||
git merge -X theirs input_25K
|
||||
git add data/
|
||||
git commit -am "After merge"
|
||||
dvc repro
|
||||
|
||||
##
|
||||
|
||||
vi code/featurization.py
|
||||
# 0.578447
|
||||
|
||||
# 2. Reproduce: change input dataset
|
||||
|
||||
dvc remove data/Posts.xml.tgz
|
||||
#dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz data/
|
||||
dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz data/
|
||||
dvc repro data/evaluation.txt
|
||||
cat data/evaluation.txt
|
||||
# AUC: 0.639861
|
||||
|
||||
# 3. Share your research
|
||||
|
||||
# Improve features
|
||||
echo " " >> code/featurization.py
|
||||
git add code/featurization.py
|
||||
git commit -m 'Include bigram'
|
||||
dvc repro data/evaluation.txt
|
||||
cat data/evaluation.txt
|
||||
@@ -1,61 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
# 1. First ML model
|
||||
|
||||
rm -rf myrepo
|
||||
mkdir myrepo
|
||||
cd myrepo
|
||||
mkdir data
|
||||
mkdir code
|
||||
wget -nv -P code/ https://s3-us-west-2.amazonaws.com/dvc-share/so/code/featurization.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/evaluate.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/train_model.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/split_train_test.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/xml_to_tsv.py \
|
||||
https://s3-us-west-2.amazonaws.com/dvc-share/so/code/requirements.txt
|
||||
pip install -U -r code/requirements.txt
|
||||
|
||||
git init
|
||||
git add code/
|
||||
git commit -m 'Download code'
|
||||
|
||||
|
||||
dvc init
|
||||
wget -P data/ https://s3-us-west-2.amazonaws.com/dvc-share/so/10K/Posts.xml.tgz
|
||||
dvc add data/Posts.xml.tgz
|
||||
dvc run -d data/Posts.xml.tgz -o data/Posts.xml tar zxf data/Posts.xml.tgz -C data/
|
||||
|
||||
dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python
|
||||
dvc run -d code/split_train_test.py -d data/Posts.tsv -o data/Posts-test.tsv python code/split_train_test.py data/Posts.tsv 0.33 20170426 data/Posts-train.tsv data/Posts-test.tsv
|
||||
dvc run -d code/featurization.py -d data/Posts-train.tsv -d data/Posts-test.tsv -o data/matrix-train.p -o data/matrix-test.p python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p
|
||||
|
||||
dvc run -d code/train_model.py -d data/matrix-train.p -o data/model.p python code/train_model.py data/matrix-train.p 20170426 data/model.p
|
||||
|
||||
dvc run -d code/evaluate.py -d data/model.p -d data/matrix-test.p -o data/evaluation.txt python code/evaluate.py data/model.p data/matrix-test.p data/evaluation.txt
|
||||
|
||||
cat data/evaluation.txt
|
||||
# AUC: 0.552980
|
||||
|
||||
exit 0
|
||||
|
||||
# 2. Reproduce: change input dataset
|
||||
|
||||
dvc remove data/Posts.xml.tgz
|
||||
#dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz data/
|
||||
|
||||
wget -P data/ https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz
|
||||
dvc add data/Posts.xml.tgz
|
||||
dvc repro -v data/evaluation.txt
|
||||
cat data/evaluation.txt
|
||||
# AUC: 0.639861
|
||||
|
||||
# 3. Share your research
|
||||
|
||||
# Improve features
|
||||
echo " " >> code/featurization.py
|
||||
git add code/featurization.py
|
||||
git commit -m 'Include bigram'
|
||||
dvc repro data/evaluation.txt
|
||||
cat data/evaluation.txt
|
||||
@@ -1,11 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
BIN=$DIR/bin
|
||||
DVC_HOME=$DIR
|
||||
|
||||
export PATH=$BIN:$PATH
|
||||
export DVC_HOME=$DIR
|
||||
|
||||
cd functest
|
||||
./run.sh
|
||||
@@ -5,7 +5,8 @@ set -e
|
||||
|
||||
|
||||
if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
|
||||
brew install graphviz
|
||||
wget https://s3-us-west-2.amazonaws.com/dataversioncontrol/graphviz-2.38.0.pkg
|
||||
sudo installer -pkg graphviz-2.38.0.pkg -target /
|
||||
else
|
||||
sudo apt-get install graphviz
|
||||
fi
|
||||
|
||||
@@ -4,7 +4,6 @@ set -x
|
||||
set -e
|
||||
|
||||
./unittests.sh
|
||||
./functests.sh
|
||||
|
||||
if [[ ! -z "$TRAVIS_TAG" ]]; then
|
||||
if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
|
||||
|
||||
@@ -23,7 +23,6 @@ class TestAdd(TestDvc):
|
||||
self.assertTrue(os.path.isfile(stage.path))
|
||||
self.assertEqual(len(stage.outs), 1)
|
||||
self.assertEqual(len(stage.deps), 0)
|
||||
self.assertTrue(stage.locked)
|
||||
self.assertEqual(stage.cmd, None)
|
||||
self.assertEqual(stage.outs[0].md5, md5)
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ from dvc.command.data_sync import CmdDataPull, CmdDataPush, CmdDataStatus
|
||||
from dvc.command.repro import CmdRepro
|
||||
from dvc.command.add import CmdAdd
|
||||
from dvc.command.remove import CmdRemove
|
||||
from dvc.command.lock import CmdLock
|
||||
from dvc.command.gc import CmdGC
|
||||
from dvc.command.config import CmdConfig
|
||||
from dvc.command.show_pipeline import CmdShowPipeline
|
||||
@@ -49,7 +48,6 @@ class TestRun(TestDvc):
|
||||
'--outs', out2,
|
||||
'-O', out_no_cache1,
|
||||
'--outs-no-cache', out_no_cache2,
|
||||
'-l', '--lock',
|
||||
'-f', fname,
|
||||
'--file', fname,
|
||||
'-c', cwd,
|
||||
@@ -125,21 +123,6 @@ class TestAdd(TestDvc):
|
||||
self.assertEqual(args.targets, [target1, target2])
|
||||
|
||||
|
||||
class TestLock(TestDvc):
|
||||
def test(self):
|
||||
target1 = '1'
|
||||
target2 = '2'
|
||||
|
||||
args = parse_args(['lock',
|
||||
target1,
|
||||
target2,
|
||||
'-u', '--unlock'])
|
||||
|
||||
self.assertIsInstance(args.func(args), CmdLock)
|
||||
self.assertEqual(args.files, [target1, target2])
|
||||
self.assertEqual(args.unlock, True)
|
||||
|
||||
|
||||
class TestGC(TestDvc):
|
||||
def test(self):
|
||||
args = parse_args(['gc'])
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
from dvc.main import main
|
||||
from dvc.stage import Stage
|
||||
|
||||
from tests.test_repro import TestRepro
|
||||
|
||||
|
||||
class TestCmdLock(TestRepro):
|
||||
def test_lock(self):
|
||||
ret = main(['lock',
|
||||
self.file1_stage])
|
||||
stage = Stage.load(self.dvc, self.file1_stage)
|
||||
self.assertEqual(ret, 0)
|
||||
self.assertTrue(stage.locked)
|
||||
|
||||
def test_unlock(self):
|
||||
ret = main(['lock',
|
||||
'-u',
|
||||
self.file1_stage])
|
||||
stage = Stage.load(self.dvc, self.file1_stage)
|
||||
self.assertEqual(ret, 0)
|
||||
self.assertFalse(stage.locked)
|
||||
|
||||
def test_twice(self):
|
||||
self.test_lock()
|
||||
self.test_lock()
|
||||
|
||||
self.test_unlock()
|
||||
self.test_unlock()
|
||||
|
||||
def test_non_existing(self):
|
||||
ret = main(['lock',
|
||||
'non-existing-file'])
|
||||
self.assertNotEqual(ret, 0)
|
||||
@@ -15,7 +15,6 @@ class TestRun(TestDvc):
|
||||
deps = [self.FOO, self.CODE]
|
||||
outs = [os.path.join(self.dvc.root_dir, 'out')]
|
||||
outs_no_cache = []
|
||||
locked = False
|
||||
fname = os.path.join(self.dvc.root_dir, 'out.dvc')
|
||||
cwd = os.curdir
|
||||
|
||||
@@ -24,7 +23,6 @@ class TestRun(TestDvc):
|
||||
deps=deps,
|
||||
outs=outs,
|
||||
outs_no_cache=outs_no_cache,
|
||||
locked=locked,
|
||||
fname=fname,
|
||||
cwd=cwd)
|
||||
|
||||
@@ -35,7 +33,6 @@ class TestRun(TestDvc):
|
||||
self.assertEqual(len(stage.outs), len(outs + outs_no_cache))
|
||||
self.assertEqual(stage.outs[0].path, outs[0])
|
||||
self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0])
|
||||
self.assertEqual(stage.locked, locked)
|
||||
self.assertTrue(stage.path, fname)
|
||||
|
||||
|
||||
@@ -45,7 +42,6 @@ class TestRunEmpty(TestDvc):
|
||||
deps=[],
|
||||
outs=[],
|
||||
outs_no_cache=[],
|
||||
locked=False,
|
||||
fname='empty.dvc',
|
||||
cwd=os.curdir)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user