Merge pull request #307 from efiop/master

Func tests and RFC

dvc/path/data_item.py

@@ -20,9 +20,9 @@ class DataDirError(DvcException):
         super(DataDirError, self).__init__(msg)
 
 
-class DataItemInStatusDirError(DataDirError):
+class DataItemInConfigDirError(DataDirError):
     def __init__(self, file):
-        msg = 'File "{}" is in state directory'.format(file)
+        msg = 'File "{}" is in config directory'.format(file)
         super(DataItemInStatusDirError, self).__init__(msg)
 
 
@@ -48,10 +48,8 @@ class DataItem(object):
         if not self._data.abs.startswith(self._git.git_dir_abs):
             raise NotInGitDirError(data_file, self._git.git_dir_abs)
 
-        if self._data.abs.startswith(self.state_dir_abs):
-            raise DataItemInStatusDirError(data_file)
-
-        pass
+        if self._data.abs.startswith(os.path.join(self._git.git_dir_abs, self._config.CONFIG_DIR)):
+            raise DataItemInConfigDirError(data_file)
 
     def copy(self, cache_file=None):
         if not cache_file:

dvc/repository_change.py

@@ -4,7 +4,7 @@ from dvc.exceptions import DvcException
 from dvc.git_wrapper import GitWrapper
 from dvc.executor import Executor
 from dvc.logger import Logger
-from dvc.path.data_item import DataDirError, DataItemInStatusDirError
+from dvc.path.data_item import DataDirError, DataItemInConfigDirError
 from dvc.path.stated_data_item import StatedDataItem
 from dvc.utils import cached_property
 from dvc.config import ConfigI
@@ -74,7 +74,7 @@ class RepositoryChange(object):
             Logger.debug('[Repository change] Add status: {} {}'.format(
                          item.status,
                          item.data.dvc))
-        except DataItemInStatusDirError:
+        except DataItemInConfigDirError:
             self._created_status_files.append(file)
         except DataDirError:
             self._externally_created_files.append(file)

functest/common.sh

@@ -22,14 +22,30 @@ TEST_REPO_REGION='us-east-2'
 TEST_REPO_GCP='dvc-test/myrepo'
 TEST_REPO_GCP_PROJECT='dvc-project'
 
-function dvc_md5() {
+function is_osx() {
 	if [ "$(uname)" == "Darwin" ]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
+function dvc_md5() {
+	if is_osx; then
 		md5 -r $@
 	else
 		md5sum $@
 	fi
 }
 
+function dvc_timestamp() {
+	if is_osx; then
+		stat -f "%m%t%Sm %N" $@
+	else
+		stat -c %y data/foo $@
+	fi
+}
+
 function dvc_print() {
 	echo -e "$1"
 }

functest/run.sh

@@ -10,6 +10,7 @@ TESTS+=" test-sync.sh"
 TESTS+=" test-repro-code.sh"
 TESTS+=" test-merge.sh"
 TESTS+=" test-checkout.sh"
+TESTS+=" test-gc.sh"
 
 for t in $TESTS; do
 	rm -rf $TEST_REPO

functest/test-gc.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -e
+
+source common.sh
+
+dvc_create_repo
+
+ORIG_CACHE=$(ls .dvc/cache)
+
+DUMMY_CACHE+=" .dvc/cache/1"
+DUMMY_CACHE+=" .dvc/cache/2"
+DUMMY_CACHE+=" .dvc/cache/3"
+
+dvc_info "Create dummy cache files"
+touch $DUMMY_CACHE
+if [ $? -ne 0 ]; then
+	dvc_error "Failed to create cache files"
+	dvc_fail
+fi
+
+dvc_info "Run 'dvc gc'"
+dvc gc
+
+dvc_info "Check that we didn't break good cache"
+dvc_check_dirs ".dvc/cache"
+pushd .dvc/cache
+dvc_check_files $ORIG_CACHE
+popd
+
+dvc_info "Check that dummy cache files were removed"
+for d in $DUMMY_CACHE; do
+	if [ -f $d ]; then
+		dvc_error "Dummy cache file $d was not removed by gc"
+		dvc_fail
+	fi
+done
+
+dvc_pass

functest/test-repro-code.sh

@@ -4,36 +4,95 @@ set -e
 
 source common.sh
 
-dvc_create_repo
+function test_generic() {
+	dvc_info '=== Generic repro test'
 
-dvc_info 'Copy foo into foo1'
-dvc run -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
+	dvc_create_repo
 
-dvc_info 'Modify code'
-echo " " >> code/code.sh 
-git commit -am 'Change code'
-    
-dvc_info 'Reproduce foo1'
-dvc repro data/foo1
-dvc_check_files data/foo1
-if [ "$(cat data/foo1)" != "foo" ]; then
-        dvc_fail
-fi
+	dvc_info 'Copy foo into foo1'
+	dvc run -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
 
-dvc_info 'Modify foo'
-dvc remove data/foo
-cp $DATA_CACHE/bar data/foo
-dvc add data/foo
+	dvc_info 'Modify code'
+	echo " " >> code/code.sh 
+	git commit -am 'Change code'
+	    
+	dvc_info 'Reproduce foo1'
+	dvc repro data/foo1
+	dvc_check_files data/foo1
+	if [ "$(cat data/foo1)" != "foo" ]; then
+		dvc_fail
+	fi
 
-dvc_info 'Set default target'
-dvc config global.target data/foo1
-git commit -am 'Set default target'
+	dvc_info 'Modify foo'
+	dvc remove data/foo
+	cp $DATA_CACHE/bar data/foo
+	dvc add data/foo
 
-dvc_info 'Reproduce foo1 as default target'
-dvc repro
-dvc_check_files data/foo1
-if [ "$(cat data/foo1)" != "bar" ]; then
-        dvc_fail
-fi
+	dvc_info 'Set default target'
+	dvc config global.target data/foo1
+	git commit -am 'Set default target'
+
+	dvc_info 'Reproduce foo1 as default target'
+	dvc repro
+	dvc_check_files data/foo1
+	if [ "$(cat data/foo1)" != "bar" ]; then
+		dvc_fail
+	fi
+}
+
+function test_partial() {
+	dvc_info "=== Partial repro test"
+
+	dvc_create_repo
+
+	cp code/code.sh code/code1.sh
+	cp code/code1.sh code/code2.sh
+	cp code/code2.sh code/code3.sh
+	git add code
+	git commit -m "copy code"
+
+	dvc_info "Create repro chain foo -> foo1 -> foo2 -> foo3"
+	dvc run -d code/code1.sh -d data/foo -o data/foo1 cp data/foo data/foo1
+	dvc run -d code/code2.sh -d data/foo1 -o data/foo2 cp data/foo1 data/foo2
+	dvc run -d code/code3.sh -d data/foo2 -o data/foo3 cp data/foo2 data/foo3
+
+	dvc_info "Save original timestamps"
+	FOO_TS=$(dvc_timestamp data/foo)
+	FOO1_TS=$(dvc_timestamp data/foo1)
+	FOO2_TS=$(dvc_timestamp data/foo2)
+	FOO3_TS=$(dvc_timestamp data/foo3)
+
+	dvc_info "Modify dependency for data/foo1, that will not result in data/foo1 contents being changed"
+	echo " " >> code/code1.sh
+	git add code
+	git commit -m "modify code1.sh"
+
+	dvc_info "Reproduce data/foo3"
+	dvc repro data/foo3
+
+	dvc_info "Check timestamps"
+	if [ "$FOO_TS" != "$(dvc_timestamp data/foo)" ]; then
+		dvc_error "data/foo timestamp changed"
+		dvc_fail
+	fi
+
+	if [ "$FOO1_TS" == "$(dvc_timestamp data/foo1)" ]; then
+		dvc_error "data/foo1 timestamp did not change"
+		dvc_fail
+	fi
+
+	if [ "$FOO2_TS" != "$(dvc_timestamp data/foo2)" ]; then
+		dvc_error "data/foo2 timestamp changed"
+		dvc_fail
+	fi
+
+	if [ "$FOO3_TS" != "$(dvc_timestamp data/foo3)" ]; then
+		dvc_error "data/foo3 timestamp changed"
+		dvc_fail
+	fi
+}
+
+test_generic
+test_partial
 
 dvc_pass