diff --git a/dvc/path/data_item.py b/dvc/path/data_item.py index 8b0f0e41c..9eafeff70 100644 --- a/dvc/path/data_item.py +++ b/dvc/path/data_item.py @@ -20,9 +20,9 @@ class DataDirError(DvcException): super(DataDirError, self).__init__(msg) -class DataItemInStatusDirError(DataDirError): +class DataItemInConfigDirError(DataDirError): def __init__(self, file): - msg = 'File "{}" is in state directory'.format(file) + msg = 'File "{}" is in config directory'.format(file) super(DataItemInStatusDirError, self).__init__(msg) @@ -48,10 +48,8 @@ class DataItem(object): if not self._data.abs.startswith(self._git.git_dir_abs): raise NotInGitDirError(data_file, self._git.git_dir_abs) - if self._data.abs.startswith(self.state_dir_abs): - raise DataItemInStatusDirError(data_file) - - pass + if self._data.abs.startswith(os.path.join(self._git.git_dir_abs, self._config.CONFIG_DIR)): + raise DataItemInConfigDirError(data_file) def copy(self, cache_file=None): if not cache_file: diff --git a/dvc/repository_change.py b/dvc/repository_change.py index 087a118d8..1a487fede 100644 --- a/dvc/repository_change.py +++ b/dvc/repository_change.py @@ -4,7 +4,7 @@ from dvc.exceptions import DvcException from dvc.git_wrapper import GitWrapper from dvc.executor import Executor from dvc.logger import Logger -from dvc.path.data_item import DataDirError, DataItemInStatusDirError +from dvc.path.data_item import DataDirError, DataItemInConfigDirError from dvc.path.stated_data_item import StatedDataItem from dvc.utils import cached_property from dvc.config import ConfigI @@ -74,7 +74,7 @@ class RepositoryChange(object): Logger.debug('[Repository change] Add status: {} {}'.format( item.status, item.data.dvc)) - except DataItemInStatusDirError: + except DataItemInConfigDirError: self._created_status_files.append(file) except DataDirError: self._externally_created_files.append(file) diff --git a/functest/common.sh b/functest/common.sh index 38565fd9c..1cb1c759c 100644 --- a/functest/common.sh +++ b/functest/common.sh @@ -22,14 +22,30 @@ TEST_REPO_REGION='us-east-2' TEST_REPO_GCP='dvc-test/myrepo' TEST_REPO_GCP_PROJECT='dvc-project' -function dvc_md5() { +function is_osx() { if [ "$(uname)" == "Darwin" ]; then + return 0 + else + return 1 + fi +} + +function dvc_md5() { + if is_osx; then md5 -r $@ else md5sum $@ fi } +function dvc_timestamp() { + if is_osx; then + stat -f "%m%t%Sm %N" $@ + else + stat -c %y data/foo $@ + fi +} + function dvc_print() { echo -e "$1" } diff --git a/functest/run.sh b/functest/run.sh index 756d9be39..3f9b1068a 100755 --- a/functest/run.sh +++ b/functest/run.sh @@ -10,6 +10,7 @@ TESTS+=" test-sync.sh" TESTS+=" test-repro-code.sh" TESTS+=" test-merge.sh" TESTS+=" test-checkout.sh" +TESTS+=" test-gc.sh" for t in $TESTS; do rm -rf $TEST_REPO diff --git a/functest/test-gc.sh b/functest/test-gc.sh new file mode 100755 index 000000000..6e52abaab --- /dev/null +++ b/functest/test-gc.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e + +source common.sh + +dvc_create_repo + +ORIG_CACHE=$(ls .dvc/cache) + +DUMMY_CACHE+=" .dvc/cache/1" +DUMMY_CACHE+=" .dvc/cache/2" +DUMMY_CACHE+=" .dvc/cache/3" + +dvc_info "Create dummy cache files" +touch $DUMMY_CACHE +if [ $? -ne 0 ]; then + dvc_error "Failed to create cache files" + dvc_fail +fi + +dvc_info "Run 'dvc gc'" +dvc gc + +dvc_info "Check that we didn't break good cache" +dvc_check_dirs ".dvc/cache" +pushd .dvc/cache +dvc_check_files $ORIG_CACHE +popd + +dvc_info "Check that dummy cache files were removed" +for d in $DUMMY_CACHE; do + if [ -f $d ]; then + dvc_error "Dummy cache file $d was not removed by gc" + dvc_fail + fi +done + +dvc_pass diff --git a/functest/test-repro-code.sh b/functest/test-repro-code.sh index 19734d1ef..ffa814aea 100755 --- a/functest/test-repro-code.sh +++ b/functest/test-repro-code.sh @@ -4,36 +4,95 @@ set -e source common.sh -dvc_create_repo +function test_generic() { + dvc_info '=== Generic repro test' -dvc_info 'Copy foo into foo1' -dvc run -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 + dvc_create_repo -dvc_info 'Modify code' -echo " " >> code/code.sh -git commit -am 'Change code' - -dvc_info 'Reproduce foo1' -dvc repro data/foo1 -dvc_check_files data/foo1 -if [ "$(cat data/foo1)" != "foo" ]; then - dvc_fail -fi + dvc_info 'Copy foo into foo1' + dvc run -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 -dvc_info 'Modify foo' -dvc remove data/foo -cp $DATA_CACHE/bar data/foo -dvc add data/foo + dvc_info 'Modify code' + echo " " >> code/code.sh + git commit -am 'Change code' + + dvc_info 'Reproduce foo1' + dvc repro data/foo1 + dvc_check_files data/foo1 + if [ "$(cat data/foo1)" != "foo" ]; then + dvc_fail + fi -dvc_info 'Set default target' -dvc config global.target data/foo1 -git commit -am 'Set default target' + dvc_info 'Modify foo' + dvc remove data/foo + cp $DATA_CACHE/bar data/foo + dvc add data/foo -dvc_info 'Reproduce foo1 as default target' -dvc repro -dvc_check_files data/foo1 -if [ "$(cat data/foo1)" != "bar" ]; then - dvc_fail -fi + dvc_info 'Set default target' + dvc config global.target data/foo1 + git commit -am 'Set default target' + + dvc_info 'Reproduce foo1 as default target' + dvc repro + dvc_check_files data/foo1 + if [ "$(cat data/foo1)" != "bar" ]; then + dvc_fail + fi +} + +function test_partial() { + dvc_info "=== Partial repro test" + + dvc_create_repo + + cp code/code.sh code/code1.sh + cp code/code1.sh code/code2.sh + cp code/code2.sh code/code3.sh + git add code + git commit -m "copy code" + + dvc_info "Create repro chain foo -> foo1 -> foo2 -> foo3" + dvc run -d code/code1.sh -d data/foo -o data/foo1 cp data/foo data/foo1 + dvc run -d code/code2.sh -d data/foo1 -o data/foo2 cp data/foo1 data/foo2 + dvc run -d code/code3.sh -d data/foo2 -o data/foo3 cp data/foo2 data/foo3 + + dvc_info "Save original timestamps" + FOO_TS=$(dvc_timestamp data/foo) + FOO1_TS=$(dvc_timestamp data/foo1) + FOO2_TS=$(dvc_timestamp data/foo2) + FOO3_TS=$(dvc_timestamp data/foo3) + + dvc_info "Modify dependency for data/foo1, that will not result in data/foo1 contents being changed" + echo " " >> code/code1.sh + git add code + git commit -m "modify code1.sh" + + dvc_info "Reproduce data/foo3" + dvc repro data/foo3 + + dvc_info "Check timestamps" + if [ "$FOO_TS" != "$(dvc_timestamp data/foo)" ]; then + dvc_error "data/foo timestamp changed" + dvc_fail + fi + + if [ "$FOO1_TS" == "$(dvc_timestamp data/foo1)" ]; then + dvc_error "data/foo1 timestamp did not change" + dvc_fail + fi + + if [ "$FOO2_TS" != "$(dvc_timestamp data/foo2)" ]; then + dvc_error "data/foo2 timestamp changed" + dvc_fail + fi + + if [ "$FOO3_TS" != "$(dvc_timestamp data/foo3)" ]; then + dvc_error "data/foo3 timestamp changed" + dvc_fail + fi +} + +test_generic +test_partial dvc_pass