From c6029d142190cbe33d31aa0f8999d406154c7b67 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 19 Oct 2017 23:27:06 +0300 Subject: [PATCH 1/4] repro: use state to obtain data item history Fixes #258 Signed-off-by: Ruslan Kuprieiev --- dvc/command/repro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index e96f79dfb..f618cd341 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -78,7 +78,7 @@ class CmdRepro(CmdRun): for data_item in data_item_list: try: - target_commit = self.git.get_target_commit(data_item.data.relative) + target_commit = self.git.get_target_commit(data_item.state.relative) if target_commit is None: msg = 'Data item "{}" cannot be reproduced: cannot obtain commit hashsum' Logger.warn(msg.format(data_item.data.relative)) From 2cf060ca5fb732011aaa94ffb1e1b747cc96d0dc Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 20 Oct 2017 22:50:09 +0300 Subject: [PATCH 2/4] merge: checkout state files Fixes #263 Fixes #260 Signed-off-by: Ruslan Kuprieiev --- dvc/command/merge.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/dvc/command/merge.py b/dvc/command/merge.py index 871a854f0..13afab86b 100644 --- a/dvc/command/merge.py +++ b/dvc/command/merge.py @@ -10,37 +10,37 @@ class CmdMerge(CmdBase): super(CmdMerge, self).__init__(settings) def print_info(self, targets): - for fname in targets: + for item in targets: Logger.info('Restored original data after merge:') - Logger.info(' {}'.format(fname)) + Logger.info(' {}'.format(item.data.relative)) def collect_targets(self): targets = [] flist = self.git.get_last_merge_changed_files() - for fname in flist: - try: - item = self.settings.path_factory.data_item(fname) - except NotInDataDirError: - continue + items = self.settings.path_factory.to_data_items(flist)[0] + for item in items: try: state = StateFile.load(item, self.git) except Exception as ex: - Logger.error('Failed to load state file for {}'.format(fname), exc_info=True) + Logger.error('Failed to load state file for {}'.format(item.data.relative), exc_info=True) return None if not state.command == StateFile.COMMAND_IMPORT_FILE: continue - targets.append(fname) + targets.append(item) return targets def checkout_targets(self, targets): - for fname in targets: - self.git.checkout_file_before_last_merge(fname) + data = [] + for item in targets: + self.git.checkout_file_before_last_merge(item.data.relative) + self.git.checkout_file_before_last_merge(item.state.relative) + data.append(item.data.relative) - msg = 'DVC merge files: {}'.format(' '.join(targets)) + msg = 'DVC merge files: {}'.format(' '.join(data)) self.commit_if_needed(msg) def run(self): From 561cca120d062e212c504e2140889bd142507c24 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 20 Oct 2017 23:45:39 +0300 Subject: [PATCH 3/4] test: add test for 'dvc merge' Basic test for 'dvc merge' functionality. We just create two branches, do some dummy work on big data set, then branch and do some changes on small data set and then merge second branch into first, calling 'dvc merge' afterwards to restore big data back. Fixes #262 Signed-off-by: Ruslan Kuprieiev --- functest/run.sh | 1 + functest/test-merge.sh | 55 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100755 functest/test-merge.sh diff --git a/functest/run.sh b/functest/run.sh index d2626de49..0cc9ac5ee 100755 --- a/functest/run.sh +++ b/functest/run.sh @@ -8,6 +8,7 @@ TESTS+=" test-init.sh" TESTS+=" test-import.sh" TESTS+=" test-sync.sh" TESTS+=" test-repro-code.sh" +TESTS+=" test-merge.sh" for t in $TESTS; do rm -rf $TEST_REPO diff --git a/functest/test-merge.sh b/functest/test-merge.sh new file mode 100755 index 000000000..5d6399af8 --- /dev/null +++ b/functest/test-merge.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -e + +source common.sh + +dvc_create_repo + +# Adding changes to the first branch +git checkout -b input_50K +echo -e "#!/bin/bash\nset -x\nset -e\ncp \$1 \$2" > code/code1.sh +chmod +x code/code1.sh +cp code/code1.sh code/code2.sh +cp code/code1.sh code/code3.sh +cp code/code1.sh code/code4.sh +git add code +git commit -s -m"add code" + +dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/50K/Posts.xml.tgz data/data + +ORIG_DATA=$(readlink -- data/data) + +dvc run ./code/code1.sh data/data data/data1 +dvc run ./code/code2.sh data/data1 data/data2 +dvc run ./code/code3.sh data/data2 data/data3 +dvc run ./code/code4.sh data/data3 data/data4 + +dvc config Global.Target data/data4 +git commit -am 'Set Target' + +# Create new branch to work on small data set +git checkout -b input_10K +dvc remove data/data +dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/10K/Posts.xml.tgz data/data +dvc repro + +echo -e "#!/bin/bash\n\n\n\ncp \$1 \$2" > code/code1.sh +git commit -am"change code1.sh" +dvc repro + +# Merge input_10K into input_50K +git checkout input_50K +git merge input_10K +# Notice we're running 'dvc merge' on input_50K branch. This +# way dvc knows to favor data files from current branch. +dvc merge + +# Verify that data and respective state file are restored properly +if [ "$(readlink -- "data/data")" != "$ORIG_DATA" ]; then + dvc_fail +fi + +grep $(basename "$ORIG_DATA") .dvc/state/data.state || dvc_fail + +dvc_pass From 9941a3f92ca84e66be2ddeff53e80ddff38ca708 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 21 Oct 2017 01:14:20 +0300 Subject: [PATCH 4/4] state: capture exception if unable to parse target metrics Fixes #265 Signed-off-by: Ruslan Kuprieiev --- dvc/state_file.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dvc/state_file.py b/dvc/state_file.py index 0b45bc625..22004fb52 100644 --- a/dvc/state_file.py +++ b/dvc/state_file.py @@ -113,7 +113,13 @@ class StateFile(object): @staticmethod def parse_target_metric(file_name): - lines = open(file_name).readlines(2) + fd = open(file_name, 'r') + + try: + lines = fd.readlines(2) + except Exception as e: + raise StateFileError('Unable to parse target file') + if len(lines) != 1: raise StateFileError('file contains more then one line')