From 863e0e4b14524b64a4ab2c70ef3088b96cb5850e Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 24 Dec 2017 18:31:15 -0800 Subject: [PATCH 01/14] Use output filename as a default stage file name but do not change dir --- dvc/cli.py | 1 - dvc/command/run.py | 23 ++++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/dvc/cli.py b/dvc/cli.py index 146c73ebe..fdd9292f2 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -118,7 +118,6 @@ def parse_args(argv=None): help='Lock data item - disable reproduction.') run_parser.add_argument('-f', '--file', - default=Stage.STAGE_FILE, help='Specify name of the state file') run_parser.add_argument('-c', '--cwd', diff --git a/dvc/command/run.py b/dvc/command/run.py index 87f321d58..325122362 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -1,14 +1,35 @@ +import os + from dvc.command.common.base import CmdBase +from dvc.logger import Logger +from dvc.stage import Stage class CmdRun(CmdBase): def run(self): + fname = self.get_stage_file(self.args.file, self.args.outs, self.args.outs_no_cache) + self.project.run(cmd=' '.join(self.args.command), outs=self.args.outs, outs_no_cache=self.args.outs_no_cache, deps=self.args.deps, deps_no_cache=self.args.deps_no_cache, locked=self.args.lock, - fname=self.args.file, + fname=fname, cwd=self.args.cwd) return 0 + + @staticmethod + def get_stage_file(args_file, args_out, args_outs_no_cache): + if args_file: + return args_file + + if args_out or args_outs_no_cache: + result = args_out[0] if args_out else args_outs_no_cache[0] + result = os.path.basename(result) + result += Stage.STAGE_FILE_SUFFIX + else: + result = Stage.STAGE_FILE + + Logger.info(u'Using \'{}\' as a stage file'.format(result)) + return result From 3473dc6ed5fcda0be8c373dceeab6da46073e12f Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 24 Dec 2017 21:13:06 -0800 Subject: [PATCH 02/14] Bugfix: not ignoring data files --- dvc/scm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/scm.py b/dvc/scm.py index 4a5082316..55321665a 100644 --- a/dvc/scm.py +++ b/dvc/scm.py @@ -59,7 +59,7 @@ class Git(Base): if not gitignore.startswith(self.root_dir): raise FileNotInRepoError() - if os.path.exists(gitignore) and entry in open(gitignore, 'r').read(): + if os.path.exists(gitignore) and entry in open(gitignore, 'r').readline(): return open(gitignore, 'a').write('\n' + entry) From d29c94f0012e10a6bc065c9066bebd180b664e60 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 24 Dec 2017 22:08:44 -0800 Subject: [PATCH 03/14] dvc checkout: remove untracked files --- dvc/command/checkout.py | 31 +++++++++++++++++++++++++++++++ dvc/scm.py | 6 ++++++ 2 files changed, 37 insertions(+) diff --git a/dvc/command/checkout.py b/dvc/command/checkout.py index cf558307b..1eda94896 100644 --- a/dvc/command/checkout.py +++ b/dvc/command/checkout.py @@ -1,7 +1,38 @@ +import os + from dvc.command.common.base import CmdBase +from dvc.logger import Logger +from dvc.system import System class CmdCheckout(CmdBase): def run(self): + self.remove_untracked_hardlinks() self.project.checkout() return 0 + + def remove_untracked_hardlinks(self): + untracked_cache_files = self.untracked_hardlinks_files() + + for file in untracked_cache_files: + Logger.info(u'Remove \'{}\''.format(file)) + os.remove(file) + + dir = os.path.dirname(file) + if not os.listdir(dir): + Logger.info(u'Remove empty directory \'{}\''.format(dir)) + os.removedirs(dir) + pass + + def untracked_hardlinks_files(self): + untracked_files = set(self.project.scm.untracked_files()) + cached_files = [] + for cache_file in self.project.cache.all(): + to_remove = set() + for file in untracked_files: + if System.samefile(cache_file, file): + cached_files.append(file) + to_remove.add(file) + untracked_files = untracked_files - to_remove + + return cached_files diff --git a/dvc/scm.py b/dvc/scm.py index 55321665a..b5c18d8dd 100644 --- a/dvc/scm.py +++ b/dvc/scm.py @@ -38,6 +38,9 @@ class Base(object): def brancher(self, branch, new_branch): return BranchChanger(self, branch, new_branch) + def untracked_files(self): + pass + class Git(Base): GITIGNORE = '.gitignore' @@ -79,6 +82,9 @@ class Git(Base): def branch(self, branch): self.repo.git.branch(branch) + def untracked_files(self): + return self.repo.untracked_files + def SCM(root_dir): if Git.is_repo(root_dir): From 68ba2c34df4b82660e9260e5b3a06a3297919d28 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 24 Dec 2017 22:09:04 -0800 Subject: [PATCH 04/14] Duble space before class declaration --- dvc/command/common/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dvc/command/common/base.py b/dvc/command/common/base.py index 99be90503..c5fcea2c1 100644 --- a/dvc/command/common/base.py +++ b/dvc/command/common/base.py @@ -1,5 +1,6 @@ from dvc.project import Project + class CmdBase(object): def __init__(self, args): self.project = Project('.') From 4a1363de76658093b0454d3610c122a7e5e4b9ed Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 24 Dec 2017 22:22:15 -0800 Subject: [PATCH 05/14] Refactoring: use lambda in untracked_hardlinks_files() --- dvc/command/checkout.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dvc/command/checkout.py b/dvc/command/checkout.py index 1eda94896..c6e5877fc 100644 --- a/dvc/command/checkout.py +++ b/dvc/command/checkout.py @@ -28,11 +28,8 @@ class CmdCheckout(CmdBase): untracked_files = set(self.project.scm.untracked_files()) cached_files = [] for cache_file in self.project.cache.all(): - to_remove = set() - for file in untracked_files: - if System.samefile(cache_file, file): - cached_files.append(file) - to_remove.add(file) - untracked_files = untracked_files - to_remove + hardlinks = list(filter(lambda f: System.samefile(cache_file, f), untracked_files)) + cached_files.extend(hardlinks) + untracked_files = untracked_files - set(hardlinks) return cached_files From 1987693f34fb05afb9fd46b436a16abb95e3a165 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 24 Dec 2017 22:40:08 -0800 Subject: [PATCH 06/14] Refactoring: extract method stage_file_basename() --- dvc/command/run.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/dvc/command/run.py b/dvc/command/run.py index 325122362..2bb7246fd 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -7,7 +7,7 @@ from dvc.stage import Stage class CmdRun(CmdBase): def run(self): - fname = self.get_stage_file(self.args.file, self.args.outs, self.args.outs_no_cache) + fname = self.stage_file_name(self.args.file, self.args.outs, self.args.outs_no_cache) self.project.run(cmd=' '.join(self.args.command), outs=self.args.outs, @@ -20,16 +20,22 @@ class CmdRun(CmdBase): return 0 @staticmethod - def get_stage_file(args_file, args_out, args_outs_no_cache): + def stage_file_name(args_file, args_outs, args_outs_no_cache): if args_file: return args_file - if args_out or args_outs_no_cache: - result = args_out[0] if args_out else args_outs_no_cache[0] - result = os.path.basename(result) - result += Stage.STAGE_FILE_SUFFIX + if args_outs: + result = CmdRun._stage_file_basename(args_outs) + elif args_outs_no_cache: + result = CmdRun._stage_file_basename(args_outs_no_cache) else: result = Stage.STAGE_FILE Logger.info(u'Using \'{}\' as a stage file'.format(result)) return result + + @staticmethod + def _stage_file_basename(fname): + result = os.path.basename(fname) + result += Stage.STAGE_FILE_SUFFIX + return result From 01a37388d87440e6826018840487cb40bae4bd4b Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 24 Dec 2017 23:28:11 -0800 Subject: [PATCH 07/14] Bugfix: dvc init --- dvc/project.py | 3 +++ dvc/scm.py | 15 +++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/dvc/project.py b/dvc/project.py index ae86e1bc9..d9a785224 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -86,6 +86,9 @@ class Project(object): scm.ignore_list([cache.cache_dir, state.state_file, lock.lock_file]) + + ignore_file = os.path.join(dvc_dir, scm.ignore_file()) + scm.add([config.config_file, ignore_file]) scm.commit('DVC init') return Project(root_dir) diff --git a/dvc/scm.py b/dvc/scm.py index b5c18d8dd..303aff9a8 100644 --- a/dvc/scm.py +++ b/dvc/scm.py @@ -17,15 +17,15 @@ class Base(object): def ignore(self, path): pass + def ignore_file(self): + pass + def ignore_list(self, p_list): return [self.ignore(path) for path in p_list] - def add(self, path): + def add(self, paths): pass - def add_list(self, p_list): - return [self.add(path) for path in p_list] - def commit(self, msg): pass @@ -67,8 +67,11 @@ class Git(Base): open(gitignore, 'a').write('\n' + entry) - def add(self, path): - self.repo.index.add(path) + def ignore_file(self): + return self.GITIGNORE + + def add(self, paths): + self.repo.index.add(paths) def commit(self, msg): self.repo.index.commit(msg) From c760e537502b5aa529f22e9f0e07db85b2215891 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 24 Dec 2017 23:29:07 -0800 Subject: [PATCH 08/14] Duble space before class declaration --- dvc/command/init.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dvc/command/init.py b/dvc/command/init.py index 8402ec017..7397d5907 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -1,5 +1,6 @@ from dvc.project import Project + class CmdInit(object): def __init__(self, args): self.args = args From 7a300131ce37c1506eae25f384d82c34bdf5597f Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 24 Dec 2017 23:31:54 -0800 Subject: [PATCH 09/14] Small bugfix: args_outs --> args_outs[0] --- dvc/command/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dvc/command/run.py b/dvc/command/run.py index 2bb7246fd..f8d7f47e8 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -25,9 +25,9 @@ class CmdRun(CmdBase): return args_file if args_outs: - result = CmdRun._stage_file_basename(args_outs) + result = CmdRun._stage_file_basename(args_outs[0]) elif args_outs_no_cache: - result = CmdRun._stage_file_basename(args_outs_no_cache) + result = CmdRun._stage_file_basename(args_outs_no_cache[0]) else: result = Stage.STAGE_FILE From 884087d68b865c82c45f88190a1fb0b64f323162 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 25 Dec 2017 00:16:02 -0800 Subject: [PATCH 10/14] .gitignore file fix and getting rid of empty line --- dvc/scm.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/dvc/scm.py b/dvc/scm.py index 303aff9a8..439a8d574 100644 --- a/dvc/scm.py +++ b/dvc/scm.py @@ -62,10 +62,17 @@ class Git(Base): if not gitignore.startswith(self.root_dir): raise FileNotInRepoError() - if os.path.exists(gitignore) and entry in open(gitignore, 'r').readline(): - return + ignore_list = [] + if os.path.exists(gitignore): + ignore_list = open(gitignore, 'r').readlines() + if entry in ignore_list: + return - open(gitignore, 'a').write('\n' + entry) + content = entry + if ignore_list: + content = '\n' + content + + open(gitignore, 'a').write(content) def ignore_file(self): return self.GITIGNORE From f6a3830de2e38461f6f2cfdfc40335db8d84cbc7 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 25 Dec 2017 00:22:34 -0800 Subject: [PATCH 11/14] Func test fix: default stage file was changed --- functest/test-repro-code.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functest/test-repro-code.sh b/functest/test-repro-code.sh index 27f071b81..e05821e85 100755 --- a/functest/test-repro-code.sh +++ b/functest/test-repro-code.sh @@ -17,7 +17,7 @@ function test_generic() { git commit -am 'Change code' dvc_info 'Reproduce foo1' - dvc repro + dvc repro foo1.dvc dvc_check_files data/foo1 data/foo1.dvc if [ "$(cat data/foo1)" != "foo" ]; then dvc_fail @@ -28,7 +28,7 @@ function test_generic() { cp $DATA_CACHE/bar data/foo dvc_info 'Reproduce foo1 as default target' - dvc repro + dvc repro foo1.dvc dvc_check_files data/foo1 data/foo1.dvc if [ "$(cat data/foo1)" != "bar" ]; then dvc_fail From 8b433c609b6fb712dfc1e9b6ed736aa13704b321 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 25 Dec 2017 01:28:30 -0800 Subject: [PATCH 12/14] Func test fix --- functest/test-lock.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functest/test-lock.sh b/functest/test-lock.sh index 3dfe6ca69..a1c10e153 100755 --- a/functest/test-lock.sh +++ b/functest/test-lock.sh @@ -6,7 +6,7 @@ source common.sh dvc_create_repo -dvc run -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 +dvc run -f Dvcfile -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 cat Dvcfile | grep 'locked' | grep 'false' || dvc_fatal "stage locked after 'dvc run'" dvc lock cat Dvcfile | grep 'locked' | grep 'true' || dvc_fatal "stage not locked after 'dvc lock'" From b2de95ebb267de18f340459daf92690dc6650e93 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 25 Dec 2017 01:42:42 -0800 Subject: [PATCH 13/14] Bugfix: dvc repro - check if files exist before actions --- dvc/stage.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dvc/stage.py b/dvc/stage.py index 0fe8a57c2..e7dfd185e 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -31,7 +31,9 @@ class Output(object): if not self.use_cache: changed = self._changed_md5() else: - changed = not os.path.samefile(self.path, self.cache) + changed = os.path.exists(self.path) \ + and os.path.exists(self.cache) \ + and not os.path.samefile(self.path, self.cache) if changed: self.project.logger.debug('{} changed'.format(self.path)) @@ -173,8 +175,9 @@ class Stage(object): def remove_outs(self): for out in self.outs: - self.project.logger.debug("Removing '{}'".format(out.path)) - os.unlink(out.path) + if os.path.exists(out.path): + self.project.logger.debug("Removing '{}'".format(out.path)) + os.unlink(out.path) def remove(self): self.remove_outs() From b9219404c2d13ba010f2269944973c0d317aa3f6 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 25 Dec 2017 03:09:52 -0800 Subject: [PATCH 14/14] Tutorial branch update - intermediate version --- functest/tutorial_branches.sh | 40 +++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/functest/tutorial_branches.sh b/functest/tutorial_branches.sh index 87620edc1..48c9cf290 100755 --- a/functest/tutorial_branches.sh +++ b/functest/tutorial_branches.sh @@ -28,40 +28,48 @@ dvc init git checkout -b input_100K mkdir data -dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz data/ -dvc run tar zxf data/Posts.xml.tgz -C data/ -dvc run python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python - +wget https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz -O data/Posts.xml.tgz +#cp ../stackoverflow_small_xml/100K/Posts.xml.tgz data/ +dvc add data/Posts.xml.tgz +dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/ +dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python +git add . +git commit -m 'Extract TSV for 100K dataset' git checkout master dvc checkout git checkout -b input_25K dvc checkout mkdir data -dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz data/ -dvc run tar zxf data/Posts.xml.tgz -C data/ +wget https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz -O data/Posts.xml.tgz +#cp ../stackoverflow_small_xml/25K/Posts.xml.tgz data/ +dvc add data/Posts.xml.tgz +dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/ -dvc run python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python -dvc run python code/split_train_test.py data/Posts.tsv 0.33 20170426 data/Posts-train.tsv data/Posts-test.tsv -dvc run python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p +dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python +dvc run -d code/split_train_test.py -d data/Posts.tsv -o data/Posts-train.tsv -o data/Posts-test.tsv python code/split_train_test.py data/Posts.tsv 0.33 20170426 data/Posts-train.tsv data/Posts-test.tsv +dvc run -d code/featurization.py -d data/Posts-train.tsv -d data/Posts-test.tsv -o data/matrix-train.p -o data/matrix-test.p python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p -dvc run python code/train_model.py data/matrix-train.p 20170426 data/model.p +dvc run -d code/train_model.py -d data/matrix-train.p -o data/model.p python code/train_model.py data/matrix-train.p 20170426 data/model.p -dvc run python code/evaluate.py data/model.p data/matrix-test.p data/eval_auc.txt +dvc run -d code/evaluate.py -d data/model.p -d data/matrix-test.p -O data/eval_auc.txt python code/evaluate.py data/model.p data/matrix-test.p data/eval_auc.txt -dvc config Global.Target data/eval_auc.txt -git commit -am 'Set Target' +#dvc config Global.Target data/eval_auc.txt +#git commit -am 'Set Target' cat data/eval_auc.txt # AUC: 0.596182 +git add . +git commit -m 'Full pipeline' + git checkout input_100K dvc checkout git merge -X theirs input_25K # git merge -X theirs input_25K -# Resolve conflicts by hends!!! -git add . -git commit -m 'Merge conflicts' +## Resolve conflicts by hands!!! +#git add . +#git commit -m 'Merge conflicts' dvc merge dvc repro # <-- Error. Reproducng Posts.xml and Posts.tsv