diff --git a/dvc/cli.py b/dvc/cli.py index 146c73ebe..fdd9292f2 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -118,7 +118,6 @@ def parse_args(argv=None): help='Lock data item - disable reproduction.') run_parser.add_argument('-f', '--file', - default=Stage.STAGE_FILE, help='Specify name of the state file') run_parser.add_argument('-c', '--cwd', diff --git a/dvc/command/checkout.py b/dvc/command/checkout.py index cf558307b..c6e5877fc 100644 --- a/dvc/command/checkout.py +++ b/dvc/command/checkout.py @@ -1,7 +1,35 @@ +import os + from dvc.command.common.base import CmdBase +from dvc.logger import Logger +from dvc.system import System class CmdCheckout(CmdBase): def run(self): + self.remove_untracked_hardlinks() self.project.checkout() return 0 + + def remove_untracked_hardlinks(self): + untracked_cache_files = self.untracked_hardlinks_files() + + for file in untracked_cache_files: + Logger.info(u'Remove \'{}\''.format(file)) + os.remove(file) + + dir = os.path.dirname(file) + if not os.listdir(dir): + Logger.info(u'Remove empty directory \'{}\''.format(dir)) + os.removedirs(dir) + pass + + def untracked_hardlinks_files(self): + untracked_files = set(self.project.scm.untracked_files()) + cached_files = [] + for cache_file in self.project.cache.all(): + hardlinks = list(filter(lambda f: System.samefile(cache_file, f), untracked_files)) + cached_files.extend(hardlinks) + untracked_files = untracked_files - set(hardlinks) + + return cached_files diff --git a/dvc/command/common/base.py b/dvc/command/common/base.py index 99be90503..c5fcea2c1 100644 --- a/dvc/command/common/base.py +++ b/dvc/command/common/base.py @@ -1,5 +1,6 @@ from dvc.project import Project + class CmdBase(object): def __init__(self, args): self.project = Project('.') diff --git a/dvc/command/init.py b/dvc/command/init.py index 8402ec017..7397d5907 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -1,5 +1,6 @@ from dvc.project import Project + class CmdInit(object): def __init__(self, args): self.args = args diff --git a/dvc/command/run.py b/dvc/command/run.py index 87f321d58..f8d7f47e8 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -1,14 +1,41 @@ +import os + from dvc.command.common.base import CmdBase +from dvc.logger import Logger +from dvc.stage import Stage class CmdRun(CmdBase): def run(self): + fname = self.stage_file_name(self.args.file, self.args.outs, self.args.outs_no_cache) + self.project.run(cmd=' '.join(self.args.command), outs=self.args.outs, outs_no_cache=self.args.outs_no_cache, deps=self.args.deps, deps_no_cache=self.args.deps_no_cache, locked=self.args.lock, - fname=self.args.file, + fname=fname, cwd=self.args.cwd) return 0 + + @staticmethod + def stage_file_name(args_file, args_outs, args_outs_no_cache): + if args_file: + return args_file + + if args_outs: + result = CmdRun._stage_file_basename(args_outs[0]) + elif args_outs_no_cache: + result = CmdRun._stage_file_basename(args_outs_no_cache[0]) + else: + result = Stage.STAGE_FILE + + Logger.info(u'Using \'{}\' as a stage file'.format(result)) + return result + + @staticmethod + def _stage_file_basename(fname): + result = os.path.basename(fname) + result += Stage.STAGE_FILE_SUFFIX + return result diff --git a/dvc/project.py b/dvc/project.py index ae86e1bc9..d9a785224 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -86,6 +86,9 @@ class Project(object): scm.ignore_list([cache.cache_dir, state.state_file, lock.lock_file]) + + ignore_file = os.path.join(dvc_dir, scm.ignore_file()) + scm.add([config.config_file, ignore_file]) scm.commit('DVC init') return Project(root_dir) diff --git a/dvc/scm.py b/dvc/scm.py index 4a5082316..439a8d574 100644 --- a/dvc/scm.py +++ b/dvc/scm.py @@ -17,15 +17,15 @@ class Base(object): def ignore(self, path): pass + def ignore_file(self): + pass + def ignore_list(self, p_list): return [self.ignore(path) for path in p_list] - def add(self, path): + def add(self, paths): pass - def add_list(self, p_list): - return [self.add(path) for path in p_list] - def commit(self, msg): pass @@ -38,6 +38,9 @@ class Base(object): def brancher(self, branch, new_branch): return BranchChanger(self, branch, new_branch) + def untracked_files(self): + pass + class Git(Base): GITIGNORE = '.gitignore' @@ -59,13 +62,23 @@ class Git(Base): if not gitignore.startswith(self.root_dir): raise FileNotInRepoError() - if os.path.exists(gitignore) and entry in open(gitignore, 'r').read(): - return + ignore_list = [] + if os.path.exists(gitignore): + ignore_list = open(gitignore, 'r').readlines() + if entry in ignore_list: + return - open(gitignore, 'a').write('\n' + entry) + content = entry + if ignore_list: + content = '\n' + content - def add(self, path): - self.repo.index.add(path) + open(gitignore, 'a').write(content) + + def ignore_file(self): + return self.GITIGNORE + + def add(self, paths): + self.repo.index.add(paths) def commit(self, msg): self.repo.index.commit(msg) @@ -79,6 +92,9 @@ class Git(Base): def branch(self, branch): self.repo.git.branch(branch) + def untracked_files(self): + return self.repo.untracked_files + def SCM(root_dir): if Git.is_repo(root_dir): diff --git a/dvc/stage.py b/dvc/stage.py index 0fe8a57c2..e7dfd185e 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -31,7 +31,9 @@ class Output(object): if not self.use_cache: changed = self._changed_md5() else: - changed = not os.path.samefile(self.path, self.cache) + changed = os.path.exists(self.path) \ + and os.path.exists(self.cache) \ + and not os.path.samefile(self.path, self.cache) if changed: self.project.logger.debug('{} changed'.format(self.path)) @@ -173,8 +175,9 @@ class Stage(object): def remove_outs(self): for out in self.outs: - self.project.logger.debug("Removing '{}'".format(out.path)) - os.unlink(out.path) + if os.path.exists(out.path): + self.project.logger.debug("Removing '{}'".format(out.path)) + os.unlink(out.path) def remove(self): self.remove_outs() diff --git a/functest/test-lock.sh b/functest/test-lock.sh index 3dfe6ca69..a1c10e153 100755 --- a/functest/test-lock.sh +++ b/functest/test-lock.sh @@ -6,7 +6,7 @@ source common.sh dvc_create_repo -dvc run -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 +dvc run -f Dvcfile -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 cat Dvcfile | grep 'locked' | grep 'false' || dvc_fatal "stage locked after 'dvc run'" dvc lock cat Dvcfile | grep 'locked' | grep 'true' || dvc_fatal "stage not locked after 'dvc lock'" diff --git a/functest/test-repro-code.sh b/functest/test-repro-code.sh index 27f071b81..e05821e85 100755 --- a/functest/test-repro-code.sh +++ b/functest/test-repro-code.sh @@ -17,7 +17,7 @@ function test_generic() { git commit -am 'Change code' dvc_info 'Reproduce foo1' - dvc repro + dvc repro foo1.dvc dvc_check_files data/foo1 data/foo1.dvc if [ "$(cat data/foo1)" != "foo" ]; then dvc_fail @@ -28,7 +28,7 @@ function test_generic() { cp $DATA_CACHE/bar data/foo dvc_info 'Reproduce foo1 as default target' - dvc repro + dvc repro foo1.dvc dvc_check_files data/foo1 data/foo1.dvc if [ "$(cat data/foo1)" != "bar" ]; then dvc_fail diff --git a/functest/tutorial_branches.sh b/functest/tutorial_branches.sh index 87620edc1..48c9cf290 100755 --- a/functest/tutorial_branches.sh +++ b/functest/tutorial_branches.sh @@ -28,40 +28,48 @@ dvc init git checkout -b input_100K mkdir data -dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz data/ -dvc run tar zxf data/Posts.xml.tgz -C data/ -dvc run python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python - +wget https://s3-us-west-2.amazonaws.com/dvc-share/so/100K/Posts.xml.tgz -O data/Posts.xml.tgz +#cp ../stackoverflow_small_xml/100K/Posts.xml.tgz data/ +dvc add data/Posts.xml.tgz +dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/ +dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python +git add . +git commit -m 'Extract TSV for 100K dataset' git checkout master dvc checkout git checkout -b input_25K dvc checkout mkdir data -dvc import https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz data/ -dvc run tar zxf data/Posts.xml.tgz -C data/ +wget https://s3-us-west-2.amazonaws.com/dvc-share/so/25K/Posts.xml.tgz -O data/Posts.xml.tgz +#cp ../stackoverflow_small_xml/25K/Posts.xml.tgz data/ +dvc add data/Posts.xml.tgz +dvc run -o data/Posts.xml -d data/Posts.xml.tgz tar zxf data/Posts.xml.tgz -C data/ -dvc run python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python -dvc run python code/split_train_test.py data/Posts.tsv 0.33 20170426 data/Posts-train.tsv data/Posts-test.tsv -dvc run python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p +dvc run -d code/xml_to_tsv.py -d data/Posts.xml -o data/Posts.tsv python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv python +dvc run -d code/split_train_test.py -d data/Posts.tsv -o data/Posts-train.tsv -o data/Posts-test.tsv python code/split_train_test.py data/Posts.tsv 0.33 20170426 data/Posts-train.tsv data/Posts-test.tsv +dvc run -d code/featurization.py -d data/Posts-train.tsv -d data/Posts-test.tsv -o data/matrix-train.p -o data/matrix-test.p python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv data/matrix-train.p data/matrix-test.p -dvc run python code/train_model.py data/matrix-train.p 20170426 data/model.p +dvc run -d code/train_model.py -d data/matrix-train.p -o data/model.p python code/train_model.py data/matrix-train.p 20170426 data/model.p -dvc run python code/evaluate.py data/model.p data/matrix-test.p data/eval_auc.txt +dvc run -d code/evaluate.py -d data/model.p -d data/matrix-test.p -O data/eval_auc.txt python code/evaluate.py data/model.p data/matrix-test.p data/eval_auc.txt -dvc config Global.Target data/eval_auc.txt -git commit -am 'Set Target' +#dvc config Global.Target data/eval_auc.txt +#git commit -am 'Set Target' cat data/eval_auc.txt # AUC: 0.596182 +git add . +git commit -m 'Full pipeline' + git checkout input_100K dvc checkout git merge -X theirs input_25K # git merge -X theirs input_25K -# Resolve conflicts by hends!!! -git add . -git commit -m 'Merge conflicts' +## Resolve conflicts by hands!!! +#git add . +#git commit -m 'Merge conflicts' dvc merge dvc repro # <-- Error. Reproducng Posts.xml and Posts.tsv