From 14181ede5ff8212ad49dd8536c24baae9bc28c45 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 4 Jan 2018 10:30:45 +0200 Subject: [PATCH] dvc: remove -D option Fixes #370 Signed-off-by: Ruslan Kuprieiev --- dvc/cli.py | 5 ----- dvc/command/run.py | 1 - dvc/project.py | 14 +----------- dvc/stage.py | 44 ++++++++++++++++++++----------------- functest/test-checkout.sh | 2 +- functest/test-lock.sh | 3 ++- functest/test-repro-code.sh | 10 +++++---- functest/test-sync.sh | 3 ++- tests/test_checkout.py | 6 ++--- tests/test_repro.py | 10 ++++----- tests/test_run.py | 22 ++++++++++++++----- 11 files changed, 60 insertions(+), 60 deletions(-) diff --git a/dvc/cli.py b/dvc/cli.py index fdd9292f2..f277e7df0 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -96,11 +96,6 @@ def parse_args(argv=None): action='append', default=[], help='Declare dependencies for reproducible cmd.') - run_parser.add_argument('-D', - '--deps-no-cache', - action='append', - default=[], - help='Declare dependencies that should not be cached for reproducible cmd.') run_parser.add_argument('-o', '--outs', action='append', diff --git a/dvc/command/run.py b/dvc/command/run.py index f8d7f47e8..81a4853ad 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -13,7 +13,6 @@ class CmdRun(CmdBase): outs=self.args.outs, outs_no_cache=self.args.outs_no_cache, deps=self.args.deps, - deps_no_cache=self.args.deps_no_cache, locked=self.args.lock, fname=fname, cwd=self.args.cwd) diff --git a/dvc/project.py b/dvc/project.py index 28685dde9..62f620f25 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -136,17 +136,9 @@ class Project(object): return stages - def _add_orphans(self, deps): - outs = [out.path for out in self.outs()] - for dep in deps: - if not dep.use_cache or dep.path in outs: - continue - self.add(dep.path) - def run(self, cmd=None, deps=[], - deps_no_cache=[], outs=[], outs_no_cache=[], locked=False, @@ -156,11 +148,7 @@ class Project(object): path = os.path.join(cwd, fname) outputs = Output.loads_from(self, outs, use_cache=True, cwd=cwd) outputs += Output.loads_from(self, outs_no_cache, use_cache=False, cwd=cwd) - deps = Dependency.loads_from(self, deps, use_cache=True, cwd=cwd) - deps += Dependency.loads_from(self, deps_no_cache, use_cache=False, cwd=cwd) - - # Add files that were specified as cached dependencies and weren't added before - self._add_orphans(deps) + deps = Dependency.loads_from(self, deps, use_cache=False, cwd=cwd) stage = Stage(project=self, path=path, diff --git a/dvc/stage.py b/dvc/stage.py index a17ddc8f3..14920d0de 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -49,7 +49,13 @@ class Output(object): self.use_cache = use_cache def _changed_md5(self): - return self.md5 != file_md5(self.path)[0] + state = self.project.state.get(self.path) + if state and state.mtime == self.mtime(): + md5 = state.md5 + else: + md5 = file_md5(self.path)[0] + + return self.md5 != md5 def changed(self): if not self.use_cache: @@ -107,15 +113,21 @@ class Output(object): def mtime(self): return os.path.getmtime(self.path) - def update(self, md5=None): - self.md5 = md5 - if not self.md5: - if not os.path.exists(self.path): - raise OutputDoesNotExistError(self.path) - if not os.path.isfile(self.path): - raise OutputIsNotFileError(self.path) + def update(self): + if not os.path.exists(self.path): + raise OutputDoesNotExistError(self.path) + if not os.path.isfile(self.path): + raise OutputIsNotFileError(self.path) + + state = self.project.state.get(self.path) + if state and state.mtime == self.mtime(): + md5 = state.md5 + msg = '{} using md5 {} from state file' + self.project.logger.debug(msg.format(self.path, md5)) + self.md5 = md5 + else: self.md5 = file_md5(self.path)[0] - self.project.state.update(self.path, self.md5, self.mtime()) + self.project.state.update(self.path, self.md5, self.mtime()) def save(self): if not self.use_cache: @@ -134,8 +146,8 @@ class Output(object): @classmethod def loadd(cls, project, d, cwd='.'): path = os.path.join(cwd, d[Output.PARAM_PATH]) - md5 = d[Output.PARAM_MD5] - use_cache = d[Output.PARAM_CACHE] + md5 = d.get(Output.PARAM_MD5, None) + use_cache = d.get(Output.PARAM_CACHE, False) return cls(project, path, md5, use_cache=use_cache) @classmethod @@ -159,15 +171,7 @@ class Output(object): class Dependency(Output): - def update(self): - md5 = None - state = self.project.state.get(self.path) - if state and state.mtime == self.mtime(): - md5 = state.md5 - msg = '{} using md5 {} from state file for dependency' - self.project.logger.debug(msg.format(self.path, md5)) - - super(Dependency, self).update(md5=md5) + pass class Stage(object): diff --git a/functest/test-checkout.sh b/functest/test-checkout.sh index 4d2a0897d..c78a466f0 100755 --- a/functest/test-checkout.sh +++ b/functest/test-checkout.sh @@ -6,7 +6,7 @@ source common.sh dvc_create_repo -dvc run -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 +dvc add data/foo rm -f data/foo dvc checkout diff --git a/functest/test-lock.sh b/functest/test-lock.sh index a1c10e153..15f6e1b2c 100755 --- a/functest/test-lock.sh +++ b/functest/test-lock.sh @@ -6,7 +6,8 @@ source common.sh dvc_create_repo -dvc run -f Dvcfile -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 +dvc add data/foo +dvc run -f Dvcfile -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 cat Dvcfile | grep 'locked' | grep 'false' || dvc_fatal "stage locked after 'dvc run'" dvc lock cat Dvcfile | grep 'locked' | grep 'true' || dvc_fatal "stage not locked after 'dvc lock'" diff --git a/functest/test-repro-code.sh b/functest/test-repro-code.sh index e05821e85..fbae55e17 100755 --- a/functest/test-repro-code.sh +++ b/functest/test-repro-code.sh @@ -10,7 +10,8 @@ function test_generic() { dvc_create_repo dvc_info 'Copy foo into foo1' - dvc run -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 + dvc add data/foo + dvc run -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 dvc_info 'Modify code' echo " " >> code/code.sh @@ -47,9 +48,10 @@ function test_partial() { git commit -m "copy code" dvc_info "Create repro chain foo -> foo1 -> foo2 -> foo3" - dvc run -f copy_foo_foo1.dvc -D code/code1.sh -d data/foo -o data/foo1 cp data/foo data/foo1 - dvc run -f copy_foo1_foo2.dvc -D code/code2.sh -d data/foo1 -o data/foo2 cp data/foo1 data/foo2 - dvc run -f copy_foo2_foo3.dvc -D code/code3.sh -d data/foo2 -o data/foo3 cp data/foo2 data/foo3 + dvc add data/foo + dvc run -f copy_foo_foo1.dvc -d code/code1.sh -d data/foo -o data/foo1 cp data/foo data/foo1 + dvc run -f copy_foo1_foo2.dvc -d code/code2.sh -d data/foo1 -o data/foo2 cp data/foo1 data/foo2 + dvc run -f copy_foo2_foo3.dvc -d code/code3.sh -d data/foo2 -o data/foo3 cp data/foo2 data/foo3 dvc_info "Save original timestamps" FOO_TS=$(dvc_timestamp data/foo) diff --git a/functest/test-sync.sh b/functest/test-sync.sh index 94463cded..3b40651ea 100755 --- a/functest/test-sync.sh +++ b/functest/test-sync.sh @@ -5,7 +5,8 @@ set -e source common.sh function test_sync() { - dvc run -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 + dvc add data/foo + dvc run -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1 dvc_info "Checking status" dvc status | grep "new file" || dvc_fail diff --git a/tests/test_checkout.py b/tests/test_checkout.py index 7d4954f3a..57ef09b7d 100644 --- a/tests/test_checkout.py +++ b/tests/test_checkout.py @@ -14,15 +14,15 @@ from tests.basic_env import TestDvc class TestCheckout(TestDvc): def test_checkout(self): cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out') - deps = [self.FOO] - deps_no_cache = [self.CODE] + deps = [self.FOO, self.CODE] outs = [os.path.join(self.dvc.root_dir, 'out')] outs_no_cache = [] locked = False fname = os.path.join(self.dvc.root_dir, 'out.dvc') cwd = os.curdir - stage = self.dvc.run(cmd, deps, deps_no_cache, outs, outs_no_cache, locked, fname, cwd) + self.dvc.add(self.FOO) + stage = self.dvc.run(cmd, deps, outs, outs_no_cache, locked, fname, cwd) shutil.copy(self.FOO, 'orig') os.chmod(self.FOO, stat.S_IWRITE) diff --git a/tests/test_repro.py b/tests/test_repro.py index 7a7359197..c5b7d556a 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -14,7 +14,7 @@ class ReproBasicEnv(TestDvc): self.create(self.file1_code_file, 'print("Hello")' + os.linesep + 'print("Mary")') stage = self.dvc.run(fname='file1.dvc', outs=[self.file_name1], - deps_no_cache=[self.file1_code_file], + deps=[self.file1_code_file], cmd='python {} --not-repro > {}'.format(self.file1_code_file, self.file_name1)) @@ -25,8 +25,7 @@ class ReproBasicEnv(TestDvc): stage = self.dvc.run(fname='file11.dvc', outs=[self.file_name11], - deps=[self.file_name1], - deps_no_cache=[self.file11_code_file], + deps=[self.file_name1, self.file11_code_file], cmd='python {} {} > {}'.format(self.file11_code_file, self.file_name1, self.file_name11)) @@ -49,8 +48,7 @@ class ReproBasicEnv(TestDvc): self.file_name_res = os.path.join('data', 'file_res') stage = self.dvc.run(fname='file_res.dvc', outs=[self.file_name_res], - deps_no_cache=[self.file_res_code_file], - deps=[self.file_name11, self.file_name2], + deps=[self.file_res_code_file, self.file_name11, self.file_name2], cmd='python {} {} {} > {}'.format(self.file_res_code_file, self.file_name11, self.file_name2, @@ -87,7 +85,7 @@ class ReproChangedDependency(ReproBasicEnv): self.create(file1_code_file, 'print("Goodbye")' + os.linesep + 'print("Jack")') self.dvc.run(fname='file1.dvc', outs=[self.file_name1], - deps_no_cache=[file1_code_file], + deps=[file1_code_file], cmd='python {} --not-repro > {}'.format(file1_code_file, self.file_name1)) diff --git a/tests/test_run.py b/tests/test_run.py index 11c9666b8..fc984df7c 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -9,20 +9,26 @@ from tests.basic_env import TestDvc class TestRun(TestDvc): def test(self): cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out') - deps = [self.FOO] - deps_no_cache = [self.CODE] + deps = [self.FOO, self.CODE] outs = [os.path.join(self.dvc.root_dir, 'out')] outs_no_cache = [] locked = False fname = os.path.join(self.dvc.root_dir, 'out.dvc') cwd = os.curdir - stage = self.dvc.run(cmd, deps, deps_no_cache, outs, outs_no_cache, locked, fname, cwd) + self.dvc.add(self.FOO) + stage = self.dvc.run(cmd=cmd, + deps=deps, + outs=outs, + outs_no_cache=outs_no_cache, + locked=locked, + fname=fname, + cwd=cwd) self.assertTrue(filecmp.cmp(self.FOO, 'out')) self.assertTrue(os.path.isfile(stage.path)) self.assertEqual(stage.cmd, cmd) - self.assertEqual(len(stage.deps), len(deps + deps_no_cache)) + self.assertEqual(len(stage.deps), len(deps)) self.assertEqual(len(stage.outs), len(outs + outs_no_cache)) self.assertEqual(stage.outs[0].path, outs[0]) self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0]) @@ -32,4 +38,10 @@ class TestRun(TestDvc): class TestRunEmpty(TestDvc): def test(self): - self.dvc.run('', [], [], [], [], False, 'empty.dvc', os.curdir) + self.dvc.run(cmd='', + deps=[], + outs=[], + outs_no_cache=[], + locked=False, + fname='empty.dvc', + cwd=os.curdir)