dvc: remove -D option

Fixes #370

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-01-04 13:34:31 +02:00
parent 9bec404efa
commit 14181ede5f
11 changed files with 60 additions and 60 deletions
-5
View File
@@ -96,11 +96,6 @@ def parse_args(argv=None):
action='append',
default=[],
help='Declare dependencies for reproducible cmd.')
run_parser.add_argument('-D',
'--deps-no-cache',
action='append',
default=[],
help='Declare dependencies that should not be cached for reproducible cmd.')
run_parser.add_argument('-o',
'--outs',
action='append',
-1
View File
@@ -13,7 +13,6 @@ class CmdRun(CmdBase):
outs=self.args.outs,
outs_no_cache=self.args.outs_no_cache,
deps=self.args.deps,
deps_no_cache=self.args.deps_no_cache,
locked=self.args.lock,
fname=fname,
cwd=self.args.cwd)
+1 -13
View File
@@ -136,17 +136,9 @@ class Project(object):
return stages
def _add_orphans(self, deps):
outs = [out.path for out in self.outs()]
for dep in deps:
if not dep.use_cache or dep.path in outs:
continue
self.add(dep.path)
def run(self,
cmd=None,
deps=[],
deps_no_cache=[],
outs=[],
outs_no_cache=[],
locked=False,
@@ -156,11 +148,7 @@ class Project(object):
path = os.path.join(cwd, fname)
outputs = Output.loads_from(self, outs, use_cache=True, cwd=cwd)
outputs += Output.loads_from(self, outs_no_cache, use_cache=False, cwd=cwd)
deps = Dependency.loads_from(self, deps, use_cache=True, cwd=cwd)
deps += Dependency.loads_from(self, deps_no_cache, use_cache=False, cwd=cwd)
# Add files that were specified as cached dependencies and weren't added before
self._add_orphans(deps)
deps = Dependency.loads_from(self, deps, use_cache=False, cwd=cwd)
stage = Stage(project=self,
path=path,
+24 -20
View File
@@ -49,7 +49,13 @@ class Output(object):
self.use_cache = use_cache
def _changed_md5(self):
return self.md5 != file_md5(self.path)[0]
state = self.project.state.get(self.path)
if state and state.mtime == self.mtime():
md5 = state.md5
else:
md5 = file_md5(self.path)[0]
return self.md5 != md5
def changed(self):
if not self.use_cache:
@@ -107,15 +113,21 @@ class Output(object):
def mtime(self):
return os.path.getmtime(self.path)
def update(self, md5=None):
self.md5 = md5
if not self.md5:
if not os.path.exists(self.path):
raise OutputDoesNotExistError(self.path)
if not os.path.isfile(self.path):
raise OutputIsNotFileError(self.path)
def update(self):
if not os.path.exists(self.path):
raise OutputDoesNotExistError(self.path)
if not os.path.isfile(self.path):
raise OutputIsNotFileError(self.path)
state = self.project.state.get(self.path)
if state and state.mtime == self.mtime():
md5 = state.md5
msg = '{} using md5 {} from state file'
self.project.logger.debug(msg.format(self.path, md5))
self.md5 = md5
else:
self.md5 = file_md5(self.path)[0]
self.project.state.update(self.path, self.md5, self.mtime())
self.project.state.update(self.path, self.md5, self.mtime())
def save(self):
if not self.use_cache:
@@ -134,8 +146,8 @@ class Output(object):
@classmethod
def loadd(cls, project, d, cwd='.'):
path = os.path.join(cwd, d[Output.PARAM_PATH])
md5 = d[Output.PARAM_MD5]
use_cache = d[Output.PARAM_CACHE]
md5 = d.get(Output.PARAM_MD5, None)
use_cache = d.get(Output.PARAM_CACHE, False)
return cls(project, path, md5, use_cache=use_cache)
@classmethod
@@ -159,15 +171,7 @@ class Output(object):
class Dependency(Output):
def update(self):
md5 = None
state = self.project.state.get(self.path)
if state and state.mtime == self.mtime():
md5 = state.md5
msg = '{} using md5 {} from state file for dependency'
self.project.logger.debug(msg.format(self.path, md5))
super(Dependency, self).update(md5=md5)
pass
class Stage(object):
+1 -1
View File
@@ -6,7 +6,7 @@ source common.sh
dvc_create_repo
dvc run -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
dvc add data/foo
rm -f data/foo
dvc checkout
+2 -1
View File
@@ -6,7 +6,8 @@ source common.sh
dvc_create_repo
dvc run -f Dvcfile -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
dvc add data/foo
dvc run -f Dvcfile -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
cat Dvcfile | grep 'locked' | grep 'false' || dvc_fatal "stage locked after 'dvc run'"
dvc lock
cat Dvcfile | grep 'locked' | grep 'true' || dvc_fatal "stage not locked after 'dvc lock'"
+6 -4
View File
@@ -10,7 +10,8 @@ function test_generic() {
dvc_create_repo
dvc_info 'Copy foo into foo1'
dvc run -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
dvc add data/foo
dvc run -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
dvc_info 'Modify code'
echo " " >> code/code.sh
@@ -47,9 +48,10 @@ function test_partial() {
git commit -m "copy code"
dvc_info "Create repro chain foo -> foo1 -> foo2 -> foo3"
dvc run -f copy_foo_foo1.dvc -D code/code1.sh -d data/foo -o data/foo1 cp data/foo data/foo1
dvc run -f copy_foo1_foo2.dvc -D code/code2.sh -d data/foo1 -o data/foo2 cp data/foo1 data/foo2
dvc run -f copy_foo2_foo3.dvc -D code/code3.sh -d data/foo2 -o data/foo3 cp data/foo2 data/foo3
dvc add data/foo
dvc run -f copy_foo_foo1.dvc -d code/code1.sh -d data/foo -o data/foo1 cp data/foo data/foo1
dvc run -f copy_foo1_foo2.dvc -d code/code2.sh -d data/foo1 -o data/foo2 cp data/foo1 data/foo2
dvc run -f copy_foo2_foo3.dvc -d code/code3.sh -d data/foo2 -o data/foo3 cp data/foo2 data/foo3
dvc_info "Save original timestamps"
FOO_TS=$(dvc_timestamp data/foo)
+2 -1
View File
@@ -5,7 +5,8 @@ set -e
source common.sh
function test_sync() {
dvc run -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
dvc add data/foo
dvc run -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
dvc_info "Checking status"
dvc status | grep "new file" || dvc_fail
+3 -3
View File
@@ -14,15 +14,15 @@ from tests.basic_env import TestDvc
class TestCheckout(TestDvc):
def test_checkout(self):
cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
deps = [self.FOO]
deps_no_cache = [self.CODE]
deps = [self.FOO, self.CODE]
outs = [os.path.join(self.dvc.root_dir, 'out')]
outs_no_cache = []
locked = False
fname = os.path.join(self.dvc.root_dir, 'out.dvc')
cwd = os.curdir
stage = self.dvc.run(cmd, deps, deps_no_cache, outs, outs_no_cache, locked, fname, cwd)
self.dvc.add(self.FOO)
stage = self.dvc.run(cmd, deps, outs, outs_no_cache, locked, fname, cwd)
shutil.copy(self.FOO, 'orig')
os.chmod(self.FOO, stat.S_IWRITE)
+4 -6
View File
@@ -14,7 +14,7 @@ class ReproBasicEnv(TestDvc):
self.create(self.file1_code_file, 'print("Hello")' + os.linesep + 'print("Mary")')
stage = self.dvc.run(fname='file1.dvc',
outs=[self.file_name1],
deps_no_cache=[self.file1_code_file],
deps=[self.file1_code_file],
cmd='python {} --not-repro > {}'.format(self.file1_code_file,
self.file_name1))
@@ -25,8 +25,7 @@ class ReproBasicEnv(TestDvc):
stage = self.dvc.run(fname='file11.dvc',
outs=[self.file_name11],
deps=[self.file_name1],
deps_no_cache=[self.file11_code_file],
deps=[self.file_name1, self.file11_code_file],
cmd='python {} {} > {}'.format(self.file11_code_file,
self.file_name1,
self.file_name11))
@@ -49,8 +48,7 @@ class ReproBasicEnv(TestDvc):
self.file_name_res = os.path.join('data', 'file_res')
stage = self.dvc.run(fname='file_res.dvc',
outs=[self.file_name_res],
deps_no_cache=[self.file_res_code_file],
deps=[self.file_name11, self.file_name2],
deps=[self.file_res_code_file, self.file_name11, self.file_name2],
cmd='python {} {} {} > {}'.format(self.file_res_code_file,
self.file_name11,
self.file_name2,
@@ -87,7 +85,7 @@ class ReproChangedDependency(ReproBasicEnv):
self.create(file1_code_file, 'print("Goodbye")' + os.linesep + 'print("Jack")')
self.dvc.run(fname='file1.dvc',
outs=[self.file_name1],
deps_no_cache=[file1_code_file],
deps=[file1_code_file],
cmd='python {} --not-repro > {}'.format(file1_code_file, self.file_name1))
+17 -5
View File
@@ -9,20 +9,26 @@ from tests.basic_env import TestDvc
class TestRun(TestDvc):
def test(self):
cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
deps = [self.FOO]
deps_no_cache = [self.CODE]
deps = [self.FOO, self.CODE]
outs = [os.path.join(self.dvc.root_dir, 'out')]
outs_no_cache = []
locked = False
fname = os.path.join(self.dvc.root_dir, 'out.dvc')
cwd = os.curdir
stage = self.dvc.run(cmd, deps, deps_no_cache, outs, outs_no_cache, locked, fname, cwd)
self.dvc.add(self.FOO)
stage = self.dvc.run(cmd=cmd,
deps=deps,
outs=outs,
outs_no_cache=outs_no_cache,
locked=locked,
fname=fname,
cwd=cwd)
self.assertTrue(filecmp.cmp(self.FOO, 'out'))
self.assertTrue(os.path.isfile(stage.path))
self.assertEqual(stage.cmd, cmd)
self.assertEqual(len(stage.deps), len(deps + deps_no_cache))
self.assertEqual(len(stage.deps), len(deps))
self.assertEqual(len(stage.outs), len(outs + outs_no_cache))
self.assertEqual(stage.outs[0].path, outs[0])
self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0])
@@ -32,4 +38,10 @@ class TestRun(TestDvc):
class TestRunEmpty(TestDvc):
def test(self):
self.dvc.run('', [], [], [], [], False, 'empty.dvc', os.curdir)
self.dvc.run(cmd='',
deps=[],
outs=[],
outs_no_cache=[],
locked=False,
fname='empty.dvc',
cwd=os.curdir)