mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-17 15:28:28 +01:00
dvc: remove -D option
Fixes #370 Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
@@ -96,11 +96,6 @@ def parse_args(argv=None):
|
||||
action='append',
|
||||
default=[],
|
||||
help='Declare dependencies for reproducible cmd.')
|
||||
run_parser.add_argument('-D',
|
||||
'--deps-no-cache',
|
||||
action='append',
|
||||
default=[],
|
||||
help='Declare dependencies that should not be cached for reproducible cmd.')
|
||||
run_parser.add_argument('-o',
|
||||
'--outs',
|
||||
action='append',
|
||||
|
||||
@@ -13,7 +13,6 @@ class CmdRun(CmdBase):
|
||||
outs=self.args.outs,
|
||||
outs_no_cache=self.args.outs_no_cache,
|
||||
deps=self.args.deps,
|
||||
deps_no_cache=self.args.deps_no_cache,
|
||||
locked=self.args.lock,
|
||||
fname=fname,
|
||||
cwd=self.args.cwd)
|
||||
|
||||
+1
-13
@@ -136,17 +136,9 @@ class Project(object):
|
||||
|
||||
return stages
|
||||
|
||||
def _add_orphans(self, deps):
|
||||
outs = [out.path for out in self.outs()]
|
||||
for dep in deps:
|
||||
if not dep.use_cache or dep.path in outs:
|
||||
continue
|
||||
self.add(dep.path)
|
||||
|
||||
def run(self,
|
||||
cmd=None,
|
||||
deps=[],
|
||||
deps_no_cache=[],
|
||||
outs=[],
|
||||
outs_no_cache=[],
|
||||
locked=False,
|
||||
@@ -156,11 +148,7 @@ class Project(object):
|
||||
path = os.path.join(cwd, fname)
|
||||
outputs = Output.loads_from(self, outs, use_cache=True, cwd=cwd)
|
||||
outputs += Output.loads_from(self, outs_no_cache, use_cache=False, cwd=cwd)
|
||||
deps = Dependency.loads_from(self, deps, use_cache=True, cwd=cwd)
|
||||
deps += Dependency.loads_from(self, deps_no_cache, use_cache=False, cwd=cwd)
|
||||
|
||||
# Add files that were specified as cached dependencies and weren't added before
|
||||
self._add_orphans(deps)
|
||||
deps = Dependency.loads_from(self, deps, use_cache=False, cwd=cwd)
|
||||
|
||||
stage = Stage(project=self,
|
||||
path=path,
|
||||
|
||||
+24
-20
@@ -49,7 +49,13 @@ class Output(object):
|
||||
self.use_cache = use_cache
|
||||
|
||||
def _changed_md5(self):
|
||||
return self.md5 != file_md5(self.path)[0]
|
||||
state = self.project.state.get(self.path)
|
||||
if state and state.mtime == self.mtime():
|
||||
md5 = state.md5
|
||||
else:
|
||||
md5 = file_md5(self.path)[0]
|
||||
|
||||
return self.md5 != md5
|
||||
|
||||
def changed(self):
|
||||
if not self.use_cache:
|
||||
@@ -107,15 +113,21 @@ class Output(object):
|
||||
def mtime(self):
|
||||
return os.path.getmtime(self.path)
|
||||
|
||||
def update(self, md5=None):
|
||||
self.md5 = md5
|
||||
if not self.md5:
|
||||
if not os.path.exists(self.path):
|
||||
raise OutputDoesNotExistError(self.path)
|
||||
if not os.path.isfile(self.path):
|
||||
raise OutputIsNotFileError(self.path)
|
||||
def update(self):
|
||||
if not os.path.exists(self.path):
|
||||
raise OutputDoesNotExistError(self.path)
|
||||
if not os.path.isfile(self.path):
|
||||
raise OutputIsNotFileError(self.path)
|
||||
|
||||
state = self.project.state.get(self.path)
|
||||
if state and state.mtime == self.mtime():
|
||||
md5 = state.md5
|
||||
msg = '{} using md5 {} from state file'
|
||||
self.project.logger.debug(msg.format(self.path, md5))
|
||||
self.md5 = md5
|
||||
else:
|
||||
self.md5 = file_md5(self.path)[0]
|
||||
self.project.state.update(self.path, self.md5, self.mtime())
|
||||
self.project.state.update(self.path, self.md5, self.mtime())
|
||||
|
||||
def save(self):
|
||||
if not self.use_cache:
|
||||
@@ -134,8 +146,8 @@ class Output(object):
|
||||
@classmethod
|
||||
def loadd(cls, project, d, cwd='.'):
|
||||
path = os.path.join(cwd, d[Output.PARAM_PATH])
|
||||
md5 = d[Output.PARAM_MD5]
|
||||
use_cache = d[Output.PARAM_CACHE]
|
||||
md5 = d.get(Output.PARAM_MD5, None)
|
||||
use_cache = d.get(Output.PARAM_CACHE, False)
|
||||
return cls(project, path, md5, use_cache=use_cache)
|
||||
|
||||
@classmethod
|
||||
@@ -159,15 +171,7 @@ class Output(object):
|
||||
|
||||
|
||||
class Dependency(Output):
|
||||
def update(self):
|
||||
md5 = None
|
||||
state = self.project.state.get(self.path)
|
||||
if state and state.mtime == self.mtime():
|
||||
md5 = state.md5
|
||||
msg = '{} using md5 {} from state file for dependency'
|
||||
self.project.logger.debug(msg.format(self.path, md5))
|
||||
|
||||
super(Dependency, self).update(md5=md5)
|
||||
pass
|
||||
|
||||
|
||||
class Stage(object):
|
||||
|
||||
@@ -6,7 +6,7 @@ source common.sh
|
||||
|
||||
dvc_create_repo
|
||||
|
||||
dvc run -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
|
||||
dvc add data/foo
|
||||
|
||||
rm -f data/foo
|
||||
dvc checkout
|
||||
|
||||
@@ -6,7 +6,8 @@ source common.sh
|
||||
|
||||
dvc_create_repo
|
||||
|
||||
dvc run -f Dvcfile -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
|
||||
dvc add data/foo
|
||||
dvc run -f Dvcfile -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
|
||||
cat Dvcfile | grep 'locked' | grep 'false' || dvc_fatal "stage locked after 'dvc run'"
|
||||
dvc lock
|
||||
cat Dvcfile | grep 'locked' | grep 'true' || dvc_fatal "stage not locked after 'dvc lock'"
|
||||
|
||||
@@ -10,7 +10,8 @@ function test_generic() {
|
||||
dvc_create_repo
|
||||
|
||||
dvc_info 'Copy foo into foo1'
|
||||
dvc run -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
|
||||
dvc add data/foo
|
||||
dvc run -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
|
||||
|
||||
dvc_info 'Modify code'
|
||||
echo " " >> code/code.sh
|
||||
@@ -47,9 +48,10 @@ function test_partial() {
|
||||
git commit -m "copy code"
|
||||
|
||||
dvc_info "Create repro chain foo -> foo1 -> foo2 -> foo3"
|
||||
dvc run -f copy_foo_foo1.dvc -D code/code1.sh -d data/foo -o data/foo1 cp data/foo data/foo1
|
||||
dvc run -f copy_foo1_foo2.dvc -D code/code2.sh -d data/foo1 -o data/foo2 cp data/foo1 data/foo2
|
||||
dvc run -f copy_foo2_foo3.dvc -D code/code3.sh -d data/foo2 -o data/foo3 cp data/foo2 data/foo3
|
||||
dvc add data/foo
|
||||
dvc run -f copy_foo_foo1.dvc -d code/code1.sh -d data/foo -o data/foo1 cp data/foo data/foo1
|
||||
dvc run -f copy_foo1_foo2.dvc -d code/code2.sh -d data/foo1 -o data/foo2 cp data/foo1 data/foo2
|
||||
dvc run -f copy_foo2_foo3.dvc -d code/code3.sh -d data/foo2 -o data/foo3 cp data/foo2 data/foo3
|
||||
|
||||
dvc_info "Save original timestamps"
|
||||
FOO_TS=$(dvc_timestamp data/foo)
|
||||
|
||||
@@ -5,7 +5,8 @@ set -e
|
||||
source common.sh
|
||||
|
||||
function test_sync() {
|
||||
dvc run -D code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
|
||||
dvc add data/foo
|
||||
dvc run -d code/code.sh -d data/foo -o data/foo1 bash code/code.sh data/foo data/foo1
|
||||
|
||||
dvc_info "Checking status"
|
||||
dvc status | grep "new file" || dvc_fail
|
||||
|
||||
@@ -14,15 +14,15 @@ from tests.basic_env import TestDvc
|
||||
class TestCheckout(TestDvc):
|
||||
def test_checkout(self):
|
||||
cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
|
||||
deps = [self.FOO]
|
||||
deps_no_cache = [self.CODE]
|
||||
deps = [self.FOO, self.CODE]
|
||||
outs = [os.path.join(self.dvc.root_dir, 'out')]
|
||||
outs_no_cache = []
|
||||
locked = False
|
||||
fname = os.path.join(self.dvc.root_dir, 'out.dvc')
|
||||
cwd = os.curdir
|
||||
|
||||
stage = self.dvc.run(cmd, deps, deps_no_cache, outs, outs_no_cache, locked, fname, cwd)
|
||||
self.dvc.add(self.FOO)
|
||||
stage = self.dvc.run(cmd, deps, outs, outs_no_cache, locked, fname, cwd)
|
||||
|
||||
shutil.copy(self.FOO, 'orig')
|
||||
os.chmod(self.FOO, stat.S_IWRITE)
|
||||
|
||||
+4
-6
@@ -14,7 +14,7 @@ class ReproBasicEnv(TestDvc):
|
||||
self.create(self.file1_code_file, 'print("Hello")' + os.linesep + 'print("Mary")')
|
||||
stage = self.dvc.run(fname='file1.dvc',
|
||||
outs=[self.file_name1],
|
||||
deps_no_cache=[self.file1_code_file],
|
||||
deps=[self.file1_code_file],
|
||||
cmd='python {} --not-repro > {}'.format(self.file1_code_file,
|
||||
self.file_name1))
|
||||
|
||||
@@ -25,8 +25,7 @@ class ReproBasicEnv(TestDvc):
|
||||
|
||||
stage = self.dvc.run(fname='file11.dvc',
|
||||
outs=[self.file_name11],
|
||||
deps=[self.file_name1],
|
||||
deps_no_cache=[self.file11_code_file],
|
||||
deps=[self.file_name1, self.file11_code_file],
|
||||
cmd='python {} {} > {}'.format(self.file11_code_file,
|
||||
self.file_name1,
|
||||
self.file_name11))
|
||||
@@ -49,8 +48,7 @@ class ReproBasicEnv(TestDvc):
|
||||
self.file_name_res = os.path.join('data', 'file_res')
|
||||
stage = self.dvc.run(fname='file_res.dvc',
|
||||
outs=[self.file_name_res],
|
||||
deps_no_cache=[self.file_res_code_file],
|
||||
deps=[self.file_name11, self.file_name2],
|
||||
deps=[self.file_res_code_file, self.file_name11, self.file_name2],
|
||||
cmd='python {} {} {} > {}'.format(self.file_res_code_file,
|
||||
self.file_name11,
|
||||
self.file_name2,
|
||||
@@ -87,7 +85,7 @@ class ReproChangedDependency(ReproBasicEnv):
|
||||
self.create(file1_code_file, 'print("Goodbye")' + os.linesep + 'print("Jack")')
|
||||
self.dvc.run(fname='file1.dvc',
|
||||
outs=[self.file_name1],
|
||||
deps_no_cache=[file1_code_file],
|
||||
deps=[file1_code_file],
|
||||
cmd='python {} --not-repro > {}'.format(file1_code_file, self.file_name1))
|
||||
|
||||
|
||||
|
||||
+17
-5
@@ -9,20 +9,26 @@ from tests.basic_env import TestDvc
|
||||
class TestRun(TestDvc):
|
||||
def test(self):
|
||||
cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
|
||||
deps = [self.FOO]
|
||||
deps_no_cache = [self.CODE]
|
||||
deps = [self.FOO, self.CODE]
|
||||
outs = [os.path.join(self.dvc.root_dir, 'out')]
|
||||
outs_no_cache = []
|
||||
locked = False
|
||||
fname = os.path.join(self.dvc.root_dir, 'out.dvc')
|
||||
cwd = os.curdir
|
||||
|
||||
stage = self.dvc.run(cmd, deps, deps_no_cache, outs, outs_no_cache, locked, fname, cwd)
|
||||
self.dvc.add(self.FOO)
|
||||
stage = self.dvc.run(cmd=cmd,
|
||||
deps=deps,
|
||||
outs=outs,
|
||||
outs_no_cache=outs_no_cache,
|
||||
locked=locked,
|
||||
fname=fname,
|
||||
cwd=cwd)
|
||||
|
||||
self.assertTrue(filecmp.cmp(self.FOO, 'out'))
|
||||
self.assertTrue(os.path.isfile(stage.path))
|
||||
self.assertEqual(stage.cmd, cmd)
|
||||
self.assertEqual(len(stage.deps), len(deps + deps_no_cache))
|
||||
self.assertEqual(len(stage.deps), len(deps))
|
||||
self.assertEqual(len(stage.outs), len(outs + outs_no_cache))
|
||||
self.assertEqual(stage.outs[0].path, outs[0])
|
||||
self.assertEqual(stage.outs[0].md5, file_md5(self.FOO)[0])
|
||||
@@ -32,4 +38,10 @@ class TestRun(TestDvc):
|
||||
|
||||
class TestRunEmpty(TestDvc):
|
||||
def test(self):
|
||||
self.dvc.run('', [], [], [], [], False, 'empty.dvc', os.curdir)
|
||||
self.dvc.run(cmd='',
|
||||
deps=[],
|
||||
outs=[],
|
||||
outs_no_cache=[],
|
||||
locked=False,
|
||||
fname='empty.dvc',
|
||||
cwd=os.curdir)
|
||||
|
||||
Reference in New Issue
Block a user