mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-21 15:28:29 +01:00
[WIP] Add dvc run --deterministic option.
Tests currently failing.
This commit is contained in:
@@ -271,6 +271,12 @@ def parse_args(argv=None):
|
||||
help="Automatic 'yes' answer to all prompts. E.g. "
|
||||
"when '.dvc' file exists and dvc asks if you "
|
||||
"want to overwrite it.")
|
||||
run_parser.add_argument(
|
||||
'--deterministic',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help=("Use cached result if previously run with "
|
||||
"current dependencies and command."))
|
||||
run_parser.add_argument(
|
||||
'command',
|
||||
nargs=argparse.REMAINDER,
|
||||
|
||||
+2
-1
@@ -31,7 +31,8 @@ class CmdRun(CmdBase):
|
||||
deps=self.args.deps,
|
||||
fname=self.args.file,
|
||||
cwd=self.args.cwd,
|
||||
no_exec=self.args.no_exec)
|
||||
no_exec=self.args.no_exec,
|
||||
deterministic=self.args.deterministic)
|
||||
except DvcException as ex:
|
||||
self.project.logger.error('Failed to run command', ex)
|
||||
return 1
|
||||
|
||||
+16
-11
@@ -363,18 +363,23 @@ class Project(object):
|
||||
fname=None,
|
||||
cwd=os.curdir,
|
||||
no_exec=False,
|
||||
overwrite=False):
|
||||
from dvc.stage import Stage
|
||||
overwrite=False,
|
||||
deterministic=False):
|
||||
|
||||
stage = Stage.loads(project=self,
|
||||
fname=fname,
|
||||
cmd=cmd,
|
||||
cwd=cwd,
|
||||
outs=outs,
|
||||
outs_no_cache=outs_no_cache,
|
||||
metrics_no_cache=metrics_no_cache,
|
||||
deps=deps,
|
||||
overwrite=overwrite)
|
||||
with self.state:
|
||||
stage = Stage.loads(project=self,
|
||||
fname=fname,
|
||||
cmd=cmd,
|
||||
cwd=cwd,
|
||||
outs=outs,
|
||||
outs_no_cache=outs_no_cache,
|
||||
metrics_no_cache=metrics_no_cache,
|
||||
deps=deps,
|
||||
overwrite=overwrite,
|
||||
deterministic=deterministic)
|
||||
|
||||
if stage is None:
|
||||
return None
|
||||
|
||||
all_stages = self.stages()
|
||||
|
||||
|
||||
+49
-1
@@ -1,3 +1,4 @@
|
||||
import collections
|
||||
import os
|
||||
import yaml
|
||||
import posixpath
|
||||
@@ -70,6 +71,42 @@ class MissingDataSource(DvcException):
|
||||
msg = u'missing data {}: {}'.format(source, ', '.join(missing_files))
|
||||
super(MissingDataSource, self).__init__(msg)
|
||||
|
||||
def _identify_dicts(dicts):
|
||||
return collections.Counter(frozenset(d.items()) for d in dicts)
|
||||
|
||||
|
||||
def _deps_equal(old, new):
|
||||
old_identities = _identify_dicts(dep.dumpd() for dep in old.deps)
|
||||
new_identities = _identify_dicts(dep.dumpd() for dep in new.deps)
|
||||
return old_identities == new_identities
|
||||
|
||||
|
||||
def _outs_equal(old, new):
|
||||
old_dicts = [out.dumpd() for out in old.outs]
|
||||
new_dicts = [out.dumpd() for out in new.outs]
|
||||
for d in old_dicts:
|
||||
del d['md5']
|
||||
old_identities = _identify_dicts(old_dicts)
|
||||
new_identities = _identify_dicts(new_dicts)
|
||||
return old_identities == new_identities
|
||||
|
||||
|
||||
def _stage_equal(old, new):
|
||||
deps_unchanged = all(dep.changed() is False for dep in old.deps)
|
||||
if not deps_unchanged:
|
||||
Logger.debug('Deps changed')
|
||||
return False
|
||||
if not _deps_equal(old, new):
|
||||
Logger.debug('Deps unequal')
|
||||
return False
|
||||
|
||||
if not _outs_equal(old, new):
|
||||
Logger.debug('Outs unequal')
|
||||
return False
|
||||
|
||||
Logger.debug('stage equal')
|
||||
return True
|
||||
|
||||
|
||||
class Stage(object):
|
||||
STAGE_FILE = 'Dvcfile'
|
||||
@@ -301,7 +338,9 @@ class Stage(object):
|
||||
cwd=os.curdir,
|
||||
locked=False,
|
||||
add=False,
|
||||
overwrite=True):
|
||||
overwrite=True,
|
||||
deterministic=False):
|
||||
|
||||
stage = Stage(project=project,
|
||||
cwd=cwd,
|
||||
cmd=cmd,
|
||||
@@ -326,6 +365,14 @@ class Stage(object):
|
||||
path = os.path.join(cwd, fname)
|
||||
|
||||
if os.path.exists(path):
|
||||
|
||||
old = Stage.load(project=project, fname=fname)
|
||||
for dep in stage.deps:
|
||||
dep.save()
|
||||
if deterministic and _stage_equal(old, stage):
|
||||
Logger.info('Stage is cached, skipping.')
|
||||
return None
|
||||
|
||||
relpath = os.path.relpath(path)
|
||||
msg = "'{}' already exists. " \
|
||||
"Do you wish to run the command and overwrite it?"
|
||||
@@ -333,6 +380,7 @@ class Stage(object):
|
||||
and not project.prompt.prompt(msg.format(relpath), False):
|
||||
raise DvcException("'{}' already exists".format(relpath))
|
||||
|
||||
|
||||
stage.cwd = cwd
|
||||
stage.path = path
|
||||
|
||||
|
||||
@@ -180,3 +180,40 @@ class TestCmdRun(TestDvc):
|
||||
def test_keyboard_interrupt(self, mock_popen):
|
||||
ret = main(['run', 'mycmd'])
|
||||
self.assertEqual(ret, 252)
|
||||
|
||||
|
||||
class TestRunDeterministic(TestDvc):
|
||||
|
||||
def test(self):
|
||||
|
||||
def _run(target, deterministic=False):
|
||||
|
||||
command = [
|
||||
'dvc', 'run', '--verbose', '-f', '{}.dvc'.format(target), '-d', 'file1', '-o', target,
|
||||
'python', '-c', shlex.quote("open('{}', 'w').write('hi')").format(target),
|
||||
]
|
||||
if deterministic:
|
||||
command.insert(2, '--deterministic')
|
||||
|
||||
return command
|
||||
|
||||
|
||||
|
||||
|
||||
subprocess.check_call([
|
||||
'dvc', 'run', '--verbose', '-f', 'f1.dvc', '-o', 'file1',
|
||||
'bash', '-c', '"echo hi > file1"',
|
||||
])
|
||||
|
||||
subprocess.check_call(_run('file2', deterministic=True))
|
||||
|
||||
# Does not run because its inputs already exist.
|
||||
subprocess.check_call(_run('file3', deterministic=True))
|
||||
|
||||
subprocess.check_call(_run('file4'))
|
||||
|
||||
self.assertTrue(os.path.exists('file1'))
|
||||
self.assertTrue(os.path.exists('file2'))
|
||||
self.assertTrue(os.path.exists('file4'))
|
||||
|
||||
self.assertFalse(os.path.exists('file3'))
|
||||
|
||||
Reference in New Issue
Block a user