[WIP] Add dvc run --deterministic option.

Tests currently failing.
This commit is contained in:
username
2018-12-02 00:35:20 -08:00
parent b45e22e2da
commit b3a6d70d91
5 changed files with 110 additions and 13 deletions
+6
View File
@@ -271,6 +271,12 @@ def parse_args(argv=None):
help="Automatic 'yes' answer to all prompts. E.g. "
"when '.dvc' file exists and dvc asks if you "
"want to overwrite it.")
run_parser.add_argument(
'--deterministic',
action='store_true',
default=False,
help=("Use cached result if previously run with "
"current dependencies and command."))
run_parser.add_argument(
'command',
nargs=argparse.REMAINDER,
+2 -1
View File
@@ -31,7 +31,8 @@ class CmdRun(CmdBase):
deps=self.args.deps,
fname=self.args.file,
cwd=self.args.cwd,
no_exec=self.args.no_exec)
no_exec=self.args.no_exec,
deterministic=self.args.deterministic)
except DvcException as ex:
self.project.logger.error('Failed to run command', ex)
return 1
+16 -11
View File
@@ -363,18 +363,23 @@ class Project(object):
fname=None,
cwd=os.curdir,
no_exec=False,
overwrite=False):
from dvc.stage import Stage
overwrite=False,
deterministic=False):
stage = Stage.loads(project=self,
fname=fname,
cmd=cmd,
cwd=cwd,
outs=outs,
outs_no_cache=outs_no_cache,
metrics_no_cache=metrics_no_cache,
deps=deps,
overwrite=overwrite)
with self.state:
stage = Stage.loads(project=self,
fname=fname,
cmd=cmd,
cwd=cwd,
outs=outs,
outs_no_cache=outs_no_cache,
metrics_no_cache=metrics_no_cache,
deps=deps,
overwrite=overwrite,
deterministic=deterministic)
if stage is None:
return None
all_stages = self.stages()
+49 -1
View File
@@ -1,3 +1,4 @@
import collections
import os
import yaml
import posixpath
@@ -70,6 +71,42 @@ class MissingDataSource(DvcException):
msg = u'missing data {}: {}'.format(source, ', '.join(missing_files))
super(MissingDataSource, self).__init__(msg)
def _identify_dicts(dicts):
return collections.Counter(frozenset(d.items()) for d in dicts)
def _deps_equal(old, new):
old_identities = _identify_dicts(dep.dumpd() for dep in old.deps)
new_identities = _identify_dicts(dep.dumpd() for dep in new.deps)
return old_identities == new_identities
def _outs_equal(old, new):
old_dicts = [out.dumpd() for out in old.outs]
new_dicts = [out.dumpd() for out in new.outs]
for d in old_dicts:
del d['md5']
old_identities = _identify_dicts(old_dicts)
new_identities = _identify_dicts(new_dicts)
return old_identities == new_identities
def _stage_equal(old, new):
deps_unchanged = all(dep.changed() is False for dep in old.deps)
if not deps_unchanged:
Logger.debug('Deps changed')
return False
if not _deps_equal(old, new):
Logger.debug('Deps unequal')
return False
if not _outs_equal(old, new):
Logger.debug('Outs unequal')
return False
Logger.debug('stage equal')
return True
class Stage(object):
STAGE_FILE = 'Dvcfile'
@@ -301,7 +338,9 @@ class Stage(object):
cwd=os.curdir,
locked=False,
add=False,
overwrite=True):
overwrite=True,
deterministic=False):
stage = Stage(project=project,
cwd=cwd,
cmd=cmd,
@@ -326,6 +365,14 @@ class Stage(object):
path = os.path.join(cwd, fname)
if os.path.exists(path):
old = Stage.load(project=project, fname=fname)
for dep in stage.deps:
dep.save()
if deterministic and _stage_equal(old, stage):
Logger.info('Stage is cached, skipping.')
return None
relpath = os.path.relpath(path)
msg = "'{}' already exists. " \
"Do you wish to run the command and overwrite it?"
@@ -333,6 +380,7 @@ class Stage(object):
and not project.prompt.prompt(msg.format(relpath), False):
raise DvcException("'{}' already exists".format(relpath))
stage.cwd = cwd
stage.path = path
+37
View File
@@ -180,3 +180,40 @@ class TestCmdRun(TestDvc):
def test_keyboard_interrupt(self, mock_popen):
ret = main(['run', 'mycmd'])
self.assertEqual(ret, 252)
class TestRunDeterministic(TestDvc):
def test(self):
def _run(target, deterministic=False):
command = [
'dvc', 'run', '--verbose', '-f', '{}.dvc'.format(target), '-d', 'file1', '-o', target,
'python', '-c', shlex.quote("open('{}', 'w').write('hi')").format(target),
]
if deterministic:
command.insert(2, '--deterministic')
return command
subprocess.check_call([
'dvc', 'run', '--verbose', '-f', 'f1.dvc', '-o', 'file1',
'bash', '-c', '"echo hi > file1"',
])
subprocess.check_call(_run('file2', deterministic=True))
# Does not run because its inputs already exist.
subprocess.check_call(_run('file3', deterministic=True))
subprocess.check_call(_run('file4'))
self.assertTrue(os.path.exists('file1'))
self.assertTrue(os.path.exists('file2'))
self.assertTrue(os.path.exists('file4'))
self.assertFalse(os.path.exists('file3'))