From b3a6d70d9186faf451789409b04b16e71c0c7b44 Mon Sep 17 00:00:00 2001 From: username <45409043+vernt@users.noreply.github.com> Date: Sun, 2 Dec 2018 00:35:20 -0800 Subject: [PATCH] [WIP] Add dvc run --deterministic option. Tests currently failing. --- dvc/cli.py | 6 ++++++ dvc/command/run.py | 3 ++- dvc/project.py | 27 +++++++++++++++---------- dvc/stage.py | 50 +++++++++++++++++++++++++++++++++++++++++++++- tests/test_run.py | 37 ++++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 13 deletions(-) diff --git a/dvc/cli.py b/dvc/cli.py index 4f3ceb90b..833fddb3d 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -271,6 +271,12 @@ def parse_args(argv=None): help="Automatic 'yes' answer to all prompts. E.g. " "when '.dvc' file exists and dvc asks if you " "want to overwrite it.") + run_parser.add_argument( + '--deterministic', + action='store_true', + default=False, + help=("Use cached result if previously run with " + "current dependencies and command.")) run_parser.add_argument( 'command', nargs=argparse.REMAINDER, diff --git a/dvc/command/run.py b/dvc/command/run.py index 0fa2c2036..2a052c4ab 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -31,7 +31,8 @@ class CmdRun(CmdBase): deps=self.args.deps, fname=self.args.file, cwd=self.args.cwd, - no_exec=self.args.no_exec) + no_exec=self.args.no_exec, + deterministic=self.args.deterministic) except DvcException as ex: self.project.logger.error('Failed to run command', ex) return 1 diff --git a/dvc/project.py b/dvc/project.py index 7703dea5c..4e10598df 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -363,18 +363,23 @@ class Project(object): fname=None, cwd=os.curdir, no_exec=False, - overwrite=False): - from dvc.stage import Stage + overwrite=False, + deterministic=False): - stage = Stage.loads(project=self, - fname=fname, - cmd=cmd, - cwd=cwd, - outs=outs, - outs_no_cache=outs_no_cache, - metrics_no_cache=metrics_no_cache, - deps=deps, - overwrite=overwrite) + with self.state: + stage = Stage.loads(project=self, + fname=fname, + cmd=cmd, + cwd=cwd, + outs=outs, + outs_no_cache=outs_no_cache, + metrics_no_cache=metrics_no_cache, + deps=deps, + overwrite=overwrite, + deterministic=deterministic) + + if stage is None: + return None all_stages = self.stages() diff --git a/dvc/stage.py b/dvc/stage.py index 30cd5b085..f9aa418cc 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -1,3 +1,4 @@ +import collections import os import yaml import posixpath @@ -70,6 +71,42 @@ class MissingDataSource(DvcException): msg = u'missing data {}: {}'.format(source, ', '.join(missing_files)) super(MissingDataSource, self).__init__(msg) +def _identify_dicts(dicts): + return collections.Counter(frozenset(d.items()) for d in dicts) + + +def _deps_equal(old, new): + old_identities = _identify_dicts(dep.dumpd() for dep in old.deps) + new_identities = _identify_dicts(dep.dumpd() for dep in new.deps) + return old_identities == new_identities + + +def _outs_equal(old, new): + old_dicts = [out.dumpd() for out in old.outs] + new_dicts = [out.dumpd() for out in new.outs] + for d in old_dicts: + del d['md5'] + old_identities = _identify_dicts(old_dicts) + new_identities = _identify_dicts(new_dicts) + return old_identities == new_identities + + +def _stage_equal(old, new): + deps_unchanged = all(dep.changed() is False for dep in old.deps) + if not deps_unchanged: + Logger.debug('Deps changed') + return False + if not _deps_equal(old, new): + Logger.debug('Deps unequal') + return False + + if not _outs_equal(old, new): + Logger.debug('Outs unequal') + return False + + Logger.debug('stage equal') + return True + class Stage(object): STAGE_FILE = 'Dvcfile' @@ -301,7 +338,9 @@ class Stage(object): cwd=os.curdir, locked=False, add=False, - overwrite=True): + overwrite=True, + deterministic=False): + stage = Stage(project=project, cwd=cwd, cmd=cmd, @@ -326,6 +365,14 @@ class Stage(object): path = os.path.join(cwd, fname) if os.path.exists(path): + + old = Stage.load(project=project, fname=fname) + for dep in stage.deps: + dep.save() + if deterministic and _stage_equal(old, stage): + Logger.info('Stage is cached, skipping.') + return None + relpath = os.path.relpath(path) msg = "'{}' already exists. " \ "Do you wish to run the command and overwrite it?" @@ -333,6 +380,7 @@ class Stage(object): and not project.prompt.prompt(msg.format(relpath), False): raise DvcException("'{}' already exists".format(relpath)) + stage.cwd = cwd stage.path = path diff --git a/tests/test_run.py b/tests/test_run.py index 2e5f0a932..ac575f26b 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -180,3 +180,40 @@ class TestCmdRun(TestDvc): def test_keyboard_interrupt(self, mock_popen): ret = main(['run', 'mycmd']) self.assertEqual(ret, 252) + + +class TestRunDeterministic(TestDvc): + + def test(self): + + def _run(target, deterministic=False): + + command = [ + 'dvc', 'run', '--verbose', '-f', '{}.dvc'.format(target), '-d', 'file1', '-o', target, + 'python', '-c', shlex.quote("open('{}', 'w').write('hi')").format(target), + ] + if deterministic: + command.insert(2, '--deterministic') + + return command + + + + + subprocess.check_call([ + 'dvc', 'run', '--verbose', '-f', 'f1.dvc', '-o', 'file1', + 'bash', '-c', '"echo hi > file1"', + ]) + + subprocess.check_call(_run('file2', deterministic=True)) + + # Does not run because its inputs already exist. + subprocess.check_call(_run('file3', deterministic=True)) + + subprocess.check_call(_run('file4')) + + self.assertTrue(os.path.exists('file1')) + self.assertTrue(os.path.exists('file2')) + self.assertTrue(os.path.exists('file4')) + + self.assertFalse(os.path.exists('file3'))