From ae5ac21bf8af7da62f36f0a7654775894936ba8b Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 3 Dec 2018 01:28:16 +0200 Subject: [PATCH] run: deterministic: copare the whole dicts We should check command, as well as any other fields. E.g. `locked`. Signed-off-by: Ruslan Kuprieiev --- dvc/stage.py | 81 ++++++++++++++----------------- tests/test_run.py | 121 +++++++++++++++++++++++++++------------------- 2 files changed, 107 insertions(+), 95 deletions(-) diff --git a/dvc/stage.py b/dvc/stage.py index 61f4e07c9..b58d1b636 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -1,4 +1,3 @@ -import collections import os import yaml import posixpath @@ -72,43 +71,6 @@ class MissingDataSource(DvcException): super(MissingDataSource, self).__init__(msg) -def _identify_dicts(dicts): - return collections.Counter(frozenset(d.items()) for d in dicts) - - -def _deps_equal(old, new): - old_identities = _identify_dicts(dep.dumpd() for dep in old.deps) - new_identities = _identify_dicts(dep.dumpd() for dep in new.deps) - return old_identities == new_identities - - -def _outs_equal(old, new): - old_dicts = [out.dumpd() for out in old.outs] - new_dicts = [out.dumpd() for out in new.outs] - for d in old_dicts: - del d['md5'] - old_identities = _identify_dicts(old_dicts) - new_identities = _identify_dicts(new_dicts) - return old_identities == new_identities - - -def _stage_equal(old, new): - deps_unchanged = all(dep.changed() is False for dep in old.deps) - if not deps_unchanged: - Logger.debug('Deps changed') - return False - if not _deps_equal(old, new): - Logger.debug('Deps unequal') - return False - - if not _outs_equal(old, new): - Logger.debug('Outs unequal') - return False - - Logger.debug('stage equal') - return True - - class Stage(object): STAGE_FILE = 'Dvcfile' STAGE_FILE_SUFFIX = '.dvc' @@ -328,6 +290,37 @@ class Stage(object): if not os.path.realpath(cwd).startswith(proj_dir): raise StageBadCwdError(cwd) + def is_cached(self): + """ + Checks if this stage has been already ran and saved to the same + dvc file. + """ + from dvc.remote.local import RemoteLOCAL + from dvc.remote.s3 import RemoteS3 + + old = Stage.load(self.project, self.path) + if old.changed(): + return False + + # NOTE: need to save checksums for deps in order to compare them + # with what is written in the old stage. + for dep in self.deps: + dep.save() + + old_d = old.dumpd() + new_d = self.dumpd() + + # NOTE: need to remove checksums from old dict in order to compare + # it to the new one, since the new one doesn't have checksums yet. + old_d.pop(self.PARAM_MD5, None) + new_d.pop(self.PARAM_MD5, None) + outs = old_d.get(self.PARAM_OUTS, []) + for out in outs: + out.pop(RemoteLOCAL.PARAM_MD5, None) + out.pop(RemoteS3.PARAM_ETAG, None) + + return old_d == new_d + @staticmethod def loads(project=None, cmd=None, @@ -365,12 +358,11 @@ class Stage(object): cwd = os.path.abspath(cwd) path = os.path.join(cwd, fname) - if os.path.exists(path): + stage.cwd = cwd + stage.path = path - old = Stage.load(project=project, fname=fname) - for dep in stage.deps: - dep.save() - if deterministic and _stage_equal(old, stage): + if os.path.exists(path): + if deterministic and stage.is_cached(): Logger.info('Stage is cached, skipping.') return None @@ -381,9 +373,6 @@ class Stage(object): and not project.prompt.prompt(msg.format(relpath), False): raise DvcException("'{}' already exists".format(relpath)) - stage.cwd = cwd - stage.path = path - return stage @staticmethod diff --git a/tests/test_run.py b/tests/test_run.py index d6dac32df..b47f76703 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -199,60 +199,83 @@ class TestCmdRun(TestDvc): self.assertIn(('deterministic', True), kwargs.items()) -class TestRunDeterministic(TestDvc): +class TestRunDeterministicBase(TestDvc): + def setUp(self): + super(TestRunDeterministicBase, self).setUp() + self.out_file = 'out' + self.stage_file = self.out_file + '.dvc' + self.cmd = 'python {} {} {}'.format(self.CODE, self.FOO, self.out_file) + self.deterministic = True + self.deps = [self.FOO, self.CODE] + self.outs = [self.out_file] + + self._run(check=False) + self.assertTrue(self.stage is not None) + self.stage_file_mtime = os.path.getmtime(self.stage_file) + self.out_file_mtime = os.path.getmtime(self.out_file) + + def _run(self, should_run=True, check=True): + self.stage = self.dvc.run(cmd=self.cmd, + fname=self.stage_file, + deterministic=self.deterministic, + overwrite=True, + deps=self.deps, + outs=self.outs) + + if not check: + return + + stage_file_mtime = os.path.getmtime(self.stage_file) + out_file_mtime = os.path.getmtime(self.out_file) + + if should_run: + self.assertTrue(self.stage is not None) + self.assertNotEqual(self.stage_file_mtime, stage_file_mtime) + self.assertNotEqual(self.out_file_mtime, out_file_mtime) + else: + self.assertTrue(self.stage is None) + self.assertEqual(self.stage_file_mtime, stage_file_mtime) + self.assertEqual(self.out_file_mtime, out_file_mtime) + + +class TestRunDeterministic(TestRunDeterministicBase): def test(self): - out_file = 'out' - stage_file = out_file + '.dvc' - cmd = 'python {} {} {}'.format(self.CODE, self.FOO, out_file) - self.dvc.add(self.FOO) - stage = self.dvc.run(cmd=cmd, - fname=stage_file, - deterministic=False, - overwrite=True, - deps=[self.FOO, self.CODE], - outs=[out_file]) - self.assertTrue(stage is not None) + self._run(False) - # save timestamps to make sure that files didn't change. - stage_file_mtime_1 = os.path.getmtime(stage_file) - out_file_mtime_1 = os.path.getmtime(out_file) - # sleep to make sure that mtime changes even on filesystems - # with very low timestamp resolution(e.g. 1 sec on APFS). - time.sleep(2) - - # dependencies didn't change, so deterministic stage shouldn't - # be ran again. - stage = self.dvc.run(cmd=cmd, - fname=stage_file, - deterministic=True, - overwrite=True, - deps=[self.FOO, self.CODE], - outs=[out_file]) - self.assertTrue(stage is None) - - stage_file_mtime_2 = os.path.getmtime(stage_file) - out_file_mtime_2 = os.path.getmtime(out_file) - - self.assertEqual(stage_file_mtime_1, stage_file_mtime_2) - self.assertEqual(out_file_mtime_1, out_file_mtime_2) - - time.sleep(2) - - # now change one of the deps and make sure stage did run +class TestRunDeterministicChangedDep(TestRunDeterministicBase): + def test(self): os.unlink(self.FOO) shutil.copy(self.BAR, self.FOO) + self._run() - stage = self.dvc.run(cmd=cmd, - fname=stage_file, - deterministic=True, - overwrite=True, - deps=[self.FOO, self.CODE], - outs=[out_file]) - self.assertTrue(stage is not None) - stage_file_mtime_3 = os.path.getmtime(stage_file) - out_file_mtime_3 = os.path.getmtime(out_file) +class TestRunDeterministicChangedDepsList(TestRunDeterministicBase): + def test(self): + self.deps = [self.BAR, self.CODE] + self._run() - self.assertNotEqual(stage_file_mtime_2, stage_file_mtime_3) - self.assertNotEqual(out_file_mtime_2, out_file_mtime_3) + +class TestRunDeterministicNewDep(TestRunDeterministicBase): + def test(self): + self.deps = [self.FOO, self.BAR, self.CODE] + self._run() + + +class TestRunDeterministicRemoveDep(TestRunDeterministicBase): + def test(self): + self.deps = [self.CODE] + self._run() + + +class TestRunDeterministicChangedOut(TestRunDeterministicBase): + def test(self): + os.unlink(self.out_file) + self.out_file_mtime = None + self._run() + + +class TestRunDeterministicChangedCmd(TestRunDeterministicBase): + def test(self): + self.cmd += ' arg' + self._run()