run: deterministic: copare the whole dicts

We should check command, as well as any other fields. E.g. `locked`.

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-12-03 01:29:29 +02:00
parent 1211abf35c
commit ae5ac21bf8
2 changed files with 107 additions and 95 deletions
+35 -46
View File
@@ -1,4 +1,3 @@
import collections
import os
import yaml
import posixpath
@@ -72,43 +71,6 @@ class MissingDataSource(DvcException):
super(MissingDataSource, self).__init__(msg)
def _identify_dicts(dicts):
return collections.Counter(frozenset(d.items()) for d in dicts)
def _deps_equal(old, new):
old_identities = _identify_dicts(dep.dumpd() for dep in old.deps)
new_identities = _identify_dicts(dep.dumpd() for dep in new.deps)
return old_identities == new_identities
def _outs_equal(old, new):
old_dicts = [out.dumpd() for out in old.outs]
new_dicts = [out.dumpd() for out in new.outs]
for d in old_dicts:
del d['md5']
old_identities = _identify_dicts(old_dicts)
new_identities = _identify_dicts(new_dicts)
return old_identities == new_identities
def _stage_equal(old, new):
deps_unchanged = all(dep.changed() is False for dep in old.deps)
if not deps_unchanged:
Logger.debug('Deps changed')
return False
if not _deps_equal(old, new):
Logger.debug('Deps unequal')
return False
if not _outs_equal(old, new):
Logger.debug('Outs unequal')
return False
Logger.debug('stage equal')
return True
class Stage(object):
STAGE_FILE = 'Dvcfile'
STAGE_FILE_SUFFIX = '.dvc'
@@ -328,6 +290,37 @@ class Stage(object):
if not os.path.realpath(cwd).startswith(proj_dir):
raise StageBadCwdError(cwd)
def is_cached(self):
"""
Checks if this stage has been already ran and saved to the same
dvc file.
"""
from dvc.remote.local import RemoteLOCAL
from dvc.remote.s3 import RemoteS3
old = Stage.load(self.project, self.path)
if old.changed():
return False
# NOTE: need to save checksums for deps in order to compare them
# with what is written in the old stage.
for dep in self.deps:
dep.save()
old_d = old.dumpd()
new_d = self.dumpd()
# NOTE: need to remove checksums from old dict in order to compare
# it to the new one, since the new one doesn't have checksums yet.
old_d.pop(self.PARAM_MD5, None)
new_d.pop(self.PARAM_MD5, None)
outs = old_d.get(self.PARAM_OUTS, [])
for out in outs:
out.pop(RemoteLOCAL.PARAM_MD5, None)
out.pop(RemoteS3.PARAM_ETAG, None)
return old_d == new_d
@staticmethod
def loads(project=None,
cmd=None,
@@ -365,12 +358,11 @@ class Stage(object):
cwd = os.path.abspath(cwd)
path = os.path.join(cwd, fname)
if os.path.exists(path):
stage.cwd = cwd
stage.path = path
old = Stage.load(project=project, fname=fname)
for dep in stage.deps:
dep.save()
if deterministic and _stage_equal(old, stage):
if os.path.exists(path):
if deterministic and stage.is_cached():
Logger.info('Stage is cached, skipping.')
return None
@@ -381,9 +373,6 @@ class Stage(object):
and not project.prompt.prompt(msg.format(relpath), False):
raise DvcException("'{}' already exists".format(relpath))
stage.cwd = cwd
stage.path = path
return stage
@staticmethod
+72 -49
View File
@@ -199,60 +199,83 @@ class TestCmdRun(TestDvc):
self.assertIn(('deterministic', True), kwargs.items())
class TestRunDeterministic(TestDvc):
class TestRunDeterministicBase(TestDvc):
def setUp(self):
super(TestRunDeterministicBase, self).setUp()
self.out_file = 'out'
self.stage_file = self.out_file + '.dvc'
self.cmd = 'python {} {} {}'.format(self.CODE, self.FOO, self.out_file)
self.deterministic = True
self.deps = [self.FOO, self.CODE]
self.outs = [self.out_file]
self._run(check=False)
self.assertTrue(self.stage is not None)
self.stage_file_mtime = os.path.getmtime(self.stage_file)
self.out_file_mtime = os.path.getmtime(self.out_file)
def _run(self, should_run=True, check=True):
self.stage = self.dvc.run(cmd=self.cmd,
fname=self.stage_file,
deterministic=self.deterministic,
overwrite=True,
deps=self.deps,
outs=self.outs)
if not check:
return
stage_file_mtime = os.path.getmtime(self.stage_file)
out_file_mtime = os.path.getmtime(self.out_file)
if should_run:
self.assertTrue(self.stage is not None)
self.assertNotEqual(self.stage_file_mtime, stage_file_mtime)
self.assertNotEqual(self.out_file_mtime, out_file_mtime)
else:
self.assertTrue(self.stage is None)
self.assertEqual(self.stage_file_mtime, stage_file_mtime)
self.assertEqual(self.out_file_mtime, out_file_mtime)
class TestRunDeterministic(TestRunDeterministicBase):
def test(self):
out_file = 'out'
stage_file = out_file + '.dvc'
cmd = 'python {} {} {}'.format(self.CODE, self.FOO, out_file)
self.dvc.add(self.FOO)
stage = self.dvc.run(cmd=cmd,
fname=stage_file,
deterministic=False,
overwrite=True,
deps=[self.FOO, self.CODE],
outs=[out_file])
self.assertTrue(stage is not None)
self._run(False)
# save timestamps to make sure that files didn't change.
stage_file_mtime_1 = os.path.getmtime(stage_file)
out_file_mtime_1 = os.path.getmtime(out_file)
# sleep to make sure that mtime changes even on filesystems
# with very low timestamp resolution(e.g. 1 sec on APFS).
time.sleep(2)
# dependencies didn't change, so deterministic stage shouldn't
# be ran again.
stage = self.dvc.run(cmd=cmd,
fname=stage_file,
deterministic=True,
overwrite=True,
deps=[self.FOO, self.CODE],
outs=[out_file])
self.assertTrue(stage is None)
stage_file_mtime_2 = os.path.getmtime(stage_file)
out_file_mtime_2 = os.path.getmtime(out_file)
self.assertEqual(stage_file_mtime_1, stage_file_mtime_2)
self.assertEqual(out_file_mtime_1, out_file_mtime_2)
time.sleep(2)
# now change one of the deps and make sure stage did run
class TestRunDeterministicChangedDep(TestRunDeterministicBase):
def test(self):
os.unlink(self.FOO)
shutil.copy(self.BAR, self.FOO)
self._run()
stage = self.dvc.run(cmd=cmd,
fname=stage_file,
deterministic=True,
overwrite=True,
deps=[self.FOO, self.CODE],
outs=[out_file])
self.assertTrue(stage is not None)
stage_file_mtime_3 = os.path.getmtime(stage_file)
out_file_mtime_3 = os.path.getmtime(out_file)
class TestRunDeterministicChangedDepsList(TestRunDeterministicBase):
def test(self):
self.deps = [self.BAR, self.CODE]
self._run()
self.assertNotEqual(stage_file_mtime_2, stage_file_mtime_3)
self.assertNotEqual(out_file_mtime_2, out_file_mtime_3)
class TestRunDeterministicNewDep(TestRunDeterministicBase):
def test(self):
self.deps = [self.FOO, self.BAR, self.CODE]
self._run()
class TestRunDeterministicRemoveDep(TestRunDeterministicBase):
def test(self):
self.deps = [self.CODE]
self._run()
class TestRunDeterministicChangedOut(TestRunDeterministicBase):
def test(self):
os.unlink(self.out_file)
self.out_file_mtime = None
self._run()
class TestRunDeterministicChangedCmd(TestRunDeterministicBase):
def test(self):
self.cmd += ' arg'
self._run()