mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
run: deterministic: copare the whole dicts
We should check command, as well as any other fields. E.g. `locked`. Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
+35
-46
@@ -1,4 +1,3 @@
|
||||
import collections
|
||||
import os
|
||||
import yaml
|
||||
import posixpath
|
||||
@@ -72,43 +71,6 @@ class MissingDataSource(DvcException):
|
||||
super(MissingDataSource, self).__init__(msg)
|
||||
|
||||
|
||||
def _identify_dicts(dicts):
|
||||
return collections.Counter(frozenset(d.items()) for d in dicts)
|
||||
|
||||
|
||||
def _deps_equal(old, new):
|
||||
old_identities = _identify_dicts(dep.dumpd() for dep in old.deps)
|
||||
new_identities = _identify_dicts(dep.dumpd() for dep in new.deps)
|
||||
return old_identities == new_identities
|
||||
|
||||
|
||||
def _outs_equal(old, new):
|
||||
old_dicts = [out.dumpd() for out in old.outs]
|
||||
new_dicts = [out.dumpd() for out in new.outs]
|
||||
for d in old_dicts:
|
||||
del d['md5']
|
||||
old_identities = _identify_dicts(old_dicts)
|
||||
new_identities = _identify_dicts(new_dicts)
|
||||
return old_identities == new_identities
|
||||
|
||||
|
||||
def _stage_equal(old, new):
|
||||
deps_unchanged = all(dep.changed() is False for dep in old.deps)
|
||||
if not deps_unchanged:
|
||||
Logger.debug('Deps changed')
|
||||
return False
|
||||
if not _deps_equal(old, new):
|
||||
Logger.debug('Deps unequal')
|
||||
return False
|
||||
|
||||
if not _outs_equal(old, new):
|
||||
Logger.debug('Outs unequal')
|
||||
return False
|
||||
|
||||
Logger.debug('stage equal')
|
||||
return True
|
||||
|
||||
|
||||
class Stage(object):
|
||||
STAGE_FILE = 'Dvcfile'
|
||||
STAGE_FILE_SUFFIX = '.dvc'
|
||||
@@ -328,6 +290,37 @@ class Stage(object):
|
||||
if not os.path.realpath(cwd).startswith(proj_dir):
|
||||
raise StageBadCwdError(cwd)
|
||||
|
||||
def is_cached(self):
|
||||
"""
|
||||
Checks if this stage has been already ran and saved to the same
|
||||
dvc file.
|
||||
"""
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
from dvc.remote.s3 import RemoteS3
|
||||
|
||||
old = Stage.load(self.project, self.path)
|
||||
if old.changed():
|
||||
return False
|
||||
|
||||
# NOTE: need to save checksums for deps in order to compare them
|
||||
# with what is written in the old stage.
|
||||
for dep in self.deps:
|
||||
dep.save()
|
||||
|
||||
old_d = old.dumpd()
|
||||
new_d = self.dumpd()
|
||||
|
||||
# NOTE: need to remove checksums from old dict in order to compare
|
||||
# it to the new one, since the new one doesn't have checksums yet.
|
||||
old_d.pop(self.PARAM_MD5, None)
|
||||
new_d.pop(self.PARAM_MD5, None)
|
||||
outs = old_d.get(self.PARAM_OUTS, [])
|
||||
for out in outs:
|
||||
out.pop(RemoteLOCAL.PARAM_MD5, None)
|
||||
out.pop(RemoteS3.PARAM_ETAG, None)
|
||||
|
||||
return old_d == new_d
|
||||
|
||||
@staticmethod
|
||||
def loads(project=None,
|
||||
cmd=None,
|
||||
@@ -365,12 +358,11 @@ class Stage(object):
|
||||
cwd = os.path.abspath(cwd)
|
||||
path = os.path.join(cwd, fname)
|
||||
|
||||
if os.path.exists(path):
|
||||
stage.cwd = cwd
|
||||
stage.path = path
|
||||
|
||||
old = Stage.load(project=project, fname=fname)
|
||||
for dep in stage.deps:
|
||||
dep.save()
|
||||
if deterministic and _stage_equal(old, stage):
|
||||
if os.path.exists(path):
|
||||
if deterministic and stage.is_cached():
|
||||
Logger.info('Stage is cached, skipping.')
|
||||
return None
|
||||
|
||||
@@ -381,9 +373,6 @@ class Stage(object):
|
||||
and not project.prompt.prompt(msg.format(relpath), False):
|
||||
raise DvcException("'{}' already exists".format(relpath))
|
||||
|
||||
stage.cwd = cwd
|
||||
stage.path = path
|
||||
|
||||
return stage
|
||||
|
||||
@staticmethod
|
||||
|
||||
+72
-49
@@ -199,60 +199,83 @@ class TestCmdRun(TestDvc):
|
||||
self.assertIn(('deterministic', True), kwargs.items())
|
||||
|
||||
|
||||
class TestRunDeterministic(TestDvc):
|
||||
class TestRunDeterministicBase(TestDvc):
|
||||
def setUp(self):
|
||||
super(TestRunDeterministicBase, self).setUp()
|
||||
self.out_file = 'out'
|
||||
self.stage_file = self.out_file + '.dvc'
|
||||
self.cmd = 'python {} {} {}'.format(self.CODE, self.FOO, self.out_file)
|
||||
self.deterministic = True
|
||||
self.deps = [self.FOO, self.CODE]
|
||||
self.outs = [self.out_file]
|
||||
|
||||
self._run(check=False)
|
||||
self.assertTrue(self.stage is not None)
|
||||
self.stage_file_mtime = os.path.getmtime(self.stage_file)
|
||||
self.out_file_mtime = os.path.getmtime(self.out_file)
|
||||
|
||||
def _run(self, should_run=True, check=True):
|
||||
self.stage = self.dvc.run(cmd=self.cmd,
|
||||
fname=self.stage_file,
|
||||
deterministic=self.deterministic,
|
||||
overwrite=True,
|
||||
deps=self.deps,
|
||||
outs=self.outs)
|
||||
|
||||
if not check:
|
||||
return
|
||||
|
||||
stage_file_mtime = os.path.getmtime(self.stage_file)
|
||||
out_file_mtime = os.path.getmtime(self.out_file)
|
||||
|
||||
if should_run:
|
||||
self.assertTrue(self.stage is not None)
|
||||
self.assertNotEqual(self.stage_file_mtime, stage_file_mtime)
|
||||
self.assertNotEqual(self.out_file_mtime, out_file_mtime)
|
||||
else:
|
||||
self.assertTrue(self.stage is None)
|
||||
self.assertEqual(self.stage_file_mtime, stage_file_mtime)
|
||||
self.assertEqual(self.out_file_mtime, out_file_mtime)
|
||||
|
||||
|
||||
class TestRunDeterministic(TestRunDeterministicBase):
|
||||
def test(self):
|
||||
out_file = 'out'
|
||||
stage_file = out_file + '.dvc'
|
||||
cmd = 'python {} {} {}'.format(self.CODE, self.FOO, out_file)
|
||||
self.dvc.add(self.FOO)
|
||||
stage = self.dvc.run(cmd=cmd,
|
||||
fname=stage_file,
|
||||
deterministic=False,
|
||||
overwrite=True,
|
||||
deps=[self.FOO, self.CODE],
|
||||
outs=[out_file])
|
||||
self.assertTrue(stage is not None)
|
||||
self._run(False)
|
||||
|
||||
# save timestamps to make sure that files didn't change.
|
||||
stage_file_mtime_1 = os.path.getmtime(stage_file)
|
||||
out_file_mtime_1 = os.path.getmtime(out_file)
|
||||
|
||||
# sleep to make sure that mtime changes even on filesystems
|
||||
# with very low timestamp resolution(e.g. 1 sec on APFS).
|
||||
time.sleep(2)
|
||||
|
||||
# dependencies didn't change, so deterministic stage shouldn't
|
||||
# be ran again.
|
||||
stage = self.dvc.run(cmd=cmd,
|
||||
fname=stage_file,
|
||||
deterministic=True,
|
||||
overwrite=True,
|
||||
deps=[self.FOO, self.CODE],
|
||||
outs=[out_file])
|
||||
self.assertTrue(stage is None)
|
||||
|
||||
stage_file_mtime_2 = os.path.getmtime(stage_file)
|
||||
out_file_mtime_2 = os.path.getmtime(out_file)
|
||||
|
||||
self.assertEqual(stage_file_mtime_1, stage_file_mtime_2)
|
||||
self.assertEqual(out_file_mtime_1, out_file_mtime_2)
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
# now change one of the deps and make sure stage did run
|
||||
class TestRunDeterministicChangedDep(TestRunDeterministicBase):
|
||||
def test(self):
|
||||
os.unlink(self.FOO)
|
||||
shutil.copy(self.BAR, self.FOO)
|
||||
self._run()
|
||||
|
||||
stage = self.dvc.run(cmd=cmd,
|
||||
fname=stage_file,
|
||||
deterministic=True,
|
||||
overwrite=True,
|
||||
deps=[self.FOO, self.CODE],
|
||||
outs=[out_file])
|
||||
self.assertTrue(stage is not None)
|
||||
|
||||
stage_file_mtime_3 = os.path.getmtime(stage_file)
|
||||
out_file_mtime_3 = os.path.getmtime(out_file)
|
||||
class TestRunDeterministicChangedDepsList(TestRunDeterministicBase):
|
||||
def test(self):
|
||||
self.deps = [self.BAR, self.CODE]
|
||||
self._run()
|
||||
|
||||
self.assertNotEqual(stage_file_mtime_2, stage_file_mtime_3)
|
||||
self.assertNotEqual(out_file_mtime_2, out_file_mtime_3)
|
||||
|
||||
class TestRunDeterministicNewDep(TestRunDeterministicBase):
|
||||
def test(self):
|
||||
self.deps = [self.FOO, self.BAR, self.CODE]
|
||||
self._run()
|
||||
|
||||
|
||||
class TestRunDeterministicRemoveDep(TestRunDeterministicBase):
|
||||
def test(self):
|
||||
self.deps = [self.CODE]
|
||||
self._run()
|
||||
|
||||
|
||||
class TestRunDeterministicChangedOut(TestRunDeterministicBase):
|
||||
def test(self):
|
||||
os.unlink(self.out_file)
|
||||
self.out_file_mtime = None
|
||||
self._run()
|
||||
|
||||
|
||||
class TestRunDeterministicChangedCmd(TestRunDeterministicBase):
|
||||
def test(self):
|
||||
self.cmd += ' arg'
|
||||
self._run()
|
||||
|
||||
Reference in New Issue
Block a user