stage: properly handle dvc file md5

Fixes #1380

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-11-30 09:21:03 +02:00
committed by Ruslan Kuprieiev
parent eebfa95c88
commit 98b043310d
5 changed files with 140 additions and 19 deletions
+29 -15
View File
@@ -131,13 +131,10 @@ class Stage(object):
return Stage.is_stage_filename(path)
def changed_md5(self):
md5 = self.dumpd().get(self.PARAM_MD5, None)
md5 = self._get_md5()
assert md5 is not None
# backward compatibility
if self.md5 is None:
return False
if self.md5 and md5 and self.md5 == md5:
if self.md5 == md5:
return False
return True
@@ -371,24 +368,22 @@ class Stage(object):
return Stage.loadd(project, yaml.safe_load(fd) or dict(), fname)
def dumpd(self):
deps = [x.dumpd() for x in self.deps]
outs = [x.dumpd() for x in self.outs]
ret = {}
if self.cmd is not None:
ret[Stage.PARAM_CMD] = self.cmd
if len(deps):
ret[Stage.PARAM_DEPS] = deps
if len(self.deps):
ret[Stage.PARAM_DEPS] = [d.dumpd() for d in self.deps]
if len(outs):
ret[Stage.PARAM_OUTS] = outs
if len(self.outs):
ret[Stage.PARAM_OUTS] = [o.dumpd() for o in self.outs]
ret[Stage.PARAM_MD5] = self.md5
if self.locked:
ret[Stage.PARAM_LOCKED] = self.locked
ret[Stage.PARAM_MD5] = dict_md5(ret)
return ret
def dump(self, fname=None):
@@ -405,6 +400,23 @@ class Stage(object):
self.project._files_to_git_add.append(os.path.relpath(fname))
def _get_md5(self):
from dvc.output.local import OutputLOCAL
# NOTE: excluding parameters that don't affect the state of the
# pipeline. Not excluding OutputLOCAL.PARAM_CACHE, because if
# it has changed, we might not have that output in our cache.
exclude = [self.PARAM_LOCKED,
OutputLOCAL.PARAM_METRIC]
d = self.dumpd()
# NOTE: removing md5 manually in order to not affect md5s in deps/outs
if self.PARAM_MD5 in d.keys():
del d[self.PARAM_MD5]
return dict_md5(d, exclude)
def save(self):
for dep in self.deps:
dep.save()
@@ -412,6 +424,8 @@ class Stage(object):
for out in self.outs:
out.save()
self.md5 = self._get_md5()
def _check_missing_deps(self):
missing = []
for dep in self.deps:
+25 -2
View File
@@ -68,8 +68,31 @@ def bytes_md5(byts):
return hasher.hexdigest()
def dict_md5(d):
byts = json.dumps(d, sort_keys=True).encode('utf-8')
def dict_filter(d, exclude=[]):
"""
Exclude specified keys from a nested dict
"""
if isinstance(d, list):
ret = []
for e in d:
ret.append(dict_filter(e, exclude))
return ret
elif isinstance(d, dict):
ret = {}
for k, v in d.items():
assert isinstance(k, str)
if k in exclude:
continue
ret[k] = dict_filter(v, exclude)
return ret
return d
def dict_md5(d, exclude=[]):
filtered = dict_filter(d, exclude)
byts = json.dumps(filtered, sort_keys=True).encode('utf-8')
return bytes_md5(byts)
+41 -2
View File
@@ -361,15 +361,54 @@ class TestReproLockedCallback(TestDvc):
self.assertEqual(len(stages), 1)
self.dvc.lock_stage(file1_stage)
stages = self.dvc.reproduce(file1_stage)
self.assertEqual(len(stages), 0)
self.dvc.lock_stage(file1_stage, unlock=True)
stages = self.dvc.reproduce(file1_stage)
self.assertEqual(len(stages), 1)
class TestReproLockedUnchanged(TestRepro):
def test(self):
"""
Check that locking/unlocking doesn't affect stage state
"""
self.dvc.lock_stage(self.file1_stage)
stages = self.dvc.reproduce(self.file1_stage)
self.assertEqual(len(stages), 0)
self.dvc.lock_stage(self.file1_stage, unlock=True)
stages = self.dvc.reproduce(self.file1_stage)
self.assertEqual(len(stages), 0)
class TestReproMetricsAddUnchanged(TestDvc):
def test(self):
"""
Check that adding/removing metrics doesn't affect stage state
"""
stages = self.dvc.add(self.FOO)
self.assertEqual(len(stages), 1)
self.assertTrue(stages[0] is not None)
file1 = 'file1'
file1_stage = file1 + '.dvc'
self.dvc.run(fname=file1_stage,
outs_no_cache=[file1],
deps=[self.FOO, self.CODE],
cmd='python {} {} {}'.format(self.CODE, self.FOO, file1))
stages = self.dvc.reproduce(file1_stage)
self.assertEqual(len(stages), 0)
self.dvc.metrics_add(file1)
stages = self.dvc.reproduce(file1_stage)
self.assertEqual(len(stages), 0)
self.dvc.metrics_remove(file1)
stages = self.dvc.reproduce(file1_stage)
self.assertEqual(len(stages), 0)
class TestReproPhony(TestReproChangedData):
def test(self):
+28
View File
@@ -62,3 +62,31 @@ class TestSchemaDepsOuts(TestSchema):
l[1][OutputLOCAL.PARAM_CACHE] = False
d = {Stage.PARAM_OUTS: l}
Stage.validate(d)
class TestReload(TestDvc):
def test(self):
import yaml
stages = self.dvc.add(self.FOO)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertTrue(stage is not None)
with open(stage.relpath, 'r') as fobj:
d = yaml.safe_load(fobj)
# NOTE: checking that reloaded stage didn't change its checksum
md5 = '11111111111111111111111111111111'
d[stage.PARAM_MD5] = md5
with open(stage.relpath, 'w') as fobj:
yaml.safe_dump(d, fobj, default_flow_style=False)
stage = Stage.load(self.dvc, stage.relpath)
self.assertTrue(stage is not None)
stage.dump()
with open(stage.relpath, 'r') as fobj:
d = yaml.safe_load(fobj)
self.assertEqual(d[stage.PARAM_MD5], md5)
+17
View File
@@ -34,3 +34,20 @@ class TestUtils(TestDvc):
fd.write(b'a\r\nb\r\nc')
self.assertEqual(utils.file_md5('cr')[0], utils.file_md5('crlf')[0])
def test_dict_md5(self):
d = {'cmd': 'python code.py foo file1',
'locked': 'true',
'outs': [{'path': 'file1',
'metric': {'type': 'raw'},
'cache': False,
'md5':'acbd18db4cc2f85cedef654fccc4a4d8'}],
'deps': [{'path': 'foo',
'md5': 'acbd18db4cc2f85cedef654fccc4a4d8'},
{'path': 'code.py',
'md5': 'd05447644b89960913c7eee5fd776adb'}]}
md5 = '8b263fa05ede6c3145c164829be694b4'
self.assertEqual(md5, utils.dict_md5(d, exclude=['metric', 'locked']))