mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 23:11:44 +01:00
stage: properly handle dvc file md5
Fixes #1380 Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
committed by
Ruslan Kuprieiev
parent
eebfa95c88
commit
98b043310d
+29
-15
@@ -131,13 +131,10 @@ class Stage(object):
|
||||
return Stage.is_stage_filename(path)
|
||||
|
||||
def changed_md5(self):
|
||||
md5 = self.dumpd().get(self.PARAM_MD5, None)
|
||||
md5 = self._get_md5()
|
||||
assert md5 is not None
|
||||
|
||||
# backward compatibility
|
||||
if self.md5 is None:
|
||||
return False
|
||||
|
||||
if self.md5 and md5 and self.md5 == md5:
|
||||
if self.md5 == md5:
|
||||
return False
|
||||
|
||||
return True
|
||||
@@ -371,24 +368,22 @@ class Stage(object):
|
||||
return Stage.loadd(project, yaml.safe_load(fd) or dict(), fname)
|
||||
|
||||
def dumpd(self):
|
||||
deps = [x.dumpd() for x in self.deps]
|
||||
outs = [x.dumpd() for x in self.outs]
|
||||
|
||||
ret = {}
|
||||
|
||||
if self.cmd is not None:
|
||||
ret[Stage.PARAM_CMD] = self.cmd
|
||||
|
||||
if len(deps):
|
||||
ret[Stage.PARAM_DEPS] = deps
|
||||
if len(self.deps):
|
||||
ret[Stage.PARAM_DEPS] = [d.dumpd() for d in self.deps]
|
||||
|
||||
if len(outs):
|
||||
ret[Stage.PARAM_OUTS] = outs
|
||||
if len(self.outs):
|
||||
ret[Stage.PARAM_OUTS] = [o.dumpd() for o in self.outs]
|
||||
|
||||
ret[Stage.PARAM_MD5] = self.md5
|
||||
|
||||
if self.locked:
|
||||
ret[Stage.PARAM_LOCKED] = self.locked
|
||||
|
||||
ret[Stage.PARAM_MD5] = dict_md5(ret)
|
||||
|
||||
return ret
|
||||
|
||||
def dump(self, fname=None):
|
||||
@@ -405,6 +400,23 @@ class Stage(object):
|
||||
|
||||
self.project._files_to_git_add.append(os.path.relpath(fname))
|
||||
|
||||
def _get_md5(self):
|
||||
from dvc.output.local import OutputLOCAL
|
||||
|
||||
# NOTE: excluding parameters that don't affect the state of the
|
||||
# pipeline. Not excluding OutputLOCAL.PARAM_CACHE, because if
|
||||
# it has changed, we might not have that output in our cache.
|
||||
exclude = [self.PARAM_LOCKED,
|
||||
OutputLOCAL.PARAM_METRIC]
|
||||
|
||||
d = self.dumpd()
|
||||
|
||||
# NOTE: removing md5 manually in order to not affect md5s in deps/outs
|
||||
if self.PARAM_MD5 in d.keys():
|
||||
del d[self.PARAM_MD5]
|
||||
|
||||
return dict_md5(d, exclude)
|
||||
|
||||
def save(self):
|
||||
for dep in self.deps:
|
||||
dep.save()
|
||||
@@ -412,6 +424,8 @@ class Stage(object):
|
||||
for out in self.outs:
|
||||
out.save()
|
||||
|
||||
self.md5 = self._get_md5()
|
||||
|
||||
def _check_missing_deps(self):
|
||||
missing = []
|
||||
for dep in self.deps:
|
||||
|
||||
+25
-2
@@ -68,8 +68,31 @@ def bytes_md5(byts):
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def dict_md5(d):
|
||||
byts = json.dumps(d, sort_keys=True).encode('utf-8')
|
||||
def dict_filter(d, exclude=[]):
|
||||
"""
|
||||
Exclude specified keys from a nested dict
|
||||
"""
|
||||
|
||||
if isinstance(d, list):
|
||||
ret = []
|
||||
for e in d:
|
||||
ret.append(dict_filter(e, exclude))
|
||||
return ret
|
||||
elif isinstance(d, dict):
|
||||
ret = {}
|
||||
for k, v in d.items():
|
||||
assert isinstance(k, str)
|
||||
if k in exclude:
|
||||
continue
|
||||
ret[k] = dict_filter(v, exclude)
|
||||
return ret
|
||||
|
||||
return d
|
||||
|
||||
|
||||
def dict_md5(d, exclude=[]):
|
||||
filtered = dict_filter(d, exclude)
|
||||
byts = json.dumps(filtered, sort_keys=True).encode('utf-8')
|
||||
return bytes_md5(byts)
|
||||
|
||||
|
||||
|
||||
+41
-2
@@ -361,15 +361,54 @@ class TestReproLockedCallback(TestDvc):
|
||||
self.assertEqual(len(stages), 1)
|
||||
|
||||
self.dvc.lock_stage(file1_stage)
|
||||
|
||||
stages = self.dvc.reproduce(file1_stage)
|
||||
self.assertEqual(len(stages), 0)
|
||||
|
||||
self.dvc.lock_stage(file1_stage, unlock=True)
|
||||
|
||||
stages = self.dvc.reproduce(file1_stage)
|
||||
self.assertEqual(len(stages), 1)
|
||||
|
||||
class TestReproLockedUnchanged(TestRepro):
|
||||
def test(self):
|
||||
"""
|
||||
Check that locking/unlocking doesn't affect stage state
|
||||
"""
|
||||
self.dvc.lock_stage(self.file1_stage)
|
||||
stages = self.dvc.reproduce(self.file1_stage)
|
||||
self.assertEqual(len(stages), 0)
|
||||
|
||||
self.dvc.lock_stage(self.file1_stage, unlock=True)
|
||||
stages = self.dvc.reproduce(self.file1_stage)
|
||||
self.assertEqual(len(stages), 0)
|
||||
|
||||
|
||||
class TestReproMetricsAddUnchanged(TestDvc):
|
||||
def test(self):
|
||||
"""
|
||||
Check that adding/removing metrics doesn't affect stage state
|
||||
"""
|
||||
stages = self.dvc.add(self.FOO)
|
||||
self.assertEqual(len(stages), 1)
|
||||
self.assertTrue(stages[0] is not None)
|
||||
|
||||
file1 = 'file1'
|
||||
file1_stage = file1 + '.dvc'
|
||||
self.dvc.run(fname=file1_stage,
|
||||
outs_no_cache=[file1],
|
||||
deps=[self.FOO, self.CODE],
|
||||
cmd='python {} {} {}'.format(self.CODE, self.FOO, file1))
|
||||
|
||||
stages = self.dvc.reproduce(file1_stage)
|
||||
self.assertEqual(len(stages), 0)
|
||||
|
||||
self.dvc.metrics_add(file1)
|
||||
stages = self.dvc.reproduce(file1_stage)
|
||||
self.assertEqual(len(stages), 0)
|
||||
|
||||
self.dvc.metrics_remove(file1)
|
||||
stages = self.dvc.reproduce(file1_stage)
|
||||
self.assertEqual(len(stages), 0)
|
||||
|
||||
|
||||
class TestReproPhony(TestReproChangedData):
|
||||
def test(self):
|
||||
|
||||
@@ -62,3 +62,31 @@ class TestSchemaDepsOuts(TestSchema):
|
||||
l[1][OutputLOCAL.PARAM_CACHE] = False
|
||||
d = {Stage.PARAM_OUTS: l}
|
||||
Stage.validate(d)
|
||||
|
||||
|
||||
class TestReload(TestDvc):
|
||||
def test(self):
|
||||
import yaml
|
||||
stages = self.dvc.add(self.FOO)
|
||||
self.assertEqual(len(stages), 1)
|
||||
stage = stages[0]
|
||||
self.assertTrue(stage is not None)
|
||||
|
||||
with open(stage.relpath, 'r') as fobj:
|
||||
d = yaml.safe_load(fobj)
|
||||
|
||||
# NOTE: checking that reloaded stage didn't change its checksum
|
||||
md5 = '11111111111111111111111111111111'
|
||||
d[stage.PARAM_MD5] = md5
|
||||
|
||||
with open(stage.relpath, 'w') as fobj:
|
||||
yaml.safe_dump(d, fobj, default_flow_style=False)
|
||||
|
||||
stage = Stage.load(self.dvc, stage.relpath)
|
||||
self.assertTrue(stage is not None)
|
||||
stage.dump()
|
||||
|
||||
with open(stage.relpath, 'r') as fobj:
|
||||
d = yaml.safe_load(fobj)
|
||||
|
||||
self.assertEqual(d[stage.PARAM_MD5], md5)
|
||||
|
||||
@@ -34,3 +34,20 @@ class TestUtils(TestDvc):
|
||||
fd.write(b'a\r\nb\r\nc')
|
||||
|
||||
self.assertEqual(utils.file_md5('cr')[0], utils.file_md5('crlf')[0])
|
||||
|
||||
def test_dict_md5(self):
|
||||
d = {'cmd': 'python code.py foo file1',
|
||||
'locked': 'true',
|
||||
'outs': [{'path': 'file1',
|
||||
'metric': {'type': 'raw'},
|
||||
'cache': False,
|
||||
'md5':'acbd18db4cc2f85cedef654fccc4a4d8'}],
|
||||
'deps': [{'path': 'foo',
|
||||
'md5': 'acbd18db4cc2f85cedef654fccc4a4d8'},
|
||||
{'path': 'code.py',
|
||||
'md5': 'd05447644b89960913c7eee5fd776adb'}]}
|
||||
|
||||
|
||||
md5 = '8b263fa05ede6c3145c164829be694b4'
|
||||
|
||||
self.assertEqual(md5, utils.dict_md5(d, exclude=['metric', 'locked']))
|
||||
|
||||
Reference in New Issue
Block a user