diff --git a/dvc/stage.py b/dvc/stage.py index e612e69fd..30cd5b085 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -131,13 +131,10 @@ class Stage(object): return Stage.is_stage_filename(path) def changed_md5(self): - md5 = self.dumpd().get(self.PARAM_MD5, None) + md5 = self._get_md5() + assert md5 is not None - # backward compatibility - if self.md5 is None: - return False - - if self.md5 and md5 and self.md5 == md5: + if self.md5 == md5: return False return True @@ -371,24 +368,22 @@ class Stage(object): return Stage.loadd(project, yaml.safe_load(fd) or dict(), fname) def dumpd(self): - deps = [x.dumpd() for x in self.deps] - outs = [x.dumpd() for x in self.outs] - ret = {} + if self.cmd is not None: ret[Stage.PARAM_CMD] = self.cmd - if len(deps): - ret[Stage.PARAM_DEPS] = deps + if len(self.deps): + ret[Stage.PARAM_DEPS] = [d.dumpd() for d in self.deps] - if len(outs): - ret[Stage.PARAM_OUTS] = outs + if len(self.outs): + ret[Stage.PARAM_OUTS] = [o.dumpd() for o in self.outs] + + ret[Stage.PARAM_MD5] = self.md5 if self.locked: ret[Stage.PARAM_LOCKED] = self.locked - ret[Stage.PARAM_MD5] = dict_md5(ret) - return ret def dump(self, fname=None): @@ -405,6 +400,23 @@ class Stage(object): self.project._files_to_git_add.append(os.path.relpath(fname)) + def _get_md5(self): + from dvc.output.local import OutputLOCAL + + # NOTE: excluding parameters that don't affect the state of the + # pipeline. Not excluding OutputLOCAL.PARAM_CACHE, because if + # it has changed, we might not have that output in our cache. + exclude = [self.PARAM_LOCKED, + OutputLOCAL.PARAM_METRIC] + + d = self.dumpd() + + # NOTE: removing md5 manually in order to not affect md5s in deps/outs + if self.PARAM_MD5 in d.keys(): + del d[self.PARAM_MD5] + + return dict_md5(d, exclude) + def save(self): for dep in self.deps: dep.save() @@ -412,6 +424,8 @@ class Stage(object): for out in self.outs: out.save() + self.md5 = self._get_md5() + def _check_missing_deps(self): missing = [] for dep in self.deps: diff --git a/dvc/utils.py b/dvc/utils.py index 7d4660aa6..89f912df6 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -68,8 +68,31 @@ def bytes_md5(byts): return hasher.hexdigest() -def dict_md5(d): - byts = json.dumps(d, sort_keys=True).encode('utf-8') +def dict_filter(d, exclude=[]): + """ + Exclude specified keys from a nested dict + """ + + if isinstance(d, list): + ret = [] + for e in d: + ret.append(dict_filter(e, exclude)) + return ret + elif isinstance(d, dict): + ret = {} + for k, v in d.items(): + assert isinstance(k, str) + if k in exclude: + continue + ret[k] = dict_filter(v, exclude) + return ret + + return d + + +def dict_md5(d, exclude=[]): + filtered = dict_filter(d, exclude) + byts = json.dumps(filtered, sort_keys=True).encode('utf-8') return bytes_md5(byts) diff --git a/tests/test_repro.py b/tests/test_repro.py index b2c4726a9..72cba2bac 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -361,15 +361,54 @@ class TestReproLockedCallback(TestDvc): self.assertEqual(len(stages), 1) self.dvc.lock_stage(file1_stage) - stages = self.dvc.reproduce(file1_stage) self.assertEqual(len(stages), 0) self.dvc.lock_stage(file1_stage, unlock=True) - stages = self.dvc.reproduce(file1_stage) self.assertEqual(len(stages), 1) +class TestReproLockedUnchanged(TestRepro): + def test(self): + """ + Check that locking/unlocking doesn't affect stage state + """ + self.dvc.lock_stage(self.file1_stage) + stages = self.dvc.reproduce(self.file1_stage) + self.assertEqual(len(stages), 0) + + self.dvc.lock_stage(self.file1_stage, unlock=True) + stages = self.dvc.reproduce(self.file1_stage) + self.assertEqual(len(stages), 0) + + +class TestReproMetricsAddUnchanged(TestDvc): + def test(self): + """ + Check that adding/removing metrics doesn't affect stage state + """ + stages = self.dvc.add(self.FOO) + self.assertEqual(len(stages), 1) + self.assertTrue(stages[0] is not None) + + file1 = 'file1' + file1_stage = file1 + '.dvc' + self.dvc.run(fname=file1_stage, + outs_no_cache=[file1], + deps=[self.FOO, self.CODE], + cmd='python {} {} {}'.format(self.CODE, self.FOO, file1)) + + stages = self.dvc.reproduce(file1_stage) + self.assertEqual(len(stages), 0) + + self.dvc.metrics_add(file1) + stages = self.dvc.reproduce(file1_stage) + self.assertEqual(len(stages), 0) + + self.dvc.metrics_remove(file1) + stages = self.dvc.reproduce(file1_stage) + self.assertEqual(len(stages), 0) + class TestReproPhony(TestReproChangedData): def test(self): diff --git a/tests/test_stage.py b/tests/test_stage.py index aee553eb3..0daae8eff 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -62,3 +62,31 @@ class TestSchemaDepsOuts(TestSchema): l[1][OutputLOCAL.PARAM_CACHE] = False d = {Stage.PARAM_OUTS: l} Stage.validate(d) + + +class TestReload(TestDvc): + def test(self): + import yaml + stages = self.dvc.add(self.FOO) + self.assertEqual(len(stages), 1) + stage = stages[0] + self.assertTrue(stage is not None) + + with open(stage.relpath, 'r') as fobj: + d = yaml.safe_load(fobj) + + # NOTE: checking that reloaded stage didn't change its checksum + md5 = '11111111111111111111111111111111' + d[stage.PARAM_MD5] = md5 + + with open(stage.relpath, 'w') as fobj: + yaml.safe_dump(d, fobj, default_flow_style=False) + + stage = Stage.load(self.dvc, stage.relpath) + self.assertTrue(stage is not None) + stage.dump() + + with open(stage.relpath, 'r') as fobj: + d = yaml.safe_load(fobj) + + self.assertEqual(d[stage.PARAM_MD5], md5) diff --git a/tests/test_utils.py b/tests/test_utils.py index d1a324822..9b503fbf2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -34,3 +34,20 @@ class TestUtils(TestDvc): fd.write(b'a\r\nb\r\nc') self.assertEqual(utils.file_md5('cr')[0], utils.file_md5('crlf')[0]) + + def test_dict_md5(self): + d = {'cmd': 'python code.py foo file1', + 'locked': 'true', + 'outs': [{'path': 'file1', + 'metric': {'type': 'raw'}, + 'cache': False, + 'md5':'acbd18db4cc2f85cedef654fccc4a4d8'}], + 'deps': [{'path': 'foo', + 'md5': 'acbd18db4cc2f85cedef654fccc4a4d8'}, + {'path': 'code.py', + 'md5': 'd05447644b89960913c7eee5fd776adb'}]} + + + md5 = '8b263fa05ede6c3145c164829be694b4' + + self.assertEqual(md5, utils.dict_md5(d, exclude=['metric', 'locked']))