From 958eab2f23ab35f6378b0912f70953aa237115a3 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Tue, 20 Mar 2018 00:31:30 +0200 Subject: [PATCH] dvc: fix checkout/repro with stage files without md5 Signed-off-by: Ruslan Kuprieiev --- dvc/output.py | 12 +++++++++--- dvc/project.py | 2 +- tests/test_checkout.py | 17 +++++++++++++++++ tests/test_repro.py | 18 ++++++++++++++++++ 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/dvc/output.py b/dvc/output.py index f93a2957c..342bed3b6 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -132,10 +132,16 @@ class Output(Dependency): @property def cache(self): + if not self.md5: + return None + return self.project.cache.get(self.md5) @property def rel_cache(self): + if not self.cache: + return None + return os.path.relpath(self.cache) def dumpd(self, cwd): @@ -198,10 +204,10 @@ class Output(Dependency): return {i['relpath']: i['md5'] for i in dir_info} def changed(self): - ret = True - if not self.use_cache: ret = super(Output, self).changed() + elif not self.cache: + ret = True elif self.is_dir_cache(self.cache): ret = self._changed_dir() else: @@ -280,7 +286,7 @@ class Output(Dependency): self.project.logger.debug(msg.format(self.rel_path, self.rel_cache)) return - if not os.path.exists(self.cache): + if not self.cache or not os.path.exists(self.cache): self.project.logger.warn(u'\'{}\': cache file not found'.format(self.rel_path)) self.remove() return diff --git a/dvc/project.py b/dvc/project.py index 46647fa80..27f5287d1 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -182,7 +182,7 @@ class Project(object): for stage in stages: for out in stage.outs: - if not out.use_cache: + if not out.use_cache or not out.cache: continue cache_set |= set([out.cache]) if out.is_dir_cache(out.cache) and os.path.isfile(out.cache): diff --git a/tests/test_checkout.py b/tests/test_checkout.py index ae3148fce..784569a5a 100644 --- a/tests/test_checkout.py +++ b/tests/test_checkout.py @@ -1,4 +1,5 @@ import os +import yaml import time import stat import shutil @@ -7,6 +8,8 @@ import filecmp from dvc.main import main from tests.basic_env import TestDvc from tests.test_repro import TestRepro +from dvc.stage import Stage +from dvc.output import Output class TestCheckout(TestRepro): @@ -146,3 +149,17 @@ class TestGitIgnoreWhenCheckout(CheckoutBase): main(['checkout']) ignored = self.read_ignored() self.assertIn(fname_branch, ignored) + + +class TestCheckoutMissingMd5InStageFile(TestRepro): + def test(self): + with open(self.file1_stage, 'r') as fd: + d = yaml.load(fd) + + del(d[Stage.PARAM_OUTS][0][Output.PARAM_MD5]) + del(d[Stage.PARAM_DEPS][0][Output.PARAM_MD5]) + + with open(self.file1_stage, 'w') as fd: + yaml.dump(d, fd) + + self.dvc.checkout() diff --git a/tests/test_repro.py b/tests/test_repro.py index afd58ddd5..0fd9eb41b 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -1,4 +1,5 @@ import os +import yaml import stat import shutil import filecmp @@ -7,6 +8,8 @@ from dvc.main import main from dvc.command.repro import CmdRepro from dvc.project import ReproductionError from dvc.utils import file_md5 +from dvc.output import Output +from dvc.stage import Stage from tests.basic_env import TestDvc @@ -135,6 +138,21 @@ class TestReproChangedDir(TestDvc): self.assertEqual(len(stages), 1) +class TestReproMissingMd5InStageFile(TestRepro): + def test(self): + with open(self.file1_stage, 'r') as fd: + d = yaml.load(fd) + + del(d[Stage.PARAM_OUTS][0][Output.PARAM_MD5]) + del(d[Stage.PARAM_DEPS][0][Output.PARAM_MD5]) + + with open(self.file1_stage, 'w') as fd: + yaml.dump(d, fd) + + stages = self.dvc.reproduce(self.file1_stage) + self.assertEqual(len(stages), 1) + + class TestCmdRepro(TestRepro): def test(self): ret = main(['repro',