From 8c4712710c87e13f5d193a1d15e36686d8776543 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Thu, 11 Jan 2018 09:45:43 -0800 Subject: [PATCH] Missing data source error message --- dvc/executor.py | 3 +- dvc/stage.py | 78 ++++++++++++++++++++++++++++++------------ tests/test_add.py | 14 ++++---- tests/test_checkout.py | 4 +-- tests/test_gc.py | 4 +-- tests/test_remove.py | 4 +-- 6 files changed, 70 insertions(+), 37 deletions(-) diff --git a/dvc/executor.py b/dvc/executor.py index 2b89c46d1..8bdd2323d 100644 --- a/dvc/executor.py +++ b/dvc/executor.py @@ -58,6 +58,5 @@ class Executor(object): code, out, err = Executor.exec_cmd(cmd, stdout_file=stdout_file, stderr_file=stderr_file, cwd=cwd, shell=shell) if code != 0: - cmd_str = ' '.join(cmd) if type(cmd) == list else cmd - raise ExecutorError('Exec command error ({}):\n{}\n{}'.format(cmd_str, out, err)) + raise ExecutorError('Exec command error:\n{}'.format(err)) return out diff --git a/dvc/stage.py b/dvc/stage.py index 3fd696478..d39c20658 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -10,33 +10,49 @@ from dvc.executor import Executor class OutputError(DvcException): + pass + + +class MissingDataSource(OutputError): + def __init__(self, missing_files): + assert len(missing_files) > 0 + + source = 'source' + if len(missing_files) > 1: + source += 's' + + msg = u'missing data {}: {}'.format(source, ', '.join(missing_files)) + super(MissingDataSource, self).__init__(msg) + + +class CmdOutputError(DvcException): def __init__(self, path, msg): - super(OutputError, self).__init__('Output file \'{}\' error: {}'.format(path, msg)) + super(CmdOutputError, self).__init__('Output file \'{}\' error: {}'.format(path, msg)) -class OutputNoCacheError(OutputError): +class CmdOutputNoCacheError(CmdOutputError): def __init__(self, path): - super(OutputNoCacheError, self).__init__(path, 'no cache') + super(CmdOutputNoCacheError, self).__init__(path, 'no cache') -class OutputOutsideOfRepoError(OutputError): +class CmdOutputOutsideOfRepoError(CmdOutputError): def __init__(self, path): - super(OutputOutsideOfRepoError, self).__init__(path, 'outside of repository') + super(CmdOutputOutsideOfRepoError, self).__init__(path, 'outside of repository') -class OutputDoesNotExistError(OutputError): +class CmdOutputDoesNotExistError(CmdOutputError): def __init__(self, path): - super(OutputDoesNotExistError, self).__init__(path, 'does not exist') + super(CmdOutputDoesNotExistError, self).__init__(path, 'does not exist') -class OutputIsNotFileError(OutputError): +class CmdOutputIsNotFileError(CmdOutputError): def __init__(self, path): - super(OutputIsNotFileError, self).__init__(path, 'not a file') + super(CmdOutputIsNotFileError, self).__init__(path, 'not a file') -class OutputAlreadyTrackedError(OutputError): +class CmdOutputAlreadyTrackedError(CmdOutputError): def __init__(self, path): - super(OutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)') + super(CmdOutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)') class Dependency(object): @@ -48,7 +64,7 @@ class Dependency(object): self.path = os.path.abspath(os.path.realpath(path)) if not self.path.startswith(self.project.root_dir): - raise OutputOutsideOfRepoError(self.path) + raise CmdOutputOutsideOfRepoError(self.path) self.md5 = md5 @@ -56,6 +72,10 @@ class Dependency(object): def dvc_path(self): return os.path.relpath(self.path, self.project.root_dir) + @property + def rel_path(self): + return os.path.relpath(self.path, '.') + def _changed_md5(self): if not os.path.exists(self.path): return True @@ -76,9 +96,9 @@ class Dependency(object): def update(self): if not os.path.exists(self.path): - raise OutputDoesNotExistError(self.path) + raise CmdOutputDoesNotExistError(self.rel_path) if not os.path.isfile(self.path): - raise OutputIsNotFileError(self.path) + raise CmdOutputIsNotFileError(self.path) state = self.project.state.get(self.path) if state and state.mtime == self.mtime(): @@ -164,10 +184,10 @@ class Output(Dependency): def link(self, checkout=False): if not self.use_cache: - raise OutputNoCacheError(self.path) + raise CmdOutputNoCacheError(self.path) if not os.path.exists(self.path) and not os.path.exists(self.cache): - raise OutputNoCacheError(self.path) + raise CmdOutputNoCacheError(self.path) if os.path.exists(self.path) and \ os.path.exists(self.cache) and \ @@ -187,7 +207,7 @@ class Output(Dependency): src = self.path link = self.cache else: - raise OutputNoCacheError(self.path) + raise CmdOutputNoCacheError(self.path) System.hardlink(src, link) @@ -207,7 +227,7 @@ class Output(Dependency): return if self.project.scm.is_tracked(self.path): - raise OutputAlreadyTrackedError(self.path) + raise CmdOutputAlreadyTrackedError(self.path) self.project.scm.ignore(self.path) self.link() @@ -249,6 +269,10 @@ class Stage(object): def dvc_path(self): return os.path.relpath(self.path, self.project.root_dir) + @property + def is_data_source(self): + return self.cmd is None + @staticmethod def is_stage_file(path): if not os.path.isfile(path): @@ -282,9 +306,7 @@ class Stage(object): # Removing outputs only if we actually have command to reproduce self.remove_outs() - self.project.logger.info("Reproducing {}:\n\t{}".format(self.relpath, self.cmd)) self.run() - self.project.logger.debug("{} was reproduced".format(self.relpath)) @staticmethod def loadd(project, d, path): @@ -335,9 +357,21 @@ class Stage(object): out.save() def run(self): - if self.cmd: + if not self.is_data_source: + self.project.logger.info(u'Reproducing \'{}\':\n\t{}'.format(self.relpath, self.cmd)) + Executor.exec_cmd_only_success(self.cmd, cwd=str(self.cwd), shell=True) - self.save() + self.save() + + self.project.logger.debug(u'\'{}\' was reproduced'.format(self.relpath)) + else: + self.project.logger.info(u'Verifying data sources in \'{}\''.format(self.relpath)) + self.check_missing_outputs() + + def check_missing_outputs(self): + missing_outs = [out.rel_path for out in self.outs if not os.path.exists(out.rel_path)] + if missing_outs: + raise MissingDataSource(missing_outs) def checkout(self): for out in self.outs: diff --git a/tests/test_add.py b/tests/test_add.py index f2778a278..3de517896 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -4,9 +4,9 @@ import filecmp from dvc.main import main from dvc.data_cloud import file_md5 -from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError -from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError -from dvc.stage import OutputAlreadyTrackedError +from dvc.stage import Stage, CmdOutputNoCacheError, CmdOutputOutsideOfRepoError +from dvc.stage import CmdOutputDoesNotExistError, CmdOutputIsNotFileError +from dvc.stage import CmdOutputAlreadyTrackedError from dvc.project import StageNotFoundError from dvc.command.add import CmdAdd @@ -30,13 +30,13 @@ class TestAdd(TestDvc): class TestAddNonExistentFile(TestDvc): def test(self): - with self.assertRaises(OutputDoesNotExistError) as cx: + with self.assertRaises(CmdOutputDoesNotExistError) as cx: self.dvc.add('non_existent_file') class TestAddFileOutsideOfRepo(TestDvc): def test(self): - with self.assertRaises(OutputOutsideOfRepoError) as cx: + with self.assertRaises(CmdOutputOutsideOfRepoError) as cx: self.dvc.add(os.path.join(os.path.dirname(self.dvc.root_dir), self.FOO)) @@ -44,7 +44,7 @@ class TestAddDirectory(TestDvc): def test(self): dname = 'directory' os.mkdir(dname) - with self.assertRaises(OutputIsNotFileError) as cx: + with self.assertRaises(CmdOutputIsNotFileError) as cx: self.dvc.add(dname) @@ -55,7 +55,7 @@ class TestAddTrackedFile(TestDvc): self.dvc.scm.add([fname]) self.dvc.scm.commit('add {}'.format(fname)) - with self.assertRaises(OutputAlreadyTrackedError) as cx: + with self.assertRaises(CmdOutputAlreadyTrackedError) as cx: self.dvc.add(fname) diff --git a/tests/test_checkout.py b/tests/test_checkout.py index c1c9ce004..62bc25193 100644 --- a/tests/test_checkout.py +++ b/tests/test_checkout.py @@ -5,8 +5,8 @@ import filecmp from dvc.main import main from dvc.data_cloud import file_md5 -from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError -from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError +from dvc.stage import Stage, CmdOutputNoCacheError, CmdOutputOutsideOfRepoError +from dvc.stage import CmdOutputDoesNotExistError, CmdOutputIsNotFileError from dvc.project import StageNotFoundError from tests.test_repro import TestRepro diff --git a/tests/test_gc.py b/tests/test_gc.py index ef06fe75a..e7e042dea 100644 --- a/tests/test_gc.py +++ b/tests/test_gc.py @@ -4,8 +4,8 @@ import filecmp from dvc.main import main from dvc.data_cloud import file_md5 -from dvc.stage import Stage, OutputNoCacheError, OutputOutsideOfRepoError -from dvc.stage import OutputDoesNotExistError, OutputIsNotFileError +from dvc.stage import Stage, CmdOutputNoCacheError, CmdOutputOutsideOfRepoError +from dvc.stage import CmdOutputDoesNotExistError, CmdOutputIsNotFileError from dvc.project import StageNotFoundError from dvc.command.gc import CmdGC diff --git a/tests/test_remove.py b/tests/test_remove.py index 591c99bd9..1e893c065 100644 --- a/tests/test_remove.py +++ b/tests/test_remove.py @@ -1,7 +1,7 @@ import os from dvc.main import main -from dvc.stage import Stage, OutputOutsideOfRepoError +from dvc.stage import Stage, CmdOutputOutsideOfRepoError from dvc.project import StageNotFoundError from dvc.command.remove import CmdRemove @@ -26,7 +26,7 @@ class TestRemoveNonExistentFile(TestDvc): class TestRemoveFileOutsideOfRepo(TestDvc): def test(self): - with self.assertRaises(OutputOutsideOfRepoError) as cx: + with self.assertRaises(CmdOutputOutsideOfRepoError) as cx: self.dvc.remove(os.path.join(os.path.dirname(self.dvc.root_dir), self.FOO))