From 5f46f05992ae0ab79c29e432255d962631d5641f Mon Sep 17 00:00:00 2001 From: Dmitry Date: Tue, 9 Jan 2018 18:23:28 -0800 Subject: [PATCH 1/3] Bugfix: create a cache dir for freshly checkout-ed repo --- dvc/cache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dvc/cache.py b/dvc/cache.py index feec198f7..a211c5916 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -8,6 +8,8 @@ class Cache(object): def __init__(self, dvc_dir): self.cache_dir = os.path.abspath(os.path.realpath(os.path.join(dvc_dir, self.CACHE_DIR))) + if not os.path.exists(self.cache_dir): + os.mkdir(self.cache_dir) @staticmethod def init(dvc_dir): From f11183446e05ff1acfcffbd836b4390b311aab0f Mon Sep 17 00:00:00 2001 From: Dmitry Date: Tue, 9 Jan 2018 19:02:42 -0800 Subject: [PATCH 2/3] repro: print correct dvc file name near with error message --- dvc/command/repro.py | 9 ++++++--- dvc/project.py | 14 +++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index c8bb524a9..08e5e7cfd 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -1,5 +1,6 @@ -from dvc.exceptions import DvcException from dvc.command.common.base import CmdBase +from dvc.project import ReproductionError + class CmdRepro(CmdBase): def run(self): @@ -9,8 +10,10 @@ class CmdRepro(CmdBase): self.project.reproduce(target, recursive=recursive, force=self.args.force) - except DvcException as ex: - msg = 'Failed to reproduce {}: {}'.format(target, str(ex)) + except ReproductionError as ex: + self.project.logger.error(ex) + except Exception as ex: + msg = 'Failed to reproduce \'{}\' - unexpected error: {}'.format(target, ex) self.project.logger.error(msg) return 1 return 0 diff --git a/dvc/project.py b/dvc/project.py index 8cf5f5084..5e38f1ed3 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -26,6 +26,11 @@ class StageNotFoundError(DvcException): super(StageNotFoundError, self).__init__(msg) +class ReproductionError(DvcException): + def __init__(self, dvc_file_name, msg): + super(ReproductionError, self).__init__(u'Failed to reproduce \'{}\': {}'.format(dvc_file_name, msg)) + + class Pipeline(object): def __init__(self, project, G): @@ -176,9 +181,12 @@ class Project(object): if recursive: for n in nx.dfs_postorder_nodes(self.graph(), node): - stages[n].reproduce(force=force) - stages[n].dump() - reproduced.append(stages[n]) + try: + stages[n].reproduce(force=force) + stages[n].dump() + reproduced.append(stages[n]) + except Exception as ex: + raise ReproductionError(n, ex.message) stages[node].reproduce(force=force) stages[node].dump() From 8cc38877693d160cbfd94e99355a8e620271e1c7 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Wed, 10 Jan 2018 09:57:22 -0800 Subject: [PATCH 3/3] Refactoring: extract method _reproduce_stages() and ReproductionError class changes --- dvc/project.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/dvc/project.py b/dvc/project.py index 5e38f1ed3..f60d9e2c1 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -28,6 +28,8 @@ class StageNotFoundError(DvcException): class ReproductionError(DvcException): def __init__(self, dvc_file_name, msg): + self.dvc_file_name = dvc_file_name + self.msg = msg super(ReproductionError, self).__init__(u'Failed to reproduce \'{}\': {}'.format(dvc_file_name, msg)) @@ -173,20 +175,12 @@ class Project(object): return stage def reproduce(self, target, recursive=True, force=False): - reproduced = [] stages = nx.get_node_attributes(self.graph(), 'stage') node = os.path.relpath(os.path.abspath(target), self.root_dir) if node not in stages: raise StageNotFoundError(target) - if recursive: - for n in nx.dfs_postorder_nodes(self.graph(), node): - try: - stages[n].reproduce(force=force) - stages[n].dump() - reproduced.append(stages[n]) - except Exception as ex: - raise ReproductionError(n, ex.message) + reproduced = self._reproduce_stages(force, node, stages) if recursive else [] stages[node].reproduce(force=force) stages[node].dump() @@ -194,6 +188,17 @@ class Project(object): return reproduced + def _reproduce_stages(self, force, node, stages): + result = [] + for n in nx.dfs_postorder_nodes(self.graph(), node): + try: + stages[n].reproduce(force=force) + stages[n].dump() + result.append(stages[n]) + except Exception as ex: + raise ReproductionError(n, ex.message) + return result + def checkout(self): for stage in self.stages(): stage.checkout()