From 720608f379df6b4db6e751d7fb3902a3626695db Mon Sep 17 00:00:00 2001 From: pared Date: Thu, 22 Aug 2019 12:55:49 +0200 Subject: [PATCH 1/5] repo: make repo.collect collect all stages, not active --- dvc/repo/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index e1139d90a..5cf814991 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -164,7 +164,7 @@ class Repo(object): from dvc.stage import Stage if not target or (recursive and os.path.isdir(target)): - return self.active_stages(target) + return self.stages(target) stage = Stage.load(self, target) if not with_deps: From d02327fb531deee299caa42682d171debf64e93b Mon Sep 17 00:00:00 2001 From: pared Date: Thu, 22 Aug 2019 12:56:52 +0200 Subject: [PATCH 2/5] repo: make repo.active_stages collect active stages only --- dvc/repo/__init__.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 5cf814991..7e8bd20ba 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -353,15 +353,25 @@ class Repo(object): return G, G_active - def pipelines(self, from_directory=None): + @staticmethod + def _pipelines(graph): import networkx as nx - G, G_active = self.graph(from_directory=from_directory) - return [ - G.subgraph(c).copy() for c in nx.weakly_connected_components(G) + graph.subgraph(c).copy() + for c in nx.weakly_connected_components(graph) ] + def pipelines(self, from_directory=None): + G, _ = self.graph(from_directory=from_directory) + + return self._pipelines(G) + + def pipelines_for_active_nodes(self, from_directory=None): + _, G_active = self.graph(from_directory=from_directory) + + return self._pipelines(G_active) + @staticmethod def _filter_out_dirs(dirs, outs, root_dir): def filter_dirs(dname): @@ -417,7 +427,7 @@ class Repo(object): import networkx as nx stages = [] - for G in self.pipelines(from_directory): + for G in self.pipelines_for_active_nodes(from_directory): stages.extend(list(nx.get_node_attributes(G, "stage").values())) return stages From 16b62d154a4b208426b3f62500e6acc6c9539760 Mon Sep 17 00:00:00 2001 From: pared Date: Thu, 22 Aug 2019 12:58:04 +0200 Subject: [PATCH 3/5] gc: remove unnecessary arguments for gc._load_all_used_cache --- dvc/repo/gc.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dvc/repo/gc.py b/dvc/repo/gc.py index 89b16e35c..bfae569ff 100644 --- a/dvc/repo/gc.py +++ b/dvc/repo/gc.py @@ -21,9 +21,7 @@ def _merge_cache_lists(clists): def _load_all_used_cache( repos, - target=None, all_branches=False, - active=True, with_deps=False, all_tags=False, remote=None, @@ -76,9 +74,7 @@ def gc( all_clists = _load_all_used_cache( all_repos, - target=None, all_branches=all_branches, - active=False, with_deps=with_deps, all_tags=all_tags, remote=remote, From babd5b2a12172cb3260fd757fee1a6054692c30d Mon Sep 17 00:00:00 2001 From: pawel Date: Mon, 26 Aug 2019 21:54:10 +0200 Subject: [PATCH 4/5] repo: drop active_stages --- dvc/repo/__init__.py | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 7e8bd20ba..d79e2882f 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -225,8 +225,6 @@ class Repo(object): stages.extend( self.collect(target, with_deps=with_deps) ) - elif active: - stages = self.active_stages() else: stages = self.stages() @@ -353,25 +351,15 @@ class Repo(object): return G, G_active - @staticmethod - def _pipelines(graph): + def pipelines(self, from_directory=None): import networkx as nx + G, G_active = self.graph(from_directory=from_directory) + return [ - graph.subgraph(c).copy() - for c in nx.weakly_connected_components(graph) + G.subgraph(c).copy() for c in nx.weakly_connected_components(G) ] - def pipelines(self, from_directory=None): - G, _ = self.graph(from_directory=from_directory) - - return self._pipelines(G) - - def pipelines_for_active_nodes(self, from_directory=None): - _, G_active = self.graph(from_directory=from_directory) - - return self._pipelines(G_active) - @staticmethod def _filter_out_dirs(dirs, outs, root_dir): def filter_dirs(dname): @@ -423,14 +411,6 @@ class Repo(object): return stages - def active_stages(self, from_directory=None): - import networkx as nx - - stages = [] - for G in self.pipelines_for_active_nodes(from_directory): - stages.extend(list(nx.get_node_attributes(G, "stage").values())) - return stages - def find_outs_by_path(self, path, outs=None, recursive=False): if not outs: # there is no `from_directory=path` argument because some data From a9efabbc6e62dc15fe03b4e3b5d60d2685c06870 Mon Sep 17 00:00:00 2001 From: pawel Date: Tue, 27 Aug 2019 14:08:00 +0200 Subject: [PATCH 5/5] repo: used_cache: remove active flag --- dvc/repo/__init__.py | 9 --------- dvc/repo/gc.py | 1 - 2 files changed, 10 deletions(-) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index d79e2882f..2f743a6ff 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -183,7 +183,6 @@ class Repo(object): self, targets=None, all_branches=False, - active=True, with_deps=False, all_tags=False, remote=None, @@ -229,14 +228,6 @@ class Repo(object): stages = self.stages() for stage in stages: - if active and not targets and stage.locked: - logger.warning( - "DVC-file '{path}' is locked. Its dependencies are" - " not going to be pushed/pulled/fetched.".format( - path=stage.relpath - ) - ) - for out in stage.outs: scheme = out.path_info.scheme used_cache = out.get_used_cache( diff --git a/dvc/repo/gc.py b/dvc/repo/gc.py index bfae569ff..c334b5ebe 100644 --- a/dvc/repo/gc.py +++ b/dvc/repo/gc.py @@ -35,7 +35,6 @@ def _load_all_used_cache( repo_clist = repo.used_cache( targets=None, all_branches=all_branches, - active=False, with_deps=with_deps, all_tags=all_tags, remote=remote,