From 470825aec507c1c348367f5b87234d607fbd94b4 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 15 Sep 2018 07:21:58 +0300 Subject: [PATCH] dvc: don't scan output directories for dvc files `dvc status` for directory with 100K files sped up from ~6 sec to ~2 sec. Fixes #1121 Signed-off-by: Ruslan Kuprieiev --- dvc/project.py | 18 +++++++++++++++++- dvc/scm.py | 4 ++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/dvc/project.py b/dvc/project.py index 98e58f59a..eb5dc80ff 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -746,12 +746,28 @@ class Project(object): def stages(self): stages = [] + outs = [] for root, dirs, files in os.walk(self.root_dir): for fname in files: path = os.path.join(root, fname) if not Stage.is_stage_file(path): continue - stages.append(Stage.load(self, path)) + stage = Stage.load(self, path) + for out in stage.outs: + outs.append(out.path + out.sep) + stages.append(stage) + + def filter_dirs(dname): + path = os.path.join(root, dname) + if path == self.dvc_dir or path == self.scm.dir: + return False + for out in outs: + if path == os.path.normpath(out) or path.startswith(out): + return False + return True + + dirs[:] = list(filter(filter_dirs, dirs)) + return stages def active_stages(self): diff --git a/dvc/scm.py b/dvc/scm.py index eadaf9943..5ac1d16ca 100644 --- a/dvc/scm.py +++ b/dvc/scm.py @@ -99,6 +99,10 @@ class Git(Base): git_dir = os.path.join(root_dir, Git.GIT_DIR) return os.path.isdir(git_dir) + @property + def dir(self): + return self.repo.git_dir + def ignore_file(self): return self.GITIGNORE