From 36d53bd4031f6703550848649c60b1088d9369c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?=
 =?UTF-8?q?=E0=A4=A4=29?= <suagatchhetri@outlook.com>
Date: Wed, 20 Apr 2022 19:09:38 +0545
Subject: [PATCH] dvc.stage: log time duration on stage collection walk in
 verbose mode

---
 .dvcignore        |  4 ++++
 .gitignore        | 16 +++++++++++++---
 dvc/repo/index.py |  5 +++++
 dvc/repo/stage.py | 22 ++++++++++++++++------
 4 files changed, 38 insertions(+), 9 deletions(-)
 create mode 100644 .dvcignore

diff --git a/.dvcignore b/.dvcignore
new file mode 100644
index 000000000..7fab7ecf4
--- /dev/null
+++ b/.dvcignore
@@ -0,0 +1,4 @@
+/tests
+/dvc
+/.github
+/bin
diff --git a/.gitignore b/.gitignore
index ce78cdb25..89824507c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,10 +1,13 @@
+__pycache__/
 neatlynx.conf
-neatlynx/__pycache__
 .idea
 /cache
 *.pyc
-.env/
-.env2.7/
+
+.env*/
+.venv
+env/
+venv/
 .python-version
 
 .dvc.conf.lock
@@ -26,6 +29,7 @@ innosetup/config.ini
 
 .coverage
 .coverage.*
+coverage.xml
 
 *.sw?
 
@@ -40,3 +44,9 @@ scripts/ci/gcp-creds.json
 
 *~
 /dvc/_dvc_version.py
+
+.mypy_cache/
+.pytest_cache/
+.nox/
+.tox/
+htmlcov/
diff --git a/dvc/repo/index.py b/dvc/repo/index.py
index f896c9375..b822efc5c 100644
--- a/dvc/repo/index.py
+++ b/dvc/repo/index.py
@@ -298,10 +298,15 @@ class Index:
 
 
 if __name__ == "__main__":
+    import logging
+
     from funcy import log_durations
 
+    from dvc.logger import setup
     from dvc.repo import Repo
 
+    setup(level=logging.TRACE)  # type: ignore[attr-defined]
+
     repo = Repo()
     index = Index(repo, repo.fs)
     print(index)
diff --git a/dvc/repo/stage.py b/dvc/repo/stage.py
index 7d3ba604c..702cad896 100644
--- a/dvc/repo/stage.py
+++ b/dvc/repo/stage.py
@@ -1,6 +1,7 @@
 import fnmatch
 import logging
 import os
+import time
 import typing
 from contextlib import suppress
 from functools import partial, wraps
@@ -15,6 +16,8 @@ from typing import (
     Union,
 )
 
+from funcy.debug import format_time
+
 from dvc.exceptions import (
     DvcException,
     NoOutputOrStageError,
@@ -36,6 +39,14 @@ if typing.TYPE_CHECKING:
 PIPELINE_FILE = "dvc.yaml"
 
 
+def log_walk(seq):
+    for root, dirs, files in seq:
+        start = time.perf_counter()
+        yield root, dirs, files
+        duration = format_time(time.perf_counter() - start)
+        logger.trace("%s in collecting stages from %s", duration, root)
+
+
 class StageInfo(NamedTuple):
     stage: "Stage"
     filter_info: Optional[str] = None
@@ -470,12 +481,11 @@ class StageLoad:
             # trailing slash needed to check if a directory is gitignored
             return dir_path in outs or is_ignored(f"{dir_path}{sep}")
 
-        for root, dirs, files in self.repo.dvcignore.walk(
-            self.fs, self.repo.root_dir
-        ):
-            logger.trace(  # type: ignore[attr-defined]
-                "Collect repo walking '%s'", root
-            )
+        walk_iter = self.repo.dvcignore.walk(self.fs, self.repo.root_dir)
+        if logger.isEnabledFor(logging.TRACE):  # type: ignore[attr-defined]
+            walk_iter = log_walk(walk_iter)
+
+        for root, dirs, files in walk_iter:
             dvcfile_filter = partial(is_dvcfile_and_not_ignored, root)
             for file in filter(dvcfile_filter, files):
                 file_path = os.path.join(root, file)