From 896acedb07ca7a1983fde5052a17d8775847042b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?= =?UTF-8?q?=E0=A4=A4=29?= Date: Mon, 25 Jul 2022 17:13:25 +0545 Subject: [PATCH] data status: optimize printing status list `--untracked-files` and `--unchanged` coupled with `--granular` puts a lot of stress on `show_status`. Currently, with those options, dvc spends ~26s on `_show_status` function. Unhappy with the above result, I looked into improving the situation with `console.out`. It did reduce that to 9s but it still felt like a lot. Then I dropped to good old colorama, which dropped to 3s(term)/1.24(redirected). I tested with buffering output which dropped this further to ~600ms(term)/~100ms(redirected). This may seem like premature optimization, but this was preventing me from figuring out perf issues in granular outputs, and I believe the user should not be penalize for using feature that we already offer. --- dvc/commands/data.py | 11 +++++++---- tests/unit/command/test_data_status.py | 3 ++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dvc/commands/data.py b/dvc/commands/data.py index b4b38c856..b459478cc 100644 --- a/dvc/commands/data.py +++ b/dvc/commands/data.py @@ -2,11 +2,12 @@ import argparse import logging from typing import TYPE_CHECKING -from funcy import compact, log_durations +from funcy import chunks, compact, log_durations from dvc.cli.command import CmdBase from dvc.cli.utils import append_doc_link, fix_subparsers from dvc.ui import ui +from dvc.utils import colorize if TYPE_CHECKING: from dvc.repo.data import Status as DataStatus @@ -72,7 +73,7 @@ class CmdDataStatus(CmdBase): label = cls.LABELS.get(stage, stage.capitalize() + " files") header = f"{label}:" - color = cls.COLORS.get(stage, "normal") + color = cls.COLORS.get(stage, None) ui.write(header) if hint := cls.HINTS.get(stage): @@ -86,8 +87,10 @@ class CmdDataStatus(CmdBase): else: items = stage_status - for item in items: - ui.write(f"\t[{color}]{item}[/]".expandtabs(8), styled=True) + tabs = "\t".expandtabs(8) + for chunk in chunks(1000, items): + out = "\n".join(tabs + item for item in chunk) + ui.write(colorize(out, color)) if (hint := cls.HINTS.get("git_dirty")) and git_info.get("is_dirty"): message = hint.format("other " if result else "") diff --git a/tests/unit/command/test_data_status.py b/tests/unit/command/test_data_status.py index eb6109091..7387ab75a 100644 --- a/tests/unit/command/test_data_status.py +++ b/tests/unit/command/test_data_status.py @@ -7,6 +7,7 @@ from dvc.cli import main, parse_args from dvc.commands.data import CmdDataStatus from dvc.repo import Repo from dvc.repo.data import Status +from tests.func.parsing.test_errors import escape_ansi @pytest.fixture @@ -135,5 +136,5 @@ DVC unchanged files: expected_out += """\ (there are other changes not tracked by dvc, use "git status" to see) """ - assert out == expected_out + assert escape_ansi(out) == expected_out assert not err