data status: optimize printing status list

`--untracked-files` and `--unchanged` coupled with `--granular`
puts a lot of stress on `show_status`. Currently, with those options,
dvc spends ~26s on `_show_status` function.

Unhappy with the above result, I looked into improving the situation with
`console.out`. It did reduce that to 9s but it still felt like a lot.

Then I dropped to good old colorama, which dropped to 3s(term)/1.24(redirected).
I tested with buffering output which dropped this further to
~600ms(term)/~100ms(redirected).

This may seem like premature optimization, but this was preventing me
from figuring out perf issues in granular outputs, and I believe the
user should not be penalize for using feature that we already offer.
This commit is contained in:
Saugat Pachhai (सौगात)
2022-07-25 17:47:23 +05:45
committed by skshetry
parent fd7dcae8aa
commit 896acedb07
2 changed files with 9 additions and 5 deletions
+7 -4
View File
@@ -2,11 +2,12 @@ import argparse
import logging
from typing import TYPE_CHECKING
from funcy import compact, log_durations
from funcy import chunks, compact, log_durations
from dvc.cli.command import CmdBase
from dvc.cli.utils import append_doc_link, fix_subparsers
from dvc.ui import ui
from dvc.utils import colorize
if TYPE_CHECKING:
from dvc.repo.data import Status as DataStatus
@@ -72,7 +73,7 @@ class CmdDataStatus(CmdBase):
label = cls.LABELS.get(stage, stage.capitalize() + " files")
header = f"{label}:"
color = cls.COLORS.get(stage, "normal")
color = cls.COLORS.get(stage, None)
ui.write(header)
if hint := cls.HINTS.get(stage):
@@ -86,8 +87,10 @@ class CmdDataStatus(CmdBase):
else:
items = stage_status
for item in items:
ui.write(f"\t[{color}]{item}[/]".expandtabs(8), styled=True)
tabs = "\t".expandtabs(8)
for chunk in chunks(1000, items):
out = "\n".join(tabs + item for item in chunk)
ui.write(colorize(out, color))
if (hint := cls.HINTS.get("git_dirty")) and git_info.get("is_dirty"):
message = hint.format("other " if result else "")
+2 -1
View File
@@ -7,6 +7,7 @@ from dvc.cli import main, parse_args
from dvc.commands.data import CmdDataStatus
from dvc.repo import Repo
from dvc.repo.data import Status
from tests.func.parsing.test_errors import escape_ansi
@pytest.fixture
@@ -135,5 +136,5 @@ DVC unchanged files:
expected_out += """\
(there are other changes not tracked by dvc, use "git status" to see)
"""
assert out == expected_out
assert escape_ansi(out) == expected_out
assert not err