mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 15:28:28 +01:00
The code are wrapped due to us using line-length=79 before, but now some of the code can easily fit in 88 chars. The formatter did not change this automatically when we changed line-length, because of magic trailing commas that gets added when they turn into multiple lines. Black/ruff wrapping them in parens and splitting them into multiple lines was making it difficult to read.
431 lines
13 KiB
Python
431 lines
13 KiB
Python
from collections import abc
|
|
from collections.abc import (
|
|
ItemsView,
|
|
Iterable,
|
|
Iterator,
|
|
Mapping,
|
|
MutableSequence,
|
|
Sequence,
|
|
)
|
|
from itertools import chain, repeat, zip_longest
|
|
from operator import itemgetter
|
|
from typing import TYPE_CHECKING, Any, Optional, Union, overload
|
|
|
|
from funcy import reraise
|
|
|
|
if TYPE_CHECKING:
|
|
from dvc.ui.table import CellT
|
|
|
|
|
|
class Column(list["CellT"]):
|
|
pass
|
|
|
|
|
|
def with_value(value, default):
|
|
return default if value is None else value
|
|
|
|
|
|
class TabularData(MutableSequence[Sequence["CellT"]]):
|
|
def __init__(self, columns: Sequence[str], fill_value: Optional[str] = ""):
|
|
self._columns: dict[str, Column] = {name: Column() for name in columns}
|
|
self._keys: list[str] = list(columns)
|
|
self._fill_value = fill_value
|
|
self._protected: set[str] = set()
|
|
|
|
@property
|
|
def columns(self) -> list[Column]:
|
|
return list(map(self.column, self.keys()))
|
|
|
|
def is_protected(self, col_name) -> bool:
|
|
return col_name in self._protected
|
|
|
|
def protect(self, *col_names: str):
|
|
self._protected.update(col_names)
|
|
|
|
def unprotect(self, *col_names: str):
|
|
self._protected = self._protected.difference(col_names)
|
|
|
|
def column(self, name: str) -> Column:
|
|
return self._columns[name]
|
|
|
|
def items(self) -> ItemsView[str, Column]:
|
|
projection = {k: self.column(k) for k in self.keys()}
|
|
return projection.items()
|
|
|
|
def keys(self) -> list[str]:
|
|
return self._keys
|
|
|
|
def _iter_col_row(self, row: Sequence["CellT"]) -> Iterator[tuple["CellT", Column]]:
|
|
for val, col in zip_longest(row, self.columns):
|
|
if col is None:
|
|
break
|
|
yield with_value(val, self._fill_value), col
|
|
|
|
def append(self, value: Sequence["CellT"]) -> None:
|
|
for val, col in self._iter_col_row(value):
|
|
col.append(val)
|
|
|
|
def extend(self, values: Iterable[Sequence["CellT"]]) -> None:
|
|
for row in values:
|
|
self.append(row)
|
|
|
|
def insert(self, index: int, value: Sequence["CellT"]) -> None:
|
|
for val, col in self._iter_col_row(value):
|
|
col.insert(index, val)
|
|
|
|
def __iter__(self) -> Iterator[list["CellT"]]:
|
|
return map(list, zip(*self.columns))
|
|
|
|
def __getattr__(self, item: str) -> Column:
|
|
with reraise(KeyError, AttributeError):
|
|
return self.column(item)
|
|
|
|
def __getitem__(self, item: Union[int, slice]):
|
|
func = itemgetter(item)
|
|
it = map(func, self.columns)
|
|
if isinstance(item, slice):
|
|
it = map(list, zip(*it))
|
|
return list(it)
|
|
|
|
@overload
|
|
def __setitem__(self, item: int, value: Sequence["CellT"]) -> None:
|
|
...
|
|
|
|
@overload
|
|
def __setitem__(self, item: slice, value: Iterable[Sequence["CellT"]]) -> None:
|
|
...
|
|
|
|
def __setitem__(self, item, value) -> None:
|
|
it = value
|
|
if isinstance(item, slice):
|
|
n = len(self.columns)
|
|
normalized_rows = (
|
|
chain(val, repeat(self._fill_value, n - len(val))) for val in value
|
|
)
|
|
# we need to transpose those rows into columnar format
|
|
# as we work in terms of column-based arrays
|
|
it = zip(*normalized_rows)
|
|
|
|
for i, col in self._iter_col_row(it):
|
|
col[item] = i
|
|
|
|
def __delitem__(self, item: Union[int, slice]) -> None:
|
|
for col in self.columns:
|
|
del col[item]
|
|
|
|
def __len__(self) -> int:
|
|
if not self._columns:
|
|
return 0
|
|
return len(self.columns[0])
|
|
|
|
@property
|
|
def shape(self) -> tuple[int, int]:
|
|
return len(self.columns), len(self)
|
|
|
|
def drop(self, *col_names: str) -> None:
|
|
for col in col_names:
|
|
if not self.is_protected(col):
|
|
self._keys.remove(col)
|
|
self._columns.pop(col)
|
|
|
|
def rename(self, from_col_name: str, to_col_name: str) -> None:
|
|
self._columns[to_col_name] = self._columns.pop(from_col_name)
|
|
self._keys[self._keys.index(from_col_name)] = to_col_name
|
|
|
|
def project(self, *col_names: str) -> None:
|
|
self.drop(*(set(self._keys) - set(col_names)))
|
|
self._keys = list(col_names)
|
|
|
|
def is_empty(self, col_name: str) -> bool:
|
|
col = self.column(col_name)
|
|
return not any(item != self._fill_value for item in col)
|
|
|
|
def to_csv(self) -> str:
|
|
import csv
|
|
from io import StringIO
|
|
|
|
buff = StringIO()
|
|
writer = csv.writer(buff)
|
|
writer.writerow(self.keys())
|
|
|
|
for row in self:
|
|
writer.writerow(row)
|
|
return buff.getvalue()
|
|
|
|
def add_column(self, name: str) -> None:
|
|
self._columns[name] = Column([self._fill_value] * len(self))
|
|
self._keys.append(name)
|
|
|
|
def row_from_dict(self, d: Mapping[str, "CellT"]) -> None:
|
|
keys = self.keys()
|
|
for key in d:
|
|
if key not in keys:
|
|
self.add_column(key)
|
|
|
|
row: list["CellT"] = [
|
|
with_value(d.get(key), self._fill_value) for key in self.keys()
|
|
]
|
|
self.append(row)
|
|
|
|
def render(self, **kwargs: Any):
|
|
from dvc.ui import ui
|
|
|
|
if kwargs.pop("csv", False):
|
|
ui.write(self.to_csv(), end="")
|
|
else:
|
|
ui.table(self, headers=self.keys(), **kwargs)
|
|
|
|
def as_dict(
|
|
self, cols: Optional[Iterable[str]] = None
|
|
) -> Iterable[dict[str, "CellT"]]:
|
|
keys = self.keys() if cols is None else set(cols)
|
|
return [{k: self._columns[k][i] for k in keys} for i in range(len(self))]
|
|
|
|
def dropna( # noqa: C901, PLR0912
|
|
self,
|
|
axis: str = "rows",
|
|
how="any",
|
|
subset: Optional[Iterable[str]] = None,
|
|
):
|
|
if axis not in ["rows", "cols"]:
|
|
raise ValueError(
|
|
f"Invalid 'axis' value {axis}.Choose one of ['rows', 'cols']"
|
|
)
|
|
if how not in ["any", "all"]:
|
|
raise ValueError(f"Invalid 'how' value {how}. Choose one of ['any', 'all']")
|
|
|
|
match_line: set = set()
|
|
match_any = True
|
|
if how == "all":
|
|
match_any = False
|
|
|
|
for n_row, row in enumerate(self):
|
|
for n_col, col in enumerate(row):
|
|
if subset and self.keys()[n_col] not in subset:
|
|
continue
|
|
if (col == self._fill_value) is match_any:
|
|
if axis == "rows":
|
|
match_line.add(n_row)
|
|
break
|
|
match_line.add(self.keys()[n_col])
|
|
|
|
to_drop = match_line
|
|
if how == "all":
|
|
if axis == "rows":
|
|
to_drop = set(range(len(self)))
|
|
else:
|
|
to_drop = set(self.keys())
|
|
to_drop -= match_line
|
|
|
|
if axis == "rows":
|
|
for name in self.keys():
|
|
self._columns[name] = Column(
|
|
[x for n, x in enumerate(self._columns[name]) if n not in to_drop]
|
|
)
|
|
else:
|
|
self.drop(*to_drop)
|
|
|
|
def drop_duplicates( # noqa: C901
|
|
self,
|
|
axis: str = "rows",
|
|
subset: Optional[Iterable[str]] = None,
|
|
ignore_empty: bool = True,
|
|
):
|
|
if axis not in ["rows", "cols"]:
|
|
raise ValueError(
|
|
f"Invalid 'axis' value {axis}.Choose one of ['rows', 'cols']"
|
|
)
|
|
|
|
if axis == "cols":
|
|
cols_to_drop: list[str] = []
|
|
for n_col, col in enumerate(self.columns):
|
|
if subset and self.keys()[n_col] not in subset:
|
|
continue
|
|
# Cast to str because Text is not hashable error
|
|
unique_vals = {str(x) for x in col}
|
|
if ignore_empty and self._fill_value in unique_vals:
|
|
unique_vals -= {self._fill_value}
|
|
if len(unique_vals) == 1:
|
|
cols_to_drop.append(self.keys()[n_col])
|
|
self.drop(*cols_to_drop)
|
|
|
|
elif axis == "rows":
|
|
unique_rows = []
|
|
rows_to_drop: list[int] = []
|
|
for n_row, row in enumerate(self):
|
|
if subset:
|
|
row = [
|
|
col
|
|
for n_col, col in enumerate(row)
|
|
if self.keys()[n_col] in subset
|
|
]
|
|
|
|
tuple_row = tuple(row)
|
|
if tuple_row in unique_rows:
|
|
rows_to_drop.append(n_row)
|
|
else:
|
|
unique_rows.append(tuple_row)
|
|
|
|
for name in self.keys():
|
|
self._columns[name] = Column(
|
|
[
|
|
x
|
|
for n, x in enumerate(self._columns[name])
|
|
if n not in rows_to_drop
|
|
]
|
|
)
|
|
|
|
|
|
def _normalize_float(val: float, precision: int):
|
|
return f"{val:.{precision}g}"
|
|
|
|
|
|
def _format_field(
|
|
val: Any, precision: Optional[int] = None, round_digits: bool = False
|
|
) -> str:
|
|
def _format(_val):
|
|
if isinstance(_val, float) and precision:
|
|
if round_digits:
|
|
return round(_val, precision)
|
|
return _normalize_float(_val, precision)
|
|
if isinstance(_val, abc.Mapping):
|
|
return {k: _format(v) for k, v in _val.items()}
|
|
if isinstance(_val, list):
|
|
return [_format(x) for x in _val]
|
|
return _val
|
|
|
|
return str(_format(val))
|
|
|
|
|
|
def diff_table(
|
|
diff,
|
|
title: str,
|
|
old: bool = True,
|
|
no_path: bool = False,
|
|
show_changes: bool = True,
|
|
precision: Optional[int] = None,
|
|
round_digits: bool = False,
|
|
on_empty_diff: Optional[str] = None,
|
|
a_rev: Optional[str] = None,
|
|
b_rev: Optional[str] = None,
|
|
) -> TabularData:
|
|
a_rev = a_rev or "HEAD"
|
|
b_rev = b_rev or "workspace"
|
|
headers: list[str] = ["Path", title, a_rev, b_rev, "Change"]
|
|
fill_value = "-"
|
|
td = TabularData(headers, fill_value=fill_value)
|
|
|
|
for fname, diff_in_file in diff.items():
|
|
for item, change in sorted(diff_in_file.items()):
|
|
old_value = with_value(change.get("old"), fill_value)
|
|
new_value = with_value(change.get("new"), fill_value)
|
|
diff_value = with_value(change.get("diff", on_empty_diff), fill_value)
|
|
td.append(
|
|
[
|
|
fname,
|
|
str(item),
|
|
_format_field(old_value, precision, round_digits),
|
|
_format_field(new_value, precision, round_digits),
|
|
_format_field(diff_value, precision, round_digits),
|
|
]
|
|
)
|
|
|
|
if no_path:
|
|
td.drop("Path")
|
|
|
|
if not show_changes:
|
|
td.drop("Change")
|
|
|
|
if not old:
|
|
td.drop(a_rev)
|
|
td.rename(b_rev, "Value")
|
|
|
|
return td
|
|
|
|
|
|
def show_diff( # noqa: PLR0913
|
|
diff,
|
|
title: str,
|
|
old: bool = True,
|
|
no_path: bool = False,
|
|
show_changes: bool = True,
|
|
precision: Optional[int] = None,
|
|
round_digits: bool = False,
|
|
on_empty_diff: Optional[str] = None,
|
|
markdown: bool = False,
|
|
a_rev: Optional[str] = None,
|
|
b_rev: Optional[str] = None,
|
|
) -> None:
|
|
td = diff_table(
|
|
diff,
|
|
title=title,
|
|
old=old,
|
|
no_path=no_path,
|
|
show_changes=show_changes,
|
|
precision=precision,
|
|
round_digits=round_digits,
|
|
on_empty_diff=on_empty_diff,
|
|
a_rev=a_rev,
|
|
b_rev=b_rev,
|
|
)
|
|
td.render(markdown=markdown)
|
|
|
|
|
|
def metrics_table(
|
|
metrics,
|
|
all_branches: bool = False,
|
|
all_tags: bool = False,
|
|
all_commits: bool = False,
|
|
precision: Optional[int] = None,
|
|
round_digits: bool = False,
|
|
):
|
|
from dvc.utils.diff import format_dict
|
|
from dvc.utils.flatten import flatten
|
|
|
|
td = TabularData(["Revision", "Path"], fill_value="-")
|
|
|
|
for branch, val in metrics.items():
|
|
for fname, metric in val.get("data", {}).items():
|
|
row_data: dict[str, str] = {"Revision": branch, "Path": fname}
|
|
metric = metric.get("data", {})
|
|
flattened = (
|
|
flatten(format_dict(metric))
|
|
if isinstance(metric, dict)
|
|
else {"": metric}
|
|
)
|
|
row_data.update(
|
|
{
|
|
k: _format_field(v, precision, round_digits)
|
|
for k, v in flattened.items()
|
|
}
|
|
)
|
|
td.row_from_dict(row_data)
|
|
|
|
rev, path, *metrics_headers = td.keys()
|
|
td.project(rev, path, *sorted(metrics_headers))
|
|
|
|
if not any([all_branches, all_tags, all_commits]):
|
|
td.drop("Revision")
|
|
|
|
return td
|
|
|
|
|
|
def show_metrics(
|
|
metrics,
|
|
markdown: bool = False,
|
|
all_branches: bool = False,
|
|
all_tags: bool = False,
|
|
all_commits: bool = False,
|
|
precision: Optional[int] = None,
|
|
round_digits: bool = False,
|
|
) -> None:
|
|
td = metrics_table(
|
|
metrics,
|
|
all_branches=all_branches,
|
|
all_tags=all_tags,
|
|
all_commits=all_commits,
|
|
precision=precision,
|
|
round_digits=round_digits,
|
|
)
|
|
td.render(markdown=markdown)
|