From 47208533a25ec99ebfbd7b62144601fc3d4ea7bf Mon Sep 17 00:00:00 2001 From: Adam Liter Date: Mon, 20 Feb 2023 16:53:38 -0500 Subject: [PATCH] Add metrics_show to dvc.api (#9042) * refactor: add metrics_show, rename module to show * tests: add functional tests for metrics_show * api.show: return empty dict for no params/metrics * test_show: Return relpaths in `metrics_repo`. --------- Co-authored-by: codamuse Co-authored-by: David de la Iglesia Castro --- dvc/api/__init__.py | 3 +- dvc/api/{params.py => show.py} | 183 ++++++++++++++--- tests/func/api/test_params.py | 207 ------------------- tests/func/api/test_show.py | 363 +++++++++++++++++++++++++++++++++ 4 files changed, 515 insertions(+), 241 deletions(-) rename dvc/api/{params.py => show.py} (63%) delete mode 100644 tests/func/api/test_params.py create mode 100644 tests/func/api/test_show.py diff --git a/dvc/api/__init__.py b/dvc/api/__init__.py index 442de716a..d4007eb2d 100644 --- a/dvc/api/__init__.py +++ b/dvc/api/__init__.py @@ -3,13 +3,14 @@ from dvc.fs.dvc import _DVCFileSystem as DVCFileSystem from .data import open # pylint: disable=redefined-builtin from .data import get_url, read from .experiments import make_checkpoint -from .params import params_show +from .show import metrics_show, params_show __all__ = [ "get_url", "make_checkpoint", "open", "params_show", + "metrics_show", "read", "DVCFileSystem", ] diff --git a/dvc/api/params.py b/dvc/api/show.py similarity index 63% rename from dvc/api/params.py rename to dvc/api/show.py index 00c03f0d9..a9156dcb2 100644 --- a/dvc/api/params.py +++ b/dvc/api/show.py @@ -4,10 +4,153 @@ from typing import Dict, Iterable, Optional, Union from funcy import first -from dvc.exceptions import DvcException from dvc.repo import Repo +def _onerror_raise(exception: Exception, *args, **kwargs): + raise exception + + +def _postprocess(results): + processed: Dict[str, Dict] = {} + for rev, rev_data in results.items(): + if not rev_data: + continue + + processed[rev] = {} + + counts: typing.Counter[str] = Counter() + for file_data in rev_data["data"].values(): + for k in file_data["data"]: + counts[k] += 1 + + for file_name, file_data in rev_data["data"].items(): + to_merge = { + (k if counts[k] == 1 else f"{file_name}:{k}"): v + for k, v in file_data["data"].items() + } + processed[rev] = {**processed[rev], **to_merge} + + if "workspace" in processed: + del processed["workspace"] + + return processed + + +def metrics_show( + *targets: str, + repo: Optional[str] = None, + rev: Optional[str] = None, +) -> Dict: + """Get metrics tracked in `repo`. + + Without arguments, this function will retrieve all metrics from all tracked + metric files, for the current working tree. + + See the options below to restrict the metrics retrieved. + + Args: + *targets (str, optional): Names of the metric files to retrieve + metrics from. For example, "classifier_eval.json, + clustering_eval.json". + If no `targets` are provided, all metric files tracked in `dvc.yaml` + will be used. + Note that targets don't necessarily have to be defined in `dvc.yaml`. + repo (str, optional): Location of the DVC repository. + Defaults to the current project (found by walking up from the + current working directory tree). + It can be a URL or a file system path. + Both HTTP and SSH protocols are supported for online Git repos + (e.g. [user@]server:project.git). + rev (str, optional): Name of the `Git revision`_ to retrieve metrics + from. + Defaults to `None`. + An example of git revision can be a branch or tag name, a commit + hash or a dvc experiment name. + If `repo` is not a Git repo, this option is ignored. + If `None`, the current working tree will be used. + + Returns: + Dict: See Examples below. + + Examples: + + - No arguments. + + Working on https://github.com/iterative/example-get-started + + >>> import dvc.api + >>> metrics = dvc.api.metrics_show() + >>> print(json.dumps(metrics, indent=4)) + { + "avg_prec": 0.9249974999612706, + "roc_auc": 0.9460213440787918 + } + + --- + + - Using `rev`. + + Working on https://github.com/iterative/example-get-started + + >>> import json + >>> import dvc.api + >>> metrics = dvc.api.metrics_show(rev="tune-hyperparams") + >>> print(json.dumps(metrics, indent=4)) + { + "avg_prec": 0.9268792615819422, + "roc_auc": 0.945093365854111 + } + + --- + + - Using `targets`. + + Working on https://github.com/iterative/example-get-started + + >>> import json + >>> import dvc.api + >>> metrics = dvc.api.metrics_show("evaluation.json") + >>> print(json.dumps(metrics, indent=4)) + { + "avg_prec": 0.9249974999612706, + "roc_auc": 0.9460213440787918 + } + + --- + + - Git URL as `repo`. + + >>> import json + >>> import dvc.api + >>> metrics = dvc.api.metrics_show( + ... repo="https://github.com/iterative/demo-fashion-mnist") + >>> print(json.dumps(metrics, indent=4)) + { + "loss": 0.25284987688064575, + "accuracy": 0.9071000218391418 + } + + + .. _Git revision: + https://git-scm.com/docs/revisions + """ + + with Repo.open(repo) as _repo: + metrics = _repo.metrics.show( + targets=targets, + revs=rev if rev is None else [rev], + onerror=_onerror_raise, + ) + + metrics = _postprocess(metrics) + + if not metrics: + return {} + + return metrics[first(metrics)] + + def params_show( *targets: str, repo: Optional[str] = None, @@ -56,9 +199,6 @@ def params_show( Returns: Dict: See Examples below. - Raises: - DvcException: If no params are found in `repo`. - Examples: - No arguments. @@ -239,34 +379,6 @@ def params_show( if isinstance(stages, str): stages = [stages] - def _onerror_raise(_result: Dict, exception: Exception, *args, **kwargs): - raise exception - - def _postprocess(params): - processed: Dict[str, Dict] = {} - for rev, rev_data in params.items(): - processed[rev] = {} - - counts: typing.Counter[str] = Counter() - for file_data in rev_data["data"].values(): - for k in file_data["data"]: - counts[k] += 1 - - for file_name, file_data in rev_data["data"].items(): - to_merge = { - (k if counts[k] == 1 else f"{file_name}:{k}"): v - for k, v in file_data["data"].items() - } - processed[rev] = {**processed[rev], **to_merge} - - if "workspace" in processed: - del processed["workspace"] - - if not processed: - raise DvcException("No params found") - - return processed[first(processed)] - with Repo.open(repo) as _repo: params = _repo.params.show( revs=rev if rev is None else [rev], @@ -276,4 +388,9 @@ def params_show( stages=stages, ) - return _postprocess(params) + params = _postprocess(params) + + if not params: + return {} + + return params[first(params)] diff --git a/tests/func/api/test_params.py b/tests/func/api/test_params.py deleted file mode 100644 index 909161a7c..000000000 --- a/tests/func/api/test_params.py +++ /dev/null @@ -1,207 +0,0 @@ -import os -from textwrap import dedent - -import pytest - -from dvc import api -from dvc.exceptions import DvcException - - -@pytest.fixture -def params_repo(tmp_dir, scm, dvc): - tmp_dir.gen("params.yaml", "foo: 1") - tmp_dir.gen("params.json", '{"bar": 2, "foobar": 3}') - tmp_dir.gen("other_params.json", '{"foo": {"bar": 4}}') - - dvc.run( - name="stage-0", - cmd="echo stage-0", - ) - - dvc.run( - name="stage-1", - cmd="echo stage-1", - params=["foo", "params.json:bar"], - ) - - dvc.run( - name="stage-2", - cmd="echo stage-2", - params=["other_params.json:foo"], - ) - - dvc.run( - name="stage-3", - cmd="echo stage-2", - params=["params.json:foobar"], - ) - - scm.add( - [ - "params.yaml", - "params.json", - "other_params.json", - "dvc.yaml", - "dvc.lock", - ] - ) - scm.commit("commit dvc files") - - tmp_dir.gen("params.yaml", "foo: 5") - scm.add(["params.yaml"]) - scm.commit("update params.yaml") - - -def test_params_show_no_args(params_repo): - assert api.params_show() == { - "params.yaml:foo": 5, - "bar": 2, - "foobar": 3, - "other_params.json:foo": {"bar": 4}, - } - - -def test_params_show_targets(params_repo): - assert api.params_show("params.yaml") == {"foo": 5} - assert api.params_show("params.yaml", "params.json") == { - "foo": 5, - "bar": 2, - "foobar": 3, - } - assert api.params_show("params.yaml", stages="stage-1") == { - "foo": 5, - } - - -def test_params_show_deps(params_repo): - params = api.params_show(deps=True) - assert params == { - "params.yaml:foo": 5, - "bar": 2, - "foobar": 3, - "other_params.json:foo": {"bar": 4}, - } - - -def test_params_show_stages(params_repo): - assert api.params_show(stages="stage-2") == {"foo": {"bar": 4}} - - assert api.params_show() == api.params_show( - stages=["stage-1", "stage-2", "stage-3"] - ) - - assert api.params_show("params.json", stages="stage-3") == {"foobar": 3} - - with pytest.raises(DvcException, match="No params found"): - api.params_show(stages="stage-0") - - -def test_params_show_stage_addressing(tmp_dir, dvc): - for subdir in {"subdir1", "subdir2"}: - subdir = tmp_dir / subdir - subdir.mkdir() - with subdir.chdir(): - subdir.gen("params.yaml", "foo: 1") - - dvc.run(name="stage-0", cmd="echo stage-0", params=["foo"]) - - for s in {"subdir1", "subdir2"}: - dvcyaml = os.path.join(s, "dvc.yaml") - assert api.params_show(stages=f"{dvcyaml}:stage-0") == {"foo": 1} - - with subdir.chdir(): - nested = subdir / "nested" - nested.mkdir() - with nested.chdir(): - dvcyaml = os.path.join("..", "dvc.yaml") - assert api.params_show(stages=f"{dvcyaml}:stage-0") == {"foo": 1} - - -def test_params_show_revs(params_repo): - assert api.params_show(rev="HEAD~1") == { - "params.yaml:foo": 1, - "bar": 2, - "foobar": 3, - "other_params.json:foo": {"bar": 4}, - } - - -def test_params_show_while_running_stage(tmp_dir, dvc): - (tmp_dir / "params.yaml").dump({"foo": {"bar": 1}}) - (tmp_dir / "params.json").dump({"bar": 2}) - - tmp_dir.gen( - "merge.py", - dedent( - """ - import json - from dvc import api - with open("merged.json", "w") as f: - json.dump(api.params_show(stages="merge"), f) - """ - ), - ) - dvc.stage.add( - name="merge", - cmd="python merge.py", - params=["foo.bar", {"params.json": ["bar"]}], - outs=["merged.json"], - ) - - dvc.reproduce() - - assert (tmp_dir / "merged.json").parse() == {"foo": {"bar": 1}, "bar": 2} - - -def test_params_show_repo(tmp_dir, erepo_dir): - with erepo_dir.chdir(): - erepo_dir.scm_gen("params.yaml", "foo: 1", commit="Create params.yaml") - erepo_dir.dvc.run( - name="stage-1", - cmd="echo stage-1", - params=["foo"], - ) - assert api.params_show(repo=erepo_dir) == {"foo": 1} - - -def test_params_show_no_params_found(tmp_dir, dvc): - # Empty repo - with pytest.raises(DvcException, match="No params found"): - api.params_show() - - # params.yaml but no dvc.yaml - (tmp_dir / "params.yaml").dump({"foo": 1}) - assert api.params_show() == {"foo": 1} - - # dvc.yaml but no params.yaml - (tmp_dir / "params.yaml").unlink() - dvc.stage.add(name="echo", cmd="echo foo") - with pytest.raises(DvcException, match="No params found"): - api.params_show() - - -def test_params_show_stage_without_params(tmp_dir, dvc): - tmp_dir.gen("params.yaml", "foo: 1") - - dvc.run( - name="stage-0", - cmd="echo stage-0", - ) - - with pytest.raises(DvcException, match="No params found"): - api.params_show(stages="stage-0") - - with pytest.raises(DvcException, match="No params found"): - api.params_show(deps=True) - - -def test_params_show_untracked_target(params_repo, tmp_dir): - tmp_dir.gen("params_foo.yaml", "foo: 1") - - assert api.params_show("params_foo.yaml") == {"foo": 1} - - with pytest.raises(DvcException, match="No params found"): - api.params_show("params_foo.yaml", stages="stage-0") - - with pytest.raises(DvcException, match="No params found"): - api.params_show("params_foo.yaml", deps=True) diff --git a/tests/func/api/test_show.py b/tests/func/api/test_show.py new file mode 100644 index 000000000..1e926e678 --- /dev/null +++ b/tests/func/api/test_show.py @@ -0,0 +1,363 @@ +import json +import os +from textwrap import dedent +from typing import Dict, List + +import pytest + +from dvc import api + +TRAIN_METRICS: List[Dict[str, Dict[str, float]]] = [ + { + "avg_prec": {"train": 0.85, "val": 0.75}, + "roc_auc": {"train": 0.80, "val": 0.70}, + }, + { + "avg_prec": {"train": 0.97, "val": 0.92}, + "roc_auc": {"train": 0.98, "val": 0.94}, + }, +] +TEST_METRICS: List[Dict[str, Dict[str, float]]] = [ + {"avg_prec": {"test": 0.72}, "roc_auc": {"test": 0.77}}, + { + "avg_prec": {"test": 0.91}, + "roc_auc": {"test": 0.92}, + }, +] + + +@pytest.fixture +def params_repo(tmp_dir, scm, dvc): + tmp_dir.gen("params.yaml", "foo: 1") + tmp_dir.gen("params.json", '{"bar": 2, "foobar": 3}') + tmp_dir.gen("other_params.json", '{"foo": {"bar": 4}}') + + dvc.run( + name="stage-0", + cmd="echo stage-0", + ) + + dvc.run( + name="stage-1", + cmd="echo stage-1", + params=["foo", "params.json:bar"], + ) + + dvc.run( + name="stage-2", + cmd="echo stage-2", + params=["other_params.json:foo"], + ) + + dvc.run( + name="stage-3", + cmd="echo stage-2", + params=["params.json:foobar"], + ) + + scm.add( + [ + "params.yaml", + "params.json", + "other_params.json", + "dvc.yaml", + "dvc.lock", + ] + ) + scm.commit("commit dvc files") + + tmp_dir.gen("params.yaml", "foo: 5") + scm.add(["params.yaml"]) + scm.commit("update params.yaml") + + +@pytest.fixture +def metrics_repo(tmp_dir, scm, dvc, run_copy_metrics): + dvc.run(name="prepare", cmd="echo preparing data") + scm.add(["dvc.yaml", "dvc.lock"]) + scm.commit("prepare data") + sub_dir = tmp_dir / "eval" + sub_dir.mkdir() + tmp_dir.gen( + "tmp_train_val_metrics.json", + json.dumps(TRAIN_METRICS[0]), + ) + train_metrics_file = os.path.join(sub_dir, "train_val_metrics.json") + run_copy_metrics( + "tmp_train_val_metrics.json", + train_metrics_file, + name="train", + metrics_no_cache=[train_metrics_file], + ) + (tmp_dir / "tmp_train_val_metrics.json").unlink() + + scm.add(["dvc.yaml", "dvc.lock", train_metrics_file]) + scm.commit("train model") + + test_metrics_file = os.path.join(sub_dir, "test_metrics.json") + tmp_dir.gen("tmp_test_metrics.json", json.dumps(TEST_METRICS[0])) + run_copy_metrics( + "tmp_test_metrics.json", + test_metrics_file, + name="test", + metrics_no_cache=[test_metrics_file], + ) + (tmp_dir / "tmp_test_metrics.json").unlink() + + scm.add(["dvc.yaml", "dvc.lock", test_metrics_file]) + scm.commit("test model") + + with tmp_dir.branch("better-model", new=True): + tmp_dir.gen( + "tmp_train_val_metrics.json", + json.dumps(TRAIN_METRICS[1]), + ) + run_copy_metrics( + "tmp_train_val_metrics.json", + train_metrics_file, + name="train", + metrics_no_cache=[train_metrics_file], + ) + (tmp_dir / "tmp_train_val_metrics.json").unlink() + + scm.add(["dvc.yaml", "dvc.lock", train_metrics_file]) + scm.commit("train better model") + + tmp_dir.gen("tmp_test_metrics.json", json.dumps(TEST_METRICS[1])) + run_copy_metrics( + "tmp_test_metrics.json", + test_metrics_file, + name="test", + metrics_no_cache=[test_metrics_file], + ) + (tmp_dir / "tmp_test_metrics.json").unlink() + + scm.add(["dvc.yaml", "dvc.lock", test_metrics_file]) + scm.commit("test better model") + + scm.checkout("master") + + return ( + os.path.relpath(train_metrics_file, tmp_dir), + os.path.relpath(test_metrics_file, tmp_dir), + ) + + +def test_params_show_no_args(params_repo): + assert api.params_show() == { + "params.yaml:foo": 5, + "bar": 2, + "foobar": 3, + "other_params.json:foo": {"bar": 4}, + } + + +def test_params_show_targets(params_repo): + assert api.params_show("params.yaml") == {"foo": 5} + assert api.params_show("params.yaml", "params.json") == { + "foo": 5, + "bar": 2, + "foobar": 3, + } + assert api.params_show("params.yaml", stages="stage-1") == { + "foo": 5, + } + + +def test_params_show_deps(params_repo): + params = api.params_show(deps=True) + assert params == { + "params.yaml:foo": 5, + "bar": 2, + "foobar": 3, + "other_params.json:foo": {"bar": 4}, + } + + +def test_params_show_stages(params_repo): + assert api.params_show(stages="stage-2") == {"foo": {"bar": 4}} + + assert api.params_show() == api.params_show( + stages=["stage-1", "stage-2", "stage-3"] + ) + + assert api.params_show("params.json", stages="stage-3") == {"foobar": 3} + + assert api.params_show(stages="stage-0") == {} + + +def test_params_show_stage_addressing(tmp_dir, dvc): + for subdir in {"subdir1", "subdir2"}: + subdir = tmp_dir / subdir + subdir.mkdir() + with subdir.chdir(): + subdir.gen("params.yaml", "foo: 1") + + dvc.run(name="stage-0", cmd="echo stage-0", params=["foo"]) + + for s in {"subdir1", "subdir2"}: + dvcyaml = os.path.join(s, "dvc.yaml") + assert api.params_show(stages=f"{dvcyaml}:stage-0") == {"foo": 1} + + with subdir.chdir(): + nested = subdir / "nested" + nested.mkdir() + with nested.chdir(): + dvcyaml = os.path.join("..", "dvc.yaml") + assert api.params_show(stages=f"{dvcyaml}:stage-0") == {"foo": 1} + + +def test_params_show_revs(params_repo): + assert api.params_show(rev="HEAD~1") == { + "params.yaml:foo": 1, + "bar": 2, + "foobar": 3, + "other_params.json:foo": {"bar": 4}, + } + + +def test_params_show_while_running_stage(tmp_dir, dvc): + (tmp_dir / "params.yaml").dump({"foo": {"bar": 1}}) + (tmp_dir / "params.json").dump({"bar": 2}) + + tmp_dir.gen( + "merge.py", + dedent( + """ + import json + from dvc import api + with open("merged.json", "w") as f: + json.dump(api.params_show(stages="merge"), f) + """ + ), + ) + dvc.stage.add( + name="merge", + cmd="python merge.py", + params=["foo.bar", {"params.json": ["bar"]}], + outs=["merged.json"], + ) + + dvc.reproduce() + + assert (tmp_dir / "merged.json").parse() == {"foo": {"bar": 1}, "bar": 2} + + +def test_params_show_repo(tmp_dir, erepo_dir): + with erepo_dir.chdir(): + erepo_dir.scm_gen("params.yaml", "foo: 1", commit="Create params.yaml") + erepo_dir.dvc.run( + name="stage-1", + cmd="echo stage-1", + params=["foo"], + ) + assert api.params_show(repo=erepo_dir) == {"foo": 1} + + +def test_params_show_no_params_found(tmp_dir, dvc): + # Empty repo + assert api.params_show() == {} + + # params.yaml but no dvc.yaml + (tmp_dir / "params.yaml").dump({"foo": 1}) + assert api.params_show() == {"foo": 1} + + # dvc.yaml but no params.yaml + (tmp_dir / "params.yaml").unlink() + dvc.stage.add(name="echo", cmd="echo foo") + assert api.params_show() == {} + + +def test_params_show_stage_without_params(tmp_dir, dvc): + tmp_dir.gen("params.yaml", "foo: 1") + + dvc.run( + name="stage-0", + cmd="echo stage-0", + ) + + assert api.params_show(stages="stage-0") == {} + + assert api.params_show(deps=True) == {} + + +def test_params_show_untracked_target(params_repo, tmp_dir): + tmp_dir.gen("params_foo.yaml", "foo: 1") + + assert api.params_show("params_foo.yaml") == {"foo": 1} + + assert api.params_show("params_foo.yaml", stages="stage-0") == {} + + assert api.params_show("params_foo.yaml", deps=True) == {} + + +def test_metrics_show_no_args(metrics_repo): + train_metrics_file, test_metrics_file = metrics_repo + assert api.metrics_show() == { + f"{train_metrics_file}:avg_prec": TRAIN_METRICS[0]["avg_prec"], + f"{train_metrics_file}:roc_auc": TRAIN_METRICS[0]["roc_auc"], + f"{test_metrics_file}:avg_prec": TEST_METRICS[0]["avg_prec"], + f"{test_metrics_file}:roc_auc": TEST_METRICS[0]["roc_auc"], + } + + +def test_metrics_show_targets(metrics_repo): + train_metrics_file, test_metrics_file = metrics_repo + assert api.metrics_show(train_metrics_file) == TRAIN_METRICS[0] + assert api.metrics_show(test_metrics_file) == TEST_METRICS[0] + assert api.metrics_show(train_metrics_file, test_metrics_file) == { + f"{train_metrics_file}:avg_prec": TRAIN_METRICS[0]["avg_prec"], + f"{train_metrics_file}:roc_auc": TRAIN_METRICS[0]["roc_auc"], + f"{test_metrics_file}:avg_prec": TEST_METRICS[0]["avg_prec"], + f"{test_metrics_file}:roc_auc": TEST_METRICS[0]["roc_auc"], + } + + +def test_metrics_show_no_metrics_found(tmp_dir, dvc): + # Empty repo + assert api.metrics_show() == {} + + # dvc.yaml but no metrics + dvc.stage.add(name="echo", cmd="echo foo") + assert api.metrics_show() == {} + + +def test_metrics_show_rev_without_metrics(metrics_repo): + assert api.metrics_show(rev="HEAD~2") == {} + + +def test_metrics_show_rev_with_metrics(metrics_repo): + train_metrics_file, test_metrics_file = metrics_repo + assert api.metrics_show(rev="HEAD~1") == TRAIN_METRICS[0] + assert api.metrics_show(rev="HEAD") == { + f"{train_metrics_file}:avg_prec": TRAIN_METRICS[0]["avg_prec"], + f"{train_metrics_file}:roc_auc": TRAIN_METRICS[0]["roc_auc"], + f"{test_metrics_file}:avg_prec": TEST_METRICS[0]["avg_prec"], + f"{test_metrics_file}:roc_auc": TEST_METRICS[0]["roc_auc"], + } + assert api.metrics_show(rev="better-model~1") == { + f"{train_metrics_file}:avg_prec": TRAIN_METRICS[1]["avg_prec"], + f"{train_metrics_file}:roc_auc": TRAIN_METRICS[1]["roc_auc"], + f"{test_metrics_file}:avg_prec": TEST_METRICS[0]["avg_prec"], + f"{test_metrics_file}:roc_auc": TEST_METRICS[0]["roc_auc"], + } + assert api.metrics_show(rev="better-model") == { + f"{train_metrics_file}:avg_prec": TRAIN_METRICS[1]["avg_prec"], + f"{train_metrics_file}:roc_auc": TRAIN_METRICS[1]["roc_auc"], + f"{test_metrics_file}:avg_prec": TEST_METRICS[1]["avg_prec"], + f"{test_metrics_file}:roc_auc": TEST_METRICS[1]["roc_auc"], + } + + +def test_metrics_show_dirty_working_dir(metrics_repo, tmp_dir): + train_metrics_file, test_metrics_file = metrics_repo + new_metrics = {"acc": 1} + (tmp_dir / train_metrics_file).unlink() + (tmp_dir / train_metrics_file).dump(new_metrics) + (tmp_dir / test_metrics_file).unlink() + (tmp_dir / test_metrics_file).dump(new_metrics) + + assert api.metrics_show() == { + f"{train_metrics_file}:acc": new_metrics["acc"], + f"{test_metrics_file}:acc": new_metrics["acc"], + }