mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
131 lines
3.6 KiB
Python
131 lines
3.6 KiB
Python
from textwrap import dedent
|
|
|
|
import pytest
|
|
|
|
from dvc import stage as stage_module
|
|
from dvc.exceptions import MetricsError
|
|
|
|
LIVE_SCRITP = dedent(
|
|
"""
|
|
import dvclive
|
|
import sys
|
|
r = 2
|
|
for i in range(r):
|
|
dvclive.log("loss", 1-i/r)
|
|
dvclive.log("accuracy", i/r)
|
|
dvclive.next_step()"""
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def live_stage(tmp_dir, scm, dvc):
|
|
|
|
pytest.skip("dvclive does not exist yet")
|
|
|
|
def make(summary=True, report=True):
|
|
tmp_dir.gen("train.py", LIVE_SCRITP)
|
|
tmp_dir.gen("params.yaml", "foo: 1")
|
|
stage = dvc.run(
|
|
cmd="python train.py",
|
|
params=["foo"],
|
|
deps=["train.py"],
|
|
name="live_stage",
|
|
live="logs",
|
|
live_summary=summary,
|
|
live_report=report,
|
|
)
|
|
|
|
scm.add(["dvc.yaml", "dvc.lock", "train.py", "params.yaml"])
|
|
scm.commit("initial: live_stage")
|
|
return stage
|
|
|
|
yield make
|
|
|
|
|
|
@pytest.mark.parametrize("report", (True, False))
|
|
@pytest.mark.parametrize("summary", (True, False))
|
|
def test_export_config_tmp(tmp_dir, dvc, mocker, summary, report):
|
|
run_spy = mocker.spy(stage_module.run, "_run")
|
|
tmp_dir.gen("src", "dependency")
|
|
dvc.run(
|
|
cmd="mkdir logs && touch logs.json",
|
|
deps=["src"],
|
|
name="run_logger",
|
|
live="logs",
|
|
live_summary=summary,
|
|
live_report=report,
|
|
)
|
|
|
|
assert run_spy.call_count == 1
|
|
_, kwargs = run_spy.call_args
|
|
|
|
assert "DVCLIVE_PATH" in kwargs["env"]
|
|
assert kwargs["env"]["DVCLIVE_PATH"] == "logs"
|
|
|
|
assert "DVCLIVE_SUMMARY" in kwargs["env"]
|
|
assert kwargs["env"]["DVCLIVE_SUMMARY"] == str(int(summary))
|
|
|
|
assert "DVCLIVE_REPORT" in kwargs["env"]
|
|
assert kwargs["env"]["DVCLIVE_REPORT"] == str(int(report))
|
|
|
|
|
|
@pytest.mark.parametrize("summary", (True, False))
|
|
def test_export_config(tmp_dir, dvc, mocker, summary, live_stage):
|
|
run_spy = mocker.spy(stage_module.run, "_run")
|
|
live_stage(summary=summary)
|
|
|
|
assert run_spy.call_count == 1
|
|
_, kwargs = run_spy.call_args
|
|
|
|
assert "DVCLIVE_PATH" in kwargs["env"]
|
|
assert kwargs["env"]["DVCLIVE_PATH"] == "logs"
|
|
|
|
assert "DVCLIVE_SUMMARY" in kwargs["env"]
|
|
assert kwargs["env"]["DVCLIVE_SUMMARY"] == str(int(summary))
|
|
|
|
|
|
def test_live_provides_metrics(tmp_dir, dvc, live_stage):
|
|
live_stage(summary=True)
|
|
|
|
assert (tmp_dir / "logs.json").is_file()
|
|
assert dvc.metrics.show() == {
|
|
"": {"logs.json": {"step": 1, "loss": 0.5, "accuracy": 0.5}}
|
|
}
|
|
|
|
assert (tmp_dir / "logs").is_dir()
|
|
plots = dvc.plots.show()
|
|
assert "logs/accuracy.tsv" in plots
|
|
assert "logs/loss.tsv" in plots
|
|
|
|
|
|
def test_live_provides_no_metrics(tmp_dir, dvc, live_stage):
|
|
live_stage(summary=False)
|
|
|
|
assert not (tmp_dir / "logs.json").is_file()
|
|
with pytest.raises(MetricsError):
|
|
assert dvc.metrics.show() == {}
|
|
|
|
assert (tmp_dir / "logs").is_dir()
|
|
plots = dvc.plots.show()
|
|
assert "logs/accuracy.tsv" in plots
|
|
assert "logs/loss.tsv" in plots
|
|
|
|
|
|
def test_experiments_track_summary(tmp_dir, scm, dvc, live_stage):
|
|
live_stage(summary=True)
|
|
baseline_rev = scm.get_rev()
|
|
|
|
experiments = dvc.experiments.run(targets=["live_stage"], params=["foo=2"])
|
|
assert len(experiments) == 1
|
|
((exp_rev, _),) = experiments.items()
|
|
|
|
res = dvc.experiments.show()
|
|
assert "logs.json" in res[baseline_rev][exp_rev]["metrics"].keys()
|
|
|
|
|
|
@pytest.mark.parametrize("report", [True, False])
|
|
def test_live_report(tmp_dir, dvc, live_stage, report):
|
|
live_stage(report=report)
|
|
|
|
assert (tmp_dir / "logs.html").is_file() == report
|