mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-07 15:28:28 +01:00
Paweł RedzyńskiandRuslan Kuprieiev
862ab8de62
* plots: allow dir plots * first scenario * separate dvclive tests * rename index to step * another test * refactor * its ok to have step in data * sort plots * introduce --logs * add logs to run commands * variable name refactoring * fix logs option * fixup * cmd: run: fix unit tests * plots: modify: fix tests * improve dir handling * fixup * dvclive: logs summary * api: introduce logs summarization method * handle no repo case * fix granular dir plots * api: dvclive: simplify * dvclive: roll back logs flag * cmd: introduce logs * api: adjust to logs * test: dvclive fix * rename logs to dvclive * live: command: add uninitialized * outs: dvclive type * dvclive: support providing config via evn variables * dvclive: treat as metrics * dvclive integration: add support for summary on/off * dvc integration: add temporary test for configuration export * remove requirements * cmd: introduce logs * rename logs to dvclive * cmd: introduce logs * remove logs leftowvers * dvclive: env: adjust to chekpoints env variables * command: live: remove unnecessary help * fix linter suggestions * type hinting * review refactor * replace dvclive with live * stage: loading: move live loading to separate method * stage: serialize: allow only single live output * live: add show/diff * stage: serialize: roll back single live enforcing * fixup * rebase fixups * output: live: force single instance * live: command: add unit tests * Update dvc/output/base.py Co-authored-by: Ruslan Kuprieiev <[email protected]> * Update dvc/stage/__init__.py Co-authored-by: Ruslan Kuprieiev <[email protected]> * Update dvc/stage/serialize.py Co-authored-by: Ruslan Kuprieiev <[email protected]> * review v2 * live: fix summary tracking by experiments * naming fixups * api: live: add tests * tests: api: separate live tests Co-authored-by: Ruslan Kuprieiev <[email protected]>
118 lines
3.1 KiB
Python
118 lines
3.1 KiB
Python
from textwrap import dedent
|
|
|
|
import pytest
|
|
|
|
from dvc import stage as stage_module
|
|
from dvc.exceptions import MetricsError
|
|
|
|
LIVE_SCRITP = dedent(
|
|
"""
|
|
from dvclive import dvclive
|
|
import sys
|
|
r = 2
|
|
for i in range(r):
|
|
dvclive.log("loss", 1-i/r)
|
|
dvclive.log("accuracy", i/r)
|
|
dvclive.next_step()"""
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def live_stage(tmp_dir, scm, dvc):
|
|
|
|
pytest.skip("dvclive does not exist yet")
|
|
|
|
def make(summary=True):
|
|
tmp_dir.gen("train.py", LIVE_SCRITP)
|
|
tmp_dir.gen("params.yaml", "foo: 1")
|
|
stage = dvc.run(
|
|
cmd="python train.py",
|
|
params=["foo"],
|
|
deps=["train.py"],
|
|
name="live_stage",
|
|
live="logs",
|
|
live_summary=summary,
|
|
)
|
|
|
|
scm.add(["dvc.yaml", "dvc.lock", "train.py", "params.yaml"])
|
|
scm.commit("initial: live_stage")
|
|
return stage
|
|
|
|
yield make
|
|
|
|
|
|
@pytest.mark.parametrize("summary", (True, False))
|
|
def test_export_config_tmp(tmp_dir, dvc, mocker, summary):
|
|
run_spy = mocker.spy(stage_module.run, "_run")
|
|
tmp_dir.gen("src", "dependency")
|
|
dvc.run(
|
|
cmd="mkdir logs && touch logs.json",
|
|
deps=["src"],
|
|
name="run_logger",
|
|
live="logs",
|
|
live_summary=summary,
|
|
)
|
|
|
|
assert run_spy.call_count == 1
|
|
_, kwargs = run_spy.call_args
|
|
|
|
assert "DVCLIVE_PATH" in kwargs["env"]
|
|
assert kwargs["env"]["DVCLIVE_PATH"] == "logs"
|
|
|
|
assert "DVCLIVE_SUMMARY" in kwargs["env"]
|
|
assert kwargs["env"]["DVCLIVE_SUMMARY"] == str(int(summary))
|
|
|
|
|
|
@pytest.mark.parametrize("summary", (True, False))
|
|
def test_export_config(tmp_dir, dvc, mocker, summary, live_stage):
|
|
run_spy = mocker.spy(stage_module.run, "_run")
|
|
live_stage(summary=summary)
|
|
|
|
assert run_spy.call_count == 1
|
|
_, kwargs = run_spy.call_args
|
|
|
|
assert "DVCLIVE_PATH" in kwargs["env"]
|
|
assert kwargs["env"]["DVCLIVE_PATH"] == "logs"
|
|
|
|
assert "DVCLIVE_SUMMARY" in kwargs["env"]
|
|
assert kwargs["env"]["DVCLIVE_SUMMARY"] == str(int(summary))
|
|
|
|
|
|
def test_live_provides_metrics(tmp_dir, dvc, live_stage):
|
|
live_stage(summary=True)
|
|
|
|
assert (tmp_dir / "logs.json").is_file()
|
|
assert dvc.metrics.show() == {
|
|
"": {"logs.json": {"step": 1, "loss": 0.5, "accuracy": 0.5}}
|
|
}
|
|
|
|
assert (tmp_dir / "logs").is_dir()
|
|
plots = dvc.plots.show()
|
|
assert "logs/accuracy.tsv" in plots
|
|
assert "logs/loss.tsv" in plots
|
|
|
|
|
|
def test_live_provides_no_metrics(tmp_dir, dvc, live_stage):
|
|
live_stage(summary=False)
|
|
|
|
assert not (tmp_dir / "logs.json").is_file()
|
|
with pytest.raises(MetricsError):
|
|
assert dvc.metrics.show() == {}
|
|
|
|
assert (tmp_dir / "logs").is_dir()
|
|
plots = dvc.plots.show()
|
|
assert "logs/accuracy.tsv" in plots
|
|
assert "logs/loss.tsv" in plots
|
|
|
|
|
|
def test_experiments_track_summary(tmp_dir, scm, dvc, live_stage):
|
|
live_stage(summary=True)
|
|
baseline_rev = scm.get_rev()
|
|
|
|
experiments = dvc.experiments.run(targets=["live_stage"], params=["foo=2"])
|
|
assert len(experiments) == 1
|
|
((exp_rev, _),) = experiments.items()
|
|
|
|
res = dvc.experiments.show()
|
|
assert "logs.json" in res[baseline_rev][exp_rev]["metrics"].keys()
|