Remove live section support.

This commit is contained in:
David de la Iglesia Castro
2022-11-07 12:45:32 +01:00
parent eb20a8e049
commit 1de983f0de
19 changed files with 13 additions and 419 deletions
-20
View File
@@ -238,26 +238,6 @@ def _add_common_args(parser):
help="Declare output plot file (do not put into DVC cache).",
metavar="<path>",
)
parser.add_argument(
"--live", help="Declare output as dvclive.", metavar="<path>"
)
parser.add_argument(
"--live-no-cache",
help="Declare output as dvclive (do not put into DVC cache).",
metavar="<path>",
)
parser.add_argument(
"--live-no-summary",
action="store_true",
default=False,
help=argparse.SUPPRESS,
)
parser.add_argument(
"--live-no-html",
action="store_true",
default=False,
help="Signal dvclive logger to not produce training report.",
)
parser.add_argument(
"-w",
"--wdir",
-3
View File
@@ -2,9 +2,6 @@ DVC_CHECKPOINT = "DVC_CHECKPOINT"
DVC_DAEMON = "DVC_DAEMON"
DVC_PAGER = "DVC_PAGER"
DVC_ROOT = "DVC_ROOT"
DVCLIVE_PATH = "DVCLIVE_PATH"
DVCLIVE_SUMMARY = "DVCLIVE_SUMMARY"
DVCLIVE_HTML = "DVCLIVE_HTML"
DVCLIVE_RESUME = "DVCLIVE_RESUME"
DVC_IGNORE_ISATTY = "DVC_IGNORE_ISATTY"
DVC_EXP_GIT_REMOTE = "DVC_EXP_GIT_REMOTE"
+3 -24
View File
@@ -87,7 +87,6 @@ def loadd_from(stage, d_list):
plot = d.pop(Output.PARAM_PLOT, False)
persist = d.pop(Output.PARAM_PERSIST, False)
checkpoint = d.pop(Output.PARAM_CHECKPOINT, False)
live = d.pop(Output.PARAM_LIVE, False)
remote = d.pop(Output.PARAM_REMOTE, None)
annot = {field: d.pop(field, None) for field in ANNOTATION_FIELDS}
files = d.pop(Output.PARAM_FILES, None)
@@ -101,7 +100,6 @@ def loadd_from(stage, d_list):
plot=plot,
persist=persist,
checkpoint=checkpoint,
live=live,
remote=remote,
**annot,
files=files,
@@ -118,7 +116,6 @@ def loads_from(
plot=False,
persist=False,
checkpoint=False,
live=False,
remote=None,
):
return [
@@ -131,7 +128,6 @@ def loads_from(
plot=plot,
persist=persist,
checkpoint=checkpoint,
live=live,
remote=remote,
)
for s in s_list
@@ -166,29 +162,21 @@ def load_from_pipeline(stage, data, typ="outs"):
stage.PARAM_OUTS,
stage.PARAM_METRICS,
stage.PARAM_PLOTS,
stage.PARAM_LIVE,
):
raise ValueError(f"'{typ}' key is not allowed for pipeline files.")
metric = typ == stage.PARAM_METRICS
plot = typ == stage.PARAM_PLOTS
live = typ == stage.PARAM_LIVE
if live:
# `live` is single object
data = [data]
d = _merge_data(data)
for path, flags in d.items():
plt_d, live_d = {}, {}
plt_d = {}
if plot:
from dvc.schema import PLOT_PROPS
plt_d, flags = _split_dict(flags, keys=PLOT_PROPS.keys())
if live:
from dvc.schema import LIVE_PROPS
live_d, flags = _split_dict(flags, keys=LIVE_PROPS.keys())
extra = project(
flags,
[
@@ -206,7 +194,6 @@ def load_from_pipeline(stage, data, typ="outs"):
info={},
plot=plt_d or plot,
metric=metric,
live=live_d or live,
**extra,
)
@@ -263,9 +250,6 @@ class Output:
PARAM_PLOT_TITLE = "title"
PARAM_PLOT_HEADER = "header"
PARAM_PERSIST = "persist"
PARAM_LIVE = "live"
PARAM_LIVE_SUMMARY = "summary"
PARAM_LIVE_HTML = "html"
PARAM_REMOTE = "remote"
METRIC_SCHEMA = Any(
@@ -292,7 +276,6 @@ class Output:
plot=False,
persist=False,
checkpoint=False,
live=False,
desc=None,
type=None, # pylint: disable=redefined-builtin
labels=None,
@@ -355,7 +338,6 @@ class Output:
self.plot = False if self.IS_DEPENDENCY else plot
self.persist = persist
self.checkpoint = checkpoint
self.live = live
self.fs_path = self._parse_path(self.fs, fs_path)
self.obj: Optional["HashFile"] = None
@@ -791,9 +773,6 @@ class Output:
if self.checkpoint:
ret[self.PARAM_CHECKPOINT] = self.checkpoint
if self.live:
ret[self.PARAM_LIVE] = self.live
if self.remote:
ret[self.PARAM_REMOTE] = self.remote
@@ -1195,11 +1174,11 @@ class Output:
@property
def is_metric(self) -> bool:
return bool(self.metric) or bool(self.live)
return bool(self.metric)
@property
def is_plot(self) -> bool:
return bool(self.plot) or bool(self.live)
return bool(self.plot)
META_SCHEMA = {
+1 -1
View File
@@ -269,7 +269,7 @@ class Repo:
except SCMError:
if self._uninitialized:
# might not be a git/dvc repo at all
# used in `params/metrics/plots/live` targets
# used in `params/metrics/plots` targets
return SCM(self.root_dir, no_scm=True)
raise
-19
View File
@@ -1,19 +0,0 @@
import logging
from typing import TYPE_CHECKING, Optional
logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from dvc.output import Output
def summary_fs_path(out: "Output") -> Optional[str]:
from dvc.output import Output
assert out.live
has_summary = True
if isinstance(out.live, dict):
has_summary = out.live.get(Output.PARAM_LIVE_SUMMARY, True)
if has_summary:
return out.fs.path.with_suffix(out.fs_path, ".json")
return None
+1 -6
View File
@@ -8,7 +8,6 @@ from dvc.fs.dvc import DVCFileSystem
from dvc.output import Output
from dvc.repo import locked
from dvc.repo.collect import StrPaths, collect
from dvc.repo.live import summary_fs_path
from dvc.scm import NoSCMError
from dvc.utils import error_handler, errored_revisions, onerror_collect
from dvc.utils.collections import ensure_list
@@ -18,7 +17,7 @@ logger = logging.getLogger(__name__)
def _is_metric(out: Output) -> bool:
return bool(out.metric) or bool(out.live)
return bool(out.metric)
def _to_fs_paths(metrics: List[Output]) -> StrPaths:
@@ -26,10 +25,6 @@ def _to_fs_paths(metrics: List[Output]) -> StrPaths:
for out in metrics:
if out.metric:
result.append(out.repo.dvcfs.from_os_path(out.fs_path))
elif out.live:
fs_path = summary_fs_path(out)
if fs_path:
result.append(out.repo.dvcfs.from_os_path(fs_path))
return result
+2 -2
View File
@@ -266,7 +266,7 @@ class Plots:
def _is_plot(out: "Output") -> bool:
return bool(out.plot) or bool(out.live)
return bool(out.plot)
def _resolve_data_sources(plots_data: Dict):
@@ -562,7 +562,7 @@ def parse(fs, path, props=None, **kwargs):
def _plot_props(out: "Output") -> Dict:
from dvc.schema import PLOT_PROPS
if not (out.plot or out.live):
if not (out.plot):
raise NotAPlotError(out)
if isinstance(out.plot, list):
raise DvcException("Multiple plots per data file not supported.")
-6
View File
@@ -72,12 +72,6 @@ def _get_stage_files(stage: "Stage") -> typing.Iterator[str]:
for out in stage.outs:
if not out.use_scm_ignore and out.is_in_repo:
yield out.fs_path
if out.live:
from dvc.repo.live import summary_fs_path
summary = summary_fs_path(out)
if summary:
yield summary
def _track_stage(stage: "Stage") -> None:
-8
View File
@@ -67,13 +67,6 @@ PLOT_PROPS = {
PLOT_PROPS_SCHEMA = {**OUT_PSTAGE_DETAILED_SCHEMA[str], **PLOT_PROPS}
PLOT_PSTAGE_SCHEMA = {str: Any(PLOT_PROPS_SCHEMA, [PLOT_PROPS_SCHEMA])}
LIVE_PROPS = {
Output.PARAM_LIVE_SUMMARY: bool,
Output.PARAM_LIVE_HTML: bool,
}
LIVE_PROPS_SCHEMA = {**PLOT_PROPS_SCHEMA, **LIVE_PROPS}
LIVE_PSTAGE_SCHEMA = {str: LIVE_PROPS_SCHEMA}
PARAM_PSTAGE_NON_DEFAULT_SCHEMA = {str: [str]}
VARS_SCHEMA = [str, dict]
@@ -93,7 +86,6 @@ STAGE_DEFINITION = {
Any(str, OUT_PSTAGE_DETAILED_SCHEMA)
],
Optional(StageParams.PARAM_PLOTS): [Any(str, PLOT_PSTAGE_SCHEMA)],
Optional(StageParams.PARAM_LIVE): Any(str, LIVE_PSTAGE_SCHEMA),
}
+2 -17
View File
@@ -280,22 +280,7 @@ class Stage(params.StageParams):
def _read_env(self, out, checkpoint_func=None) -> Env:
env: Env = {}
if out.live:
from dvc.env import DVCLIVE_HTML, DVCLIVE_PATH, DVCLIVE_SUMMARY
from dvc.output import Output
from dvc.schema import LIVE_PROPS
env[DVCLIVE_PATH] = relpath(out.fs_path, self.wdir)
if isinstance(out.live, dict):
config = project(out.live, LIVE_PROPS)
env[DVCLIVE_SUMMARY] = str(
int(config.get(Output.PARAM_LIVE_SUMMARY, True))
)
env[DVCLIVE_HTML] = str(
int(config.get(Output.PARAM_LIVE_HTML, True))
)
elif out.checkpoint and checkpoint_func:
if out.checkpoint and checkpoint_func:
from dvc.env import DVC_CHECKPOINT
env.update({DVC_CHECKPOINT: "1"})
@@ -379,7 +364,7 @@ class Stage(params.StageParams):
def remove_outs(self, ignore_remove=False, force=False):
"""Used mainly for `dvc remove --outs` and :func:`Stage.reproduce`."""
for out in self.outs:
if (out.persist or out.checkpoint or out.live) and not force:
if (out.persist or out.checkpoint) and not force:
out.unprotect()
continue
-1
View File
@@ -97,7 +97,6 @@ class StageLoader(Mapping):
stage.PARAM_OUTS,
stage.PARAM_METRICS,
stage.PARAM_PLOTS,
stage.PARAM_LIVE,
],
)
stage.outs = lcat(
-4
View File
@@ -20,10 +20,6 @@ class CheckpointKilledError(StageCmdFailedError):
pass
class LiveKilledError(StageCmdFailedError):
pass
@dataclass
class MonitorTask:
stage: "Stage"
-1
View File
@@ -12,4 +12,3 @@ class StageParams:
PARAM_METRICS = "metrics"
PARAM_PLOTS = "plots"
PARAM_DESC = "desc"
PARAM_LIVE = "live"
+3 -10
View File
@@ -52,8 +52,6 @@ def _get_flags(out):
# `out.plot` is in the same order as is in the file when read
# and, should be dumped as-is without any sorting
yield from out.plot.items()
if out.live and isinstance(out.live, dict):
yield from out.live.items()
if out.remote:
yield PARAM_REMOTE, out.remote
@@ -65,19 +63,15 @@ def _serialize_out(out):
@no_type_check
def _serialize_outs(outputs: List[Output]):
outs, metrics, plots, live = [], [], [], None
outs, metrics, plots = [], [], []
for out in sort_by_path(outputs):
bucket = outs
if out.plot:
bucket = plots
elif out.metric:
bucket = metrics
elif out.live:
assert live is None
live = _serialize_out(out)
continue
bucket.append(_serialize_out(out))
return outs, metrics, plots, live
return outs, metrics, plots
def _serialize_params_keys(params):
@@ -126,7 +120,7 @@ def to_pipeline_file(stage: "PipelineStage"):
deps = sorted(d.def_path for d in deps)
params = _serialize_params_keys(params)
outs, metrics, plots, live = _serialize_outs(stage.outs)
outs, metrics, plots = _serialize_outs(stage.outs)
cmd = stage.cmd
assert cmd, (
@@ -142,7 +136,6 @@ def to_pipeline_file(stage: "PipelineStage"):
(stage.PARAM_OUTS, outs),
(stage.PARAM_METRICS, metrics),
(stage.PARAM_PLOTS, plots),
(stage.PARAM_LIVE, live),
(stage.PARAM_FROZEN, stage.frozen),
(stage.PARAM_ALWAYS_CHANGED, stage.always_changed),
(stage.PARAM_META, stage.meta),
+1 -43
View File
@@ -2,7 +2,7 @@ import os
import pathlib
from typing import TYPE_CHECKING, Union
from funcy import concat, first, lsplit, rpartial, without
from funcy import concat, first, lsplit, rpartial
from dvc.annotations import ANNOTATION_FIELDS
from dvc.exceptions import InvalidArgumentError
@@ -66,8 +66,6 @@ def fill_stage_outputs(stage, **kwargs):
stage.outs = []
stage.outs += _load_live_output(stage, **kwargs)
for key in keys:
stage.outs += loads_from(
stage,
@@ -80,34 +78,6 @@ def fill_stage_outputs(stage, **kwargs):
)
def _load_live_output(
stage,
live=None,
live_no_cache=None,
live_summary=False,
live_html=False,
**kwargs,
):
from dvc.output import Output, loads_from
outs = []
if live or live_no_cache:
assert bool(live) != bool(live_no_cache)
path = live or live_no_cache
outs += loads_from(
stage,
[path],
use_cache=not bool(live_no_cache),
live={
Output.PARAM_LIVE_SUMMARY: live_summary,
Output.PARAM_LIVE_HTML: live_html,
},
)
return outs
def fill_stage_dependencies(
stage, deps=None, erepo=None, params=None, fs_config=None
):
@@ -274,7 +244,6 @@ def prepare_file_path(kwargs):
kwargs.get("outs_persist", []),
kwargs.get("outs_persist_no_cache", []),
kwargs.get("checkpoints", []),
without([kwargs.get("live", None)], None),
)
)
@@ -332,15 +301,4 @@ def validate_kwargs(single_stage: bool = False, fname: str = None, **kwargs):
if single_stage:
kwargs.pop("name", None)
if kwargs.get("live") and kwargs.get("live_no_cache"):
raise InvalidArgumentError(
"cannot specify both `--live` and `--live-no-cache`"
)
kwargs.update(
{
"live_summary": not kwargs.pop("live_no_summary", False),
"live_html": not kwargs.pop("live_no_html", False),
}
)
return kwargs
-1
View File
@@ -101,7 +101,6 @@ tests = [
# optional dependencies
"dvclive>=1.0",
"pywin32>=225; sys_platform == 'win32'",
"dvclive[image]>=0.7.3",
"beautifulsoup4==4.11.1",
# pylint requirements
"pylint==2.15.5",
-228
View File
@@ -1,228 +0,0 @@
from copy import deepcopy
from textwrap import dedent
import pytest
from funcy import first
from dvc import stage as stage_module
pytest.importorskip("dvclive", reason="no dvclive")
LIVE_SCRIPT = dedent(
"""
from dvclive import Live
from PIL import Image
import sys
r = 2
metrics_logger = Live()
for i in range(r):
metrics_logger.log("loss", 1-i/r)
metrics_logger.log("accuracy", i/r)
image = Image.new("RGB", (10,10), (i,i,i))
metrics_logger.log_image("image.jpg", image)
metrics_logger.next_step()"""
)
LIVE_CHECKPOINT_SCRIPT = dedent(
"""
import os
from dvclive import Live
def read(path):
value=0
if os.path.exists(path):
with open(path, 'r') as fobj:
try:
value = int(fobj.read())
except ValueError:
pass
return value
def dump(value, path):
with open(path, "w") as fobj:
fobj.write(str(value))
r = 3
checkpoint_file = "checkpoint"
metrics_logger = Live()
value = read(checkpoint_file)
for i in range(1,r):
m = i + value
dump(m, checkpoint_file)
metrics_logger.log("metric1", m)
metrics_logger.log("metric2", m * 2)
metrics_logger.next_step()"""
)
@pytest.fixture
def live_stage(tmp_dir, scm, dvc, mocker):
mocker.patch("dvc.stage.run.Monitor.AWAIT", 0.01)
def make(
html=True,
live=None,
live_no_cache=None,
code=LIVE_SCRIPT,
):
assert bool(live) != bool(live_no_cache)
tmp_dir.gen("train.py", code)
tmp_dir.gen("params.yaml", "foo: 1")
stage = dvc.run(
cmd="python train.py",
params=["foo"],
deps=["train.py"],
name="live_stage",
live=live,
live_no_cache=live_no_cache,
live_no_html=not html,
)
scm.add(["dvc.yaml", "dvc.lock", "train.py", "params.yaml"])
scm.commit("initial: live_stage")
return stage
yield make
@pytest.mark.parametrize("html", (True, False))
def test_export_config(tmp_dir, dvc, mocker, live_stage, html):
run_spy = mocker.spy(stage_module.run, "_run")
live_stage(html=html, live="logs")
assert run_spy.call_count == 1
_, kwargs = run_spy.call_args
assert "DVCLIVE_PATH" in kwargs["env"]
assert kwargs["env"]["DVCLIVE_PATH"] == "logs"
assert "DVCLIVE_HTML" in kwargs["env"]
assert kwargs["env"]["DVCLIVE_HTML"] == str(int(html))
def test_live_provides_metrics(tmp_dir, dvc, live_stage):
live_stage(live="logs")
assert (tmp_dir / "logs.json").is_file()
assert dvc.metrics.show() == {
"": {
"data": {
"logs.json": {
"data": {"accuracy": 0.5, "loss": 0.5, "step": 1}
}
}
}
}
assert (tmp_dir / "logs").is_dir()
plots_data = dvc.plots.show()
files = list(plots_data["workspace"]["sources"]["data"])
assert "logs/scalars/accuracy.tsv" in files
assert "logs/scalars/loss.tsv" in files
assert "logs/images/0/image.jpg" in files
assert "logs/images/1/image.jpg" in files
@pytest.mark.parametrize("typ", ("live", "live_no_cache"))
def test_experiments_track_summary(tmp_dir, scm, dvc, live_stage, typ):
live_stage(**{typ: "logs"})
baseline_rev = scm.get_rev()
experiments = dvc.experiments.run(targets=["live_stage"], params=["foo=2"])
assert len(experiments) == 1
((exp_rev, _),) = experiments.items()
res = dvc.experiments.show()
assert "logs.json" in res[baseline_rev][exp_rev]["data"]["metrics"].keys()
@pytest.mark.parametrize("html", [True, False])
def test_live_html(tmp_dir, dvc, live_stage, html):
live_stage(live="logs", html=html)
assert (tmp_dir / "logs_dvc_plots" / "index.html").is_file() == html
if html:
html_text = (tmp_dir / "logs_dvc_plots" / "index.html").read_text()
assert 'http-equiv="refresh"' in html_text
@pytest.fixture
def live_checkpoint_stage(tmp_dir, scm, dvc, mocker):
mocker.patch("dvc.stage.run.Monitor.AWAIT", 0.01)
def make(live=None, live_no_cache=None):
assert bool(live) != bool(live_no_cache)
tmp_dir.gen("train.py", LIVE_CHECKPOINT_SCRIPT)
tmp_dir.gen("params.yaml", "foo: 1")
stage = dvc.run(
cmd="python train.py",
params=["foo"],
deps=["train.py"],
name="live_stage",
live=live,
live_no_cache=live_no_cache,
checkpoints=["checkpoint"],
no_exec=True,
)
scm.add(["dvc.yaml", "train.py", "params.yaml", ".gitignore"])
scm.commit("initial: live_stage")
return stage
yield make
def checkpoints_metric(show_results, metric_file, metric_name):
tmp = deepcopy(show_results)
tmp.pop("workspace")
tmp = first(tmp.values())
tmp.pop("baseline")
return [
exp["data"]["metrics"][metric_file]["data"][metric_name]
for exp in tmp.values()
]
@pytest.mark.parametrize("typ", ("live", "live_no_cache"))
def test_live_checkpoints_resume(
tmp_dir, scm, dvc, live_checkpoint_stage, typ
):
stage = live_checkpoint_stage(**{typ: "logs"})
results = dvc.experiments.run(
stage.addressing, params=["foo=2"], tmp_dir=False
)
checkpoint_resume = first(results)
dvc.experiments.run(
stage.addressing, checkpoint_resume=checkpoint_resume, tmp_dir=False
)
results = dvc.experiments.show()
assert checkpoints_metric(results, "logs.json", "step") == [3, 2, 1, 0]
assert checkpoints_metric(results, "logs.json", "metric1") == [4, 3, 2, 1]
assert checkpoints_metric(results, "logs.json", "metric2") == [8, 6, 4, 2]
def test_dvclive_stage_with_different_wdir(tmp_dir, scm, dvc):
(tmp_dir / "subdir").gen("train.py", LIVE_SCRIPT)
(tmp_dir / "subdir" / "params.yaml").dump({"foo": 1})
dvc.stage.add(
cmd="python train.py",
params=["foo"],
deps=["train.py"],
name="live_stage",
live="dvclive",
live_no_html=True,
wdir="subdir",
)
dvc.reproduce()
assert (tmp_dir / "subdir" / "dvclive").is_dir()
assert (tmp_dir / "subdir" / "dvclive.json").is_file()
-18
View File
@@ -23,12 +23,6 @@ def test_run(mocker, dvc):
"plots",
"--plots-no-cache",
"plots-no-cache",
"--live",
"live",
"--live-no-cache",
"live-no-cache",
"--live-no-summary",
"--live-no-html",
"--file",
"file",
"--wdir",
@@ -74,10 +68,6 @@ def test_run(mocker, dvc):
outs_persist_no_cache=["outs-persist-no-cache"],
checkpoints=["checkpoints"],
params=["file:param1,param2", "param3"],
live="live",
live_no_cache="live-no-cache",
live_no_summary=True,
live_no_html=True,
fname="file",
wdir="wdir",
no_exec=True,
@@ -107,10 +97,6 @@ def test_run_args_from_cli(mocker, dvc):
metrics_no_cache=[],
plots=[],
plots_no_cache=[],
live=None,
live_no_cache=None,
live_no_summary=False,
live_no_html=False,
outs_persist=[],
outs_persist_no_cache=[],
checkpoints=[],
@@ -144,10 +130,6 @@ def test_run_args_with_spaces(mocker, dvc):
metrics_no_cache=[],
plots=[],
plots_no_cache=[],
live=None,
live_no_cache=None,
live_no_summary=False,
live_no_html=False,
outs_persist=[],
outs_persist_no_cache=[],
checkpoints=[],
-7
View File
@@ -35,10 +35,6 @@ def test_stage_add(mocker, dvc, command, parsed_command):
"plots",
"--plots-no-cache",
"plots-no-cache",
"--live",
"live",
"--live-no-summary",
"--live-no-html",
"--wdir",
"wdir",
"--force",
@@ -80,9 +76,6 @@ def test_stage_add(mocker, dvc, command, parsed_command):
metrics_no_cache=["metrics-no-cache"],
plots=["plots"],
plots_no_cache=["plots-no-cache"],
live="live",
live_no_summary=True,
live_no_html=True,
wdir="wdir",
outs_persist=["outs-persist"],
outs_persist_no_cache=["outs-persist-no-cache"],