exp run: Track top level files.

This commit is contained in:
David de la Iglesia Castro
2023-03-31 19:23:40 +02:00
parent d2812fa48f
commit 7c8e6e3156
2 changed files with 37 additions and 1 deletions
+16 -1
View File
@@ -3,6 +3,8 @@ from functools import partial
from typing import TYPE_CHECKING, Iterator, List
from dvc.exceptions import DvcException, ReproductionError
from dvc.repo.metrics.show import _collect_top_level_metrics
from dvc.repo.params.show import _collect_top_level_params
from dvc.repo.scm_context import scm_context
from dvc.stage.exceptions import CheckpointKilledError
@@ -74,6 +76,17 @@ def _track_stage(stage: "Stage") -> None:
return context.track_changed_files()
def _track_top_level(repo: "Repo") -> None:
context = repo.scm_context
for metric_file in _collect_top_level_metrics(repo):
context.track_file(metric_file)
for param_file in _collect_top_level_params(repo):
context.track_file(param_file)
for plot_file in repo.index._plot_sources: # pylint: disable=protected-access
context.track_file(plot_file)
context.track_changed_files()
@locked
@scm_context
def reproduce( # noqa: C901
@@ -125,7 +138,9 @@ def reproduce( # noqa: C901
)
)
return _reproduce_stages(self.index.graph, list(stages), **kwargs)
result = _reproduce_stages(self.index.graph, list(stages), **kwargs)
_track_top_level(self)
return result
def _reproduce_stages( # noqa: C901
@@ -683,3 +683,24 @@ def test_local_config_is_propagated_to_tmp(tmp_dir, scm, dvc):
with fs.open("file") as fobj:
conf_obj = ConfigObj(fobj)
assert conf_obj["cache"]["type"] == "hardlink"
@pytest.mark.parametrize("tmp", [True, False])
def test_untracked_top_level_files_are_included_in_exp(tmp_dir, scm, dvc, tmp):
(tmp_dir / "dvc.yaml").dump(
{
"metrics": ["metrics.json"],
"params": ["params.yaml"],
"plots": ["plots.csv"],
}
)
stage = dvc.stage.add(
cmd="touch metrics.json && touch params.yaml && touch plots.csv",
name="top-level",
)
scm.add_commit(["dvc.yaml"], message="add dvc.yaml")
results = dvc.experiments.run(stage.addressing, tmp_dir=tmp)
exp = first(results)
fs = scm.get_fs(exp)
for file in ["metrics.json", "params.yaml", "plots.csv"]:
assert fs.exists(file)