tests: remove single_stage stage runs (#9549)

This commit is contained in:
skshetry
2023-06-06 18:38:07 +05:45
committed by GitHub
parent 2c4365513e
commit 3759535d43
23 changed files with 353 additions and 263 deletions
+3
View File
@@ -6,6 +6,7 @@ from contextlib import suppress
import pytest
from dvc import env
from dvc.stage import PipelineStage
from dvc.testing.fixtures import * # noqa, pylint: disable=wildcard-import
from .dir_helpers import * # noqa, pylint: disable=wildcard-import
@@ -241,6 +242,8 @@ def run_copy_metrics(tmp_dir, copy_script):
if hasattr(tmp_dir.dvc, "scm"):
files = [stage.path]
if isinstance(stage, PipelineStage):
files += [stage.dvcfile._lockfile.path]
files += [out.fs_path for out in stage.outs if not out.use_cache]
tmp_dir.dvc.scm.add(files)
if commit:
+1 -1
View File
@@ -91,7 +91,7 @@ def test_open_not_cached(dvc):
metric_content = "0.6"
metric_code = f"open('{metric_file}', 'w').write('{metric_content}')"
dvc.run(
single_stage=True,
name="write-metric",
metrics_no_cache=[metric_file],
cmd=f'python -c "{metric_code}"',
)
+5 -1
View File
@@ -584,7 +584,11 @@ def test_modified_data_dep(tmp_dir, scm, dvc, workspace, params, target, copy_sc
def test_exp_run_recursive(tmp_dir, scm, dvc, run_copy_metrics):
tmp_dir.dvc_gen("metric_t.json", '{"foo": 1}')
run_copy_metrics(
"metric_t.json", "metric.json", metrics=["metric.json"], no_exec=True
"metric_t.json",
"metric.json",
metrics=["metric.json"],
no_exec=True,
name="copy-metric",
)
assert dvc.experiments.run(".", recursive=True)
assert (tmp_dir / "metric.json").parse() == {"foo": 1}
+43 -8
View File
@@ -10,7 +10,9 @@ from dvc.utils import relpath
def test_metrics_diff_simple(tmp_dir, scm, dvc, run_copy_metrics):
def _gen(val):
tmp_dir.gen({"m_temp.yaml": str(val)})
run_copy_metrics("m_temp.yaml", "m.yaml", metrics=["m.yaml"])
run_copy_metrics(
"m_temp.yaml", "m.yaml", name="copy-metrics", metrics=["m.yaml"]
)
dvc.scm.commit(str(val))
_gen(1)
@@ -26,7 +28,13 @@ def test_metrics_diff_yaml(tmp_dir, scm, dvc, run_copy_metrics):
def _gen(val):
metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
(tmp_dir / "m_temp.yaml").dump(metrics)
run_copy_metrics("m_temp.yaml", "m.yaml", metrics=["m.yaml"], commit=str(val))
run_copy_metrics(
"m_temp.yaml",
"m.yaml",
name="copy-metrics",
metrics=["m.yaml"],
commit=str(val),
)
_gen(1)
_gen(2)
@@ -46,7 +54,13 @@ def test_metrics_diff_json(tmp_dir, scm, dvc, run_copy_metrics):
def _gen(val):
metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
(tmp_dir / "m_temp.json").dump(metrics)
run_copy_metrics("m_temp.json", "m.json", metrics=["m.json"], commit=str(val))
run_copy_metrics(
"m_temp.json",
"m.json",
name="copy-metrics",
metrics=["m.json"],
commit=str(val),
)
_gen(1)
_gen(2)
@@ -65,7 +79,13 @@ def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc, run_copy_metrics):
def _gen(val):
metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
(tmp_dir / "m_temp.json").dump(metrics)
run_copy_metrics("m_temp.json", "m.json", metrics=["m.json"], commit=str(val))
run_copy_metrics(
"m_temp.json",
"m.json",
name="copy-metrics",
metrics=["m.json"],
commit=str(val),
)
_gen(1)
_gen(2)
@@ -80,6 +100,7 @@ def test_metrics_diff_broken_json(tmp_dir, scm, dvc, run_copy_metrics):
run_copy_metrics(
"m_temp.json",
"m.json",
name="copy-metrics",
metrics_no_cache=["m.json"],
commit="add metrics",
)
@@ -103,7 +124,9 @@ def test_metrics_diff_no_metrics(tmp_dir, scm, dvc):
def test_metrics_diff_new_metric(tmp_dir, scm, dvc, run_copy_metrics):
metrics = {"a": {"b": {"c": 1, "d": 1, "e": "3"}}}
(tmp_dir / "m_temp.json").dump(metrics)
run_copy_metrics("m_temp.json", "m.json", metrics_no_cache=["m.json"])
run_copy_metrics(
"m_temp.json", "m.json", name="copy-metrics", metrics_no_cache=["m.json"]
)
assert dvc.metrics.diff() == {
"m.json": {
@@ -120,6 +143,7 @@ def test_metrics_diff_deleted_metric(tmp_dir, scm, dvc, run_copy_metrics):
run_copy_metrics(
"m_temp.json",
"m.json",
name="copy-metrics",
metrics_no_cache=["m.json"],
commit="add metrics",
)
@@ -140,6 +164,7 @@ def test_metrics_diff_with_unchanged(tmp_dir, scm, dvc, run_copy_metrics):
run_copy_metrics(
"metrics_temp.yaml",
"metrics.yaml",
name="copy-metrics",
metrics_no_cache=["metrics.yaml"],
commit="1",
)
@@ -168,7 +193,9 @@ def test_no_commits(tmp_dir):
def test_metrics_diff_dirty(tmp_dir, scm, dvc, run_copy_metrics):
def _gen(val):
tmp_dir.gen({"m_temp.yaml": str(val)})
run_copy_metrics("m_temp.yaml", "m.yaml", metrics=["m.yaml"])
run_copy_metrics(
"m_temp.yaml", "m.yaml", name="copy-metrics", metrics=["m.yaml"]
)
dvc.scm.commit(str(val))
_gen(1)
@@ -185,7 +212,9 @@ def test_metrics_diff_dirty(tmp_dir, scm, dvc, run_copy_metrics):
def test_metrics_diff_cli(tmp_dir, scm, dvc, run_copy_metrics, caplog, capsys):
def _gen(val):
tmp_dir.gen({"m_temp.yaml": f"foo: {val}"})
run_copy_metrics("m_temp.yaml", "m.yaml", metrics=["m.yaml"])
run_copy_metrics(
"m_temp.yaml", "m.yaml", name="copy-metrics", metrics=["m.yaml"]
)
dvc.scm.commit(str(val))
_gen(1.23456789)
@@ -243,7 +272,13 @@ def test_metrics_diff_active_branch_unchanged(tmp_dir, scm, dvc, run_copy_metric
def _gen(val):
metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}}
(tmp_dir / "m_temp.yaml").dump(metrics)
run_copy_metrics("m_temp.yaml", "m.yaml", metrics=["m.yaml"], commit=str(val))
run_copy_metrics(
"m_temp.yaml",
"m.yaml",
name="copy-metric",
metrics=["m.yaml"],
commit=str(val),
)
_gen(1)
_gen(2)
+31 -13
View File
@@ -12,7 +12,9 @@ from dvc.utils.serialize import JSONFileCorruptedError, YAMLFileCorruptedError
def test_show_simple(tmp_dir, dvc, run_copy_metrics):
tmp_dir.gen("metrics_t.yaml", "1.1")
run_copy_metrics("metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"])
run_copy_metrics(
"metrics_t.yaml", "metrics.yaml", name="copy-metrics", metrics=["metrics.yaml"]
)
assert dvc.metrics.show() == {"": {"data": {"metrics.yaml": {"data": 1.1}}}}
@@ -23,6 +25,7 @@ def test_show_simple_from_subdir(tmp_dir, dvc, run_copy_metrics):
run_copy_metrics(
"metrics_t.yaml",
"subdir/metrics.yaml",
name="copy-metrics",
metrics=["subdir/metrics.yaml"],
)
@@ -40,7 +43,9 @@ def test_show_simple_from_subdir(tmp_dir, dvc, run_copy_metrics):
def test_show(tmp_dir, dvc, run_copy_metrics):
tmp_dir.gen("metrics_t.yaml", "foo: 1.1")
run_copy_metrics("metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"])
run_copy_metrics(
"metrics_t.yaml", "metrics.yaml", name="copy-metrics", metrics=["metrics.yaml"]
)
assert dvc.metrics.show() == {
"": {"data": {"metrics.yaml": {"data": {"foo": 1.1}}}}
}
@@ -48,7 +53,9 @@ def test_show(tmp_dir, dvc, run_copy_metrics):
def test_show_toml(tmp_dir, dvc, run_copy_metrics):
tmp_dir.gen("metrics_t.toml", "[foo]\nbar = 1.2")
run_copy_metrics("metrics_t.toml", "metrics.toml", metrics=["metrics.toml"])
run_copy_metrics(
"metrics_t.toml", "metrics.toml", name="copy-metrics", metrics=["metrics.toml"]
)
assert dvc.metrics.show() == {
"": {"data": {"metrics.toml": {"data": {"foo": {"bar": 1.2}}}}}
}
@@ -56,7 +63,9 @@ def test_show_toml(tmp_dir, dvc, run_copy_metrics):
def test_show_targets(tmp_dir, dvc, run_copy_metrics):
tmp_dir.gen("metrics_t.yaml", "foo: 1.1")
run_copy_metrics("metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"])
run_copy_metrics(
"metrics_t.yaml", "metrics.yaml", name="copy-metrics", metrics=["metrics.yaml"]
)
expected = {"": {"data": {"metrics.yaml": {"data": {"foo": 1.1}}}}}
assert dvc.metrics.show(targets=["metrics.yaml"]) == expected
assert dvc.metrics.show(targets=(tmp_dir / "metrics.yaml").fs_path) == expected
@@ -65,8 +74,8 @@ def test_show_targets(tmp_dir, dvc, run_copy_metrics):
def test_show_multiple(tmp_dir, dvc, run_copy_metrics):
tmp_dir.gen("foo_temp", "foo: 1\n")
tmp_dir.gen("baz_temp", "baz: 2\n")
run_copy_metrics("foo_temp", "foo", fname="foo.dvc", metrics=["foo"])
run_copy_metrics("baz_temp", "baz", fname="baz.dvc", metrics=["baz"])
run_copy_metrics("foo_temp", "foo", name="copy-to-foo", metrics=["foo"])
run_copy_metrics("baz_temp", "baz", name="copy-to-baz", metrics=["baz"])
assert dvc.metrics.show() == {
"": {"data": {"foo": {"data": {"foo": 1}}, "baz": {"data": {"baz": 2}}}}
}
@@ -75,7 +84,10 @@ def test_show_multiple(tmp_dir, dvc, run_copy_metrics):
def test_show_branch(tmp_dir, scm, dvc, run_copy_metrics):
tmp_dir.gen("metrics_temp.yaml", "foo: 1")
run_copy_metrics(
"metrics_temp.yaml", "metrics.yaml", metrics_no_cache=["metrics.yaml"]
"metrics_temp.yaml",
"metrics.yaml",
name="copy-metrics",
metrics_no_cache=["metrics.yaml"],
)
scm.add(["metrics.yaml", "metrics.yaml.dvc"])
scm.commit("init")
@@ -104,13 +116,14 @@ def test_show_subrepo_with_preexisting_tags(tmp_dir, scm):
dvc.run(
cmd="echo foo: 1 > metrics.yaml",
metrics=["metrics.yaml"],
single_stage=True,
name="generate-metrics",
)
scm.add(
[
str(subrepo_dir / "metrics.yaml"),
str(subrepo_dir / "metrics.yaml.dvc"),
str(subrepo_dir / "dvc.yaml"),
str(subrepo_dir / "dvc.lock"),
]
)
scm.commit("init metrics")
@@ -125,11 +138,16 @@ def test_show_subrepo_with_preexisting_tags(tmp_dir, scm):
def test_missing_cache(tmp_dir, dvc, run_copy_metrics):
tmp_dir.gen("metrics_t.yaml", "1.1")
run_copy_metrics("metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"])
run_copy_metrics(
"metrics_t.yaml", "metrics.yaml", name="copy-metrics", metrics=["metrics.yaml"]
)
# This one should be skipped
stage = run_copy_metrics(
"metrics_t.yaml", "metrics2.yaml", metrics=["metrics2.yaml"]
"metrics_t.yaml",
"metrics2.yaml",
name="copy-metrics2",
metrics=["metrics2.yaml"],
)
remove(stage.outs[0].fspath)
remove(stage.outs[0].cache_path)
@@ -183,11 +201,11 @@ def test_show_non_metric_branch(tmp_dir, scm, use_dvc):
assert not (tmp_dir / ".dvc").exists()
def test_non_metric_and_recurisve_show(tmp_dir, dvc, run_copy_metrics):
def test_non_metric_and_recursive_show(tmp_dir, dvc, run_copy_metrics):
tmp_dir.gen({"metrics_t.yaml": "foo: 1.1", "metrics": {"metric1.yaml": "bar: 1.2"}})
metric2 = os.fspath(tmp_dir / "metrics" / "metric2.yaml")
run_copy_metrics("metrics_t.yaml", metric2, metrics=[metric2])
run_copy_metrics("metrics_t.yaml", metric2, name="copy-metric2", metrics=[metric2])
assert dvc.metrics.show(targets=["metrics_t.yaml", "metrics"], recursive=True) == {
"": {
+7 -2
View File
@@ -32,6 +32,7 @@ def test_plot_cache_missing(tmp_dir, scm, dvc, caplog, run_copy_metrics):
"metric_t.json",
"metric.json",
plots=["metric.json"],
name="copy-metric",
commit="there is metric",
)
scm.tag("v1")
@@ -43,6 +44,7 @@ def test_plot_cache_missing(tmp_dir, scm, dvc, caplog, run_copy_metrics):
"metric_t.json",
"metric.json",
plots=["metric.json"],
name="copy-metric",
commit="there is an another metric",
)
scm.tag("v2")
@@ -64,6 +66,7 @@ def test_plot_wrong_metric_type(tmp_dir, scm, dvc, run_copy_metrics):
"metric_t.txt",
"metric.txt",
plots_no_cache=["metric.txt"],
name="copy-metric",
commit="add text metric",
)
@@ -210,7 +213,9 @@ def test_ignore_parsing_error(tmp_dir, dvc, run_copy_metrics):
with open("file", "wb", encoding=None) as fobj:
fobj.write(b"\xc1")
run_copy_metrics("file", "plot_file.json", plots=["plot_file.json"])
run_copy_metrics(
"file", "plot_file.json", plots=["plot_file.json"], name="copy-metric"
)
result = dvc.plots.show(onerror=onerror_collect)
assert isinstance(
@@ -304,7 +309,6 @@ def test_collect_non_existing_dir(tmp_dir, dvc, run_copy_metrics):
f"python copy.py {sname} {p2}"
),
deps=[sname],
single_stage=False,
plots=["subdir"],
name="copy_double",
)
@@ -313,6 +317,7 @@ def test_collect_non_existing_dir(tmp_dir, dvc, run_copy_metrics):
pname,
"plot.json",
plots=["plot.json"],
name="copy-metric",
commit="there is metric",
)
+15 -26
View File
@@ -9,12 +9,10 @@ import pytest
from dvc.cli import main
from dvc.dvcfile import PROJECT_FILE, load_file
from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit, NoOutputOrStageError
from dvc.fs import LocalFileSystem, system
from dvc.stage import Stage
from dvc.fs import system
from dvc.stage.exceptions import StageFileDoesNotExistError
from dvc.utils import relpath
from dvc.utils.fs import remove
from dvc.utils.serialize import dump_yaml, load_yaml
from tests.utils import get_gitignore_content
logger = logging.getLogger("dvc")
@@ -29,11 +27,10 @@ def walk_files(directory):
def test_checkout(tmp_dir, dvc, copy_script):
tmp_dir.dvc_gen({"foo": "foo", "data": {"file": "file"}})
dvc.run(
fname="file1.dvc",
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
single_stage=True,
name="copy-foo-file1",
)
remove(tmp_dir / "foo")
remove("data")
@@ -46,11 +43,10 @@ def test_checkout(tmp_dir, dvc, copy_script):
def test_checkout_cli(tmp_dir, dvc, copy_script):
tmp_dir.dvc_gen({"foo": "foo", "data": {"file": "file"}})
dvc.run(
fname="file1.dvc",
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
single_stage=True,
name="copy-foo-file1",
)
remove(tmp_dir / "foo")
remove("data")
@@ -179,10 +175,10 @@ def test_gitignore_basic(tmp_dir, dvc, scm):
tmp_dir.dvc_gen("file1", "random text1", commit="add file1")
tmp_dir.dvc_gen("file2", "random text2", commit="add file2")
dvc.run(
single_stage=True,
cmd="cp foo file3",
deps=["foo"],
outs_no_cache=["file3"],
name="cp-foo-file3",
)
assert get_gitignore_content() == ["/file1", "/file2"]
@@ -211,19 +207,14 @@ def test_gitignore_when_checkout(tmp_dir, dvc, scm):
assert "/file_in_a_branch" in ignored
def test_checkout_missing_md5_in_stage_file(tmp_dir, dvc, copy_script):
def test_checkout_missing_md5_in_lock_file_for_outs_deps(tmp_dir, dvc, copy_script):
tmp_dir.dvc_gen({"foo": "foo", "data": {"file": "file"}})
stage = dvc.run(
fname="file1.dvc",
dvc.stage.add(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
single_stage=True,
name="copy-file",
)
d = load_yaml(stage.relpath)
del d[Stage.PARAM_OUTS][0][LocalFileSystem.PARAM_CHECKSUM]
del d[Stage.PARAM_DEPS][0][LocalFileSystem.PARAM_CHECKSUM]
dump_yaml(stage.relpath, d)
with pytest.raises(CheckoutError):
dvc.checkout(force=True)
@@ -249,7 +240,7 @@ def test_checkout_not_cached_file(tmp_dir, dvc):
cmd="cp foo bar",
deps=["foo"],
outs_no_cache=["bar"],
single_stage=True,
name="copy-file",
)
stats = dvc.checkout(force=True)
assert not any(stats.values())
@@ -258,11 +249,10 @@ def test_checkout_not_cached_file(tmp_dir, dvc):
def test_checkout_with_deps_cli(tmp_dir, dvc, copy_script):
tmp_dir.dvc_gen({"foo": "foo", "data": {"file": "file"}})
dvc.run(
fname="file1.dvc",
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
single_stage=True,
name="copy-file",
)
remove("foo")
remove("file1")
@@ -270,7 +260,7 @@ def test_checkout_with_deps_cli(tmp_dir, dvc, copy_script):
assert not os.path.exists("foo")
assert not os.path.exists("file1")
ret = main(["checkout", "--force", "file1.dvc", "--with-deps"])
ret = main(["checkout", "--force", "copy-file", "--with-deps"])
assert ret == 0
assert os.path.exists("foo")
@@ -364,10 +354,10 @@ def test_checkout_moved_cache_dir_with_symlinks(tmp_dir, dvc):
def test_checkout_no_checksum(tmp_dir, dvc):
tmp_dir.gen("file", "file content")
stage = dvc.run(outs=["file"], no_exec=True, cmd="somecmd", single_stage=True)
dvc.run(outs=["file"], no_exec=True, cmd="somecmd", name="stage1")
with pytest.raises(CheckoutError):
dvc.checkout([stage.path], force=True)
dvc.checkout(["stage1"], force=True)
assert not os.path.exists("file")
@@ -586,21 +576,20 @@ def test_checkout_with_relink_existing(tmp_dir, dvc, link):
def test_checkout_with_deps(tmp_dir, dvc):
tmp_dir.dvc_gen({"foo": "foo"})
dvc.run(
fname="copy_file.dvc",
cmd="echo foo > bar",
outs=["bar"],
deps=["foo"],
single_stage=True,
name="copy-file",
)
(tmp_dir / "bar").unlink()
(tmp_dir / "foo").unlink()
stats = dvc.checkout(["copy_file.dvc"], with_deps=False)
stats = dvc.checkout(["copy-file"], with_deps=False)
assert stats == {**empty_checkout, "added": ["bar"]}
(tmp_dir / "bar").unlink()
stats = dvc.checkout(["copy_file.dvc"], with_deps=True)
stats = dvc.checkout(["copy-file"], with_deps=True)
assert set(stats["added"]) == {"foo", "bar"}
+2 -3
View File
@@ -84,14 +84,13 @@ def test_commit_preserve_fields(tmp_dir, dvc):
)
@pytest.mark.parametrize("run_kw", [{"single_stage": True}, {"name": "copy"}])
def test_commit_with_deps(tmp_dir, dvc, run_copy, run_kw):
def test_commit_with_deps(tmp_dir, dvc, run_copy):
tmp_dir.gen("foo", "foo")
(foo_stage,) = dvc.add("foo", no_commit=True)
assert foo_stage is not None
assert len(foo_stage.outs) == 1
stage = run_copy("foo", "file", no_commit=True, **run_kw)
stage = run_copy("foo", "file", no_commit=True, name="copy")
assert stage is not None
assert len(stage.outs) == 1
+9 -11
View File
@@ -14,6 +14,7 @@ from dvc.testing.remote_tests import TestRemote # noqa, pylint: disable=unused-
from dvc.utils.fs import remove
from dvc_data.hashfile.db import HashFileDB
from dvc_data.hashfile.db.local import LocalHashFileDB
from dvc_data.hashfile.hash_info import HashInfo
def test_cloud_cli(tmp_dir, dvc, remote, mocker): # noqa: PLR0915
@@ -118,19 +119,17 @@ def test_data_cloud_error_cli(dvc):
def test_warn_on_outdated_stage(tmp_dir, dvc, local_remote, caplog):
stage = dvc.run(outs=["bar"], cmd="echo bar > bar", single_stage=True)
assert main(["push"]) == 0
stage = dvc.run(outs=["bar"], cmd="echo bar > bar", name="gen-bar")
dvc.push()
stage_file_path = stage.relpath
content = (tmp_dir / stage_file_path).parse()
del content["outs"][0]["md5"]
(tmp_dir / stage_file_path).dump(content)
stage.outs[0].hash_info = HashInfo()
stage.dump()
with caplog.at_level(logging.WARNING, logger="dvc"):
caplog.clear()
assert main(["status", "-c"]) == 0
expected_warning = (
"Output 'bar'(stage: 'bar.dvc') is missing version info. "
"Output 'bar'(stage: 'gen-bar') is missing version info. "
"Cache for it will not be collected. "
"Use `dvc repro` to get your pipeline up to date."
)
@@ -311,7 +310,7 @@ def recurse_list_dir(d):
def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
(stage0,) = tmp_dir.dvc_gen("foo", "foo")
stage1 = run_copy("foo", "bar", single_stage=True)
stage1 = run_copy("foo", "bar", name="copy-foo-bar")
stage2 = run_copy("bar", "foobar", name="copy-bar-foobar")
dvc.push()
@@ -343,7 +342,6 @@ def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, local_remote):
path = local_remote.url
tmp_dir.dvc_gen("foo", "foo")
run_copy("foo", "bar", single_stage=True)
tmp_dir.dvc_gen("lorem", "lorem")
run_copy("lorem", "lorem2", name="copy-lorem-lorem2")
@@ -351,13 +349,13 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, local_remote):
tmp_dir.dvc_gen("ipsum", "ipsum")
run_copy("ipsum", "baz", name="copy-ipsum-baz")
outs = ["foo", "bar", "lorem", "ipsum", "baz", "lorem2"]
outs = ["foo", "lorem", "ipsum", "baz", "lorem2"]
remove(dvc.stage_cache.cache_dir)
dvc.push()
outs = ["foo", "bar", "lorem", "ipsum", "baz", "lorem2"]
outs = ["foo", "lorem", "ipsum", "baz", "lorem2"]
# each one's a copy of other, hence 3
assert len(recurse_list_dir(path)) == 3
+5 -27
View File
@@ -65,14 +65,7 @@ def test_run_load_one_for_multistage_non_existing_stage_name(tmp_dir, dvc):
def test_run_load_one_on_single_stage(tmp_dir, dvc):
tmp_dir.gen("foo", "foo")
stage = dvc.run(
cmd="cp foo foo2",
deps=["foo"],
metrics=["foo2"],
always_changed=True,
single_stage=True,
)
(stage,) = tmp_dir.dvc_gen("foo", "foo")
assert isinstance(load_file(dvc, stage.path), SingleStageFile)
assert load_file(dvc, stage.path).stages.get("random-name") == stage
assert load_file(dvc, stage.path).stage == stage
@@ -120,15 +113,8 @@ def test_load_all_multistage(tmp_dir, dvc):
def test_load_all_singlestage(tmp_dir, dvc):
tmp_dir.gen("foo", "foo")
stage1 = dvc.run(
cmd="cp foo foo2",
deps=["foo"],
metrics=["foo2"],
always_changed=True,
single_stage=True,
)
dvcfile = load_file(dvc, "foo2.dvc")
(stage1,) = tmp_dir.dvc_gen("foo", "foo")
dvcfile = load_file(dvc, "foo.dvc")
assert isinstance(dvcfile, SingleStageFile)
assert len(dvcfile.stages) == 1
stages = dvcfile.stages.values()
@@ -167,14 +153,7 @@ def test_stage_collection(tmp_dir, dvc):
metrics=["foo2"],
always_changed=True,
)
stage3 = dvc.run(
cmd="cp bar bar2",
deps=["bar"],
metrics=["bar2"],
always_changed=True,
single_stage=True,
)
assert set(dvc.index.stages) == {stage1, stage3, stage2}
assert set(dvc.index.stages) == {stage1, stage2}
def test_remove_stage(tmp_dir, dvc, run_copy):
@@ -227,8 +206,7 @@ def test_remove_stage_lockfile(tmp_dir, dvc, run_copy):
def test_remove_stage_dvcfiles(tmp_dir, dvc, run_copy):
tmp_dir.gen("foo", "foo")
stage = run_copy("foo", "bar", single_stage=True)
(stage,) = tmp_dir.dvc_gen("foo", "foo")
dvc_file = load_file(dvc, stage.path)
assert dvc_file.exists()
+2 -4
View File
@@ -149,11 +149,9 @@ def test_non_cached_output(tmp_dir, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc.run(
outs_no_cache=[src],
cmd="echo hello > non_cached_file",
single_stage=True,
outs_no_cache=[src], cmd="echo hello > non_cached_file", name="gen"
)
erepo_dir.scm_add([src, src + ".dvc"], commit="add non-cached output")
erepo_dir.scm_add(["dvc.lock", "dvc.yaml"], commit="add non-cached output")
Repo.get(os.fspath(erepo_dir), src, dst)
+3 -1
View File
@@ -160,7 +160,9 @@ def test_import_non_cached(erepo_dir, tmp_dir, dvc, scm):
with erepo_dir.chdir():
erepo_dir.dvc.run(
cmd=f"echo hello > {src}", outs_no_cache=[src], single_stage=True
cmd=f"echo hello > {src}",
outs_no_cache=[src],
name="gen",
)
erepo_dir.scm_add([os.fspath(erepo_dir / src)], commit="add a non-cached out")
+6 -5
View File
@@ -44,10 +44,9 @@ def create_dvc_pipeline(tmp_dir, dvc):
cmd="python script.py {}".format(os.path.join("out", "file")),
outs=[os.path.join("out", "file")],
deps=["dep"],
fname="out.dvc",
single_stage=True,
name="touch",
)
tmp_dir.scm_add(["out.dvc"], commit="run")
tmp_dir.scm_add(["dvc.yaml", "dvc.lock"], commit="run")
shutil.rmtree("out")
@@ -243,7 +242,8 @@ def test_ls_repo_with_removed_dvc_dir(tmp_dir, dvc, scm):
(
(("script.py",), False),
(("dep.dvc",), False),
(("out.dvc",), False),
(("dvc.yaml",), False),
(("dvc.lock",), False),
(("dep",), True),
(("out",), False),
((".dvcignore",), False),
@@ -261,7 +261,8 @@ def test_ls_repo_with_removed_dvc_dir_recursive(tmp_dir, dvc, scm):
(
(("script.py",), False),
(("dep.dvc",), False),
(("out.dvc",), False),
(("dvc.yaml",), False),
(("dvc.lock",), False),
(("dep",), True),
(("out", "file"), True),
((".dvcignore",), False),
+1 -1
View File
@@ -39,7 +39,7 @@ def test_move_not_data_source(tmp_dir, dvc):
cmd="cp foo file1",
outs=["file1"],
deps=["foo"],
single_stage=True,
name="copy-foo-file1",
)
with pytest.raises(MoveNotDataSourceError):
+2 -2
View File
@@ -13,7 +13,7 @@ from tests.utils import get_gitignore_content
@pytest.mark.parametrize("remove_outs", [True, False])
def test_remove(tmp_dir, scm, dvc, run_copy, remove_outs):
(stage1,) = tmp_dir.dvc_gen("foo", "foo")
stage2 = run_copy("foo", "bar", single_stage=True)
stage2 = run_copy("foo", "bar", name="copy-foo-bar")
stage3 = run_copy("bar", "foobar", name="copy-bar-foobar")
assert "/foo" in get_gitignore_content()
@@ -96,7 +96,7 @@ def test_cmd_remove_gitignore_single_stage(tmp_dir, scm, dvc, run_copy):
def test_cmd_remove_gitignore_multistage(tmp_dir, scm, dvc, run_copy):
(stage,) = tmp_dir.dvc_gen("foo", "foo")
stage1 = run_copy("foo", "foo1", single_stage=True)
stage1 = run_copy("foo", "foo1", name="copy-foo-foo1")
stage2 = run_copy("foo1", "foo2", name="copy-foo1-foo2")
assert (tmp_dir / ".gitignore").exists()
+1 -2
View File
@@ -36,7 +36,7 @@ def test_destroy(tmp_dir, dvc, run_copy):
tmp_dir.dvc_gen("file", "text")
tmp_dir.dvc_gen({"dir": {"file": "lorem", "subdir/file": "ipsum"}})
run_copy("file", "file2", single_stage=True)
run_copy("file", "file2", name="copy-file-file2")
run_copy("file2", "file3", name="copy-file2-file3")
run_copy("file3", "file4", name="copy-file3-file4")
@@ -47,7 +47,6 @@ def test_destroy(tmp_dir, dvc, run_copy):
".dvc",
".dvcignore",
"file.dvc",
"file2.dvc",
"dir.dvc",
PROJECT_FILE,
LOCK_FILE,
+162 -93
View File
@@ -19,37 +19,9 @@ from dvc.utils.serialize import dump_yaml, load_yaml
from dvc_data.hashfile.hash import file_md5
@pytest.fixture(
params=(
{"single_stage": True},
{"single_stage": False},
),
ids=["single_stage", "multi_stage"],
)
def run_stage(dvc, request):
def inner(*args, name=None, **kwargs):
assert name
# these should not be passed
assert "single_stage" not in kwargs
assert "fname" not in kwargs
if request.param["single_stage"]:
kwargs.update(
{
"fname": name + ".dvc",
"single_stage": True,
}
)
else:
kwargs["name"] = name
return dvc.run(*args, **kwargs)
return inner
def test_repro_fail(tmp_dir, run_stage, copy_script):
def test_repro_fail(tmp_dir, dvc, copy_script):
tmp_dir.dvc_gen("foo", "foo")
stage = run_stage(
stage = dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
@@ -59,15 +31,18 @@ def test_repro_fail(tmp_dir, run_stage, copy_script):
assert main(["repro", stage.addressing]) != 0
def test_repro_cyclic_graph(tmp_dir, dvc, run_stage):
def test_repro_cyclic_graph(
tmp_dir,
dvc,
):
tmp_dir.gen("foo", "foo")
run_stage(
dvc.run(
deps=["foo"],
outs=["bar.txt"],
cmd="echo bar > bar.txt",
name="copybarbar-txt",
)
run_stage(
dvc.run(
deps=["bar.txt"],
outs=["baz.txt"],
cmd="echo baz > baz.txt",
@@ -196,11 +171,15 @@ class TestReproWorkingDirectoryAsOutput:
dvc.reproduce(stage)
def test_repro_dep_under_dir(tmp_dir, dvc, copy_script, run_stage):
def test_repro_dep_under_dir(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("foo", "foo")
tmp_dir.dvc_gen("data", {"file": "file", "sub": {"foo": "foo"}})
stage = run_stage(
stage = dvc.run(
outs=["file1"],
deps=["data/file", "copy.py"],
cmd="python copy.py data/file file1",
@@ -217,17 +196,21 @@ def test_repro_dep_under_dir(tmp_dir, dvc, copy_script, run_stage):
assert filecmp.cmp("file1", "foo", shallow=False)
def test_repro_dep_dir_with_outputs_under_it(tmp_dir, dvc, copy_script, run_stage):
def test_repro_dep_dir_with_outputs_under_it(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("foo", "foo")
file_stage, _ = tmp_dir.dvc_gen(
{"data/file": "file", "data/sub": {"foo": "foo", "bar": "bar"}}
)
run_stage(
dvc.run(
cmd="ls data/file data/sub",
deps=["data/file", "data/sub"],
name="list-files",
)
copy_stage = run_stage(
copy_stage = dvc.run(
deps=["data"],
outs=["file1"],
cmd="python copy.py data file1",
@@ -238,9 +221,9 @@ def test_repro_dep_dir_with_outputs_under_it(tmp_dir, dvc, copy_script, run_stag
assert dvc.reproduce(copy_stage.addressing) == [file_stage, copy_stage]
def test_repro_force(tmp_dir, dvc, run_stage, copy_script):
def test_repro_force(tmp_dir, dvc, copy_script):
tmp_dir.dvc_gen("foo", "foo")
stage = run_stage(
stage = dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
@@ -250,10 +233,14 @@ def test_repro_force(tmp_dir, dvc, run_stage, copy_script):
assert len(stages) == 2
def test_repro_changed_code(tmp_dir, dvc, copy_script, run_stage):
def test_repro_changed_code(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = run_stage(
stage = dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
@@ -267,10 +254,14 @@ def test_repro_changed_code(tmp_dir, dvc, copy_script, run_stage):
assert len(stages) == 1
def test_repro_changed_data(tmp_dir, dvc, copy_script, run_stage):
def test_repro_changed_data(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = run_stage(
stage = dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
@@ -284,10 +275,14 @@ def test_repro_changed_data(tmp_dir, dvc, copy_script, run_stage):
assert len(stages) == 2
def test_repro_dry(tmp_dir, dvc, copy_script, run_stage):
def test_repro_dry(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = run_stage(
stage = dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
@@ -305,10 +300,14 @@ def test_repro_dry(tmp_dir, dvc, copy_script, run_stage):
assert not filecmp.cmp("file1", "bar", shallow=False)
def test_repro_up_to_date(tmp_dir, copy_script, run_stage):
def test_repro_up_to_date(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = run_stage(
stage = dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
@@ -366,16 +365,20 @@ def test_repro_dry_no_exec(tmp_dir, dvc):
assert ret == 0
def test_repro_changed_deep_data(tmp_dir, dvc, copy_script, run_stage):
def test_repro_changed_deep_data(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
run_stage(
dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
name="run1",
)
file2_stage = run_stage(
file2_stage = dvc.run(
outs=["file2"],
deps=["file1", "copy.py"],
cmd="python copy.py file1 file2",
@@ -434,16 +437,20 @@ def test_repro_force_downstream(tmp_dir, dvc, copy_script):
assert stages[1].path == file3_stage.path
def test_repro_pipeline(tmp_dir, dvc, copy_script, run_stage):
def test_repro_pipeline(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
run_stage(
dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
name="run1",
)
stage = run_stage(
stage = dvc.run(
outs=["file2"],
deps=["file1", "copy.py"],
cmd="python copy.py file1 file2",
@@ -453,10 +460,14 @@ def test_repro_pipeline(tmp_dir, dvc, copy_script, run_stage):
assert len(stages) == 3
def test_repro_pipeline_cli(tmp_dir, dvc, copy_script, run_stage):
def test_repro_pipeline_cli(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = run_stage(
stage = dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
@@ -466,15 +477,19 @@ def test_repro_pipeline_cli(tmp_dir, dvc, copy_script, run_stage):
assert ret == 0
def test_repro_pipelines(tmp_dir, dvc, copy_script, run_stage):
def test_repro_pipelines(
tmp_dir,
dvc,
copy_script,
):
foo_stage, bar_stage = tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
file1_stage = run_stage(
file1_stage = dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
name="copy-FOO-file1",
)
file2_stage = run_stage(
file2_stage = dvc.run(
outs=["file2"],
deps=["bar", "copy.py"],
cmd="python copy.py bar file2",
@@ -488,15 +503,19 @@ def test_repro_pipelines(tmp_dir, dvc, copy_script, run_stage):
}
def test_repro_pipelines_cli(tmp_dir, dvc, copy_script, run_stage):
def test_repro_pipelines_cli(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
run_stage(
dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
name="copy-FOO-file1",
)
run_stage(
dvc.run(
outs=["file2"],
deps=["bar", "copy.py"],
cmd="python copy.py bar file2",
@@ -505,16 +524,20 @@ def test_repro_pipelines_cli(tmp_dir, dvc, copy_script, run_stage):
assert main(["repro", "-f", "-P"]) == 0
def test_repro_frozen(tmp_dir, dvc, copy_script, run_stage):
def test_repro_frozen(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
run_stage(
dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
name="run1",
)
file2_stage = run_stage(
file2_stage = dvc.run(
outs=["file2"],
deps=["file1", "copy.py"],
cmd="python copy.py file1 file2",
@@ -556,11 +579,15 @@ def test_freeze_non_existing(dvc, target):
assert ret != 0
def test_repro_frozen_callback(tmp_dir, dvc, copy_script, run_stage):
def test_repro_frozen_callback(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("foo", "foo")
# NOTE: purposefully not specifying deps or outs
# to create a callback stage.
stage = run_stage(
stage = dvc.run(
cmd="python copy.py foo file1",
name="copy-FOO-file1",
)
@@ -577,13 +604,17 @@ def test_repro_frozen_callback(tmp_dir, dvc, copy_script, run_stage):
assert len(stages) == 1
def test_repro_frozen_unchanged(tmp_dir, dvc, copy_script, run_stage):
def test_repro_frozen_unchanged(
tmp_dir,
dvc,
copy_script,
):
"""
Check that freezing/unfreezing doesn't affect stage state
"""
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = run_stage(
stage = dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
@@ -636,16 +667,20 @@ def test_repro_metrics_add_unchanged(tmp_dir, dvc, copy_script):
assert len(stages) == 0
def test_repro_phony(tmp_dir, dvc, copy_script, run_stage):
def test_repro_phony(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
run_stage(
dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
name="run1",
)
stage = run_stage(cmd="cat file1", deps=["file1"], name="cat")
stage = dvc.run(cmd="cat file1", deps=["file1"], name="cat")
shutil.copyfile("bar", "foo")
dvc.reproduce(stage.addressing)
@@ -653,10 +688,14 @@ def test_repro_phony(tmp_dir, dvc, copy_script, run_stage):
assert filecmp.cmp("file1", "bar", shallow=False)
def test_non_existing_output(tmp_dir, dvc, copy_script, run_stage):
def test_non_existing_output(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = run_stage(
stage = dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
@@ -668,10 +707,14 @@ def test_non_existing_output(tmp_dir, dvc, copy_script, run_stage):
dvc.reproduce(stage.addressing)
def test_repro_data_source(tmp_dir, dvc, copy_script, run_stage):
def test_repro_data_source(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = run_stage(
stage = dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
@@ -685,11 +728,15 @@ def test_repro_data_source(tmp_dir, dvc, copy_script, run_stage):
assert stages[0].outs[0].hash_info.value == file_md5("bar")
def test_repro_changed_dir(tmp_dir, dvc, copy_script, run_stage):
def test_repro_changed_dir(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen({"foo": "foo", "bar": "bar"})
shutil.copyfile("foo", "file")
stage = run_stage(
stage = dvc.run(
outs=["dir"],
deps=["file", "copy.py"],
cmd="mkdir dir && python copy.py foo dir/foo",
@@ -706,9 +753,13 @@ def test_repro_changed_dir(tmp_dir, dvc, copy_script, run_stage):
assert len(stages) == 1
def test_repro_changed_dir_data(tmp_dir, dvc, copy_script, run_stage):
def test_repro_changed_dir_data(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen({"data": {"foo": "foo"}, "bar": "bar"})
stage = run_stage(
stage = dvc.run(
outs=["dir"],
deps=["data", "copy.py"],
cmd="python copy.py data dir",
@@ -753,10 +804,14 @@ def test_repro_missing_md5_in_stage_file(tmp_dir, dvc, copy_script):
assert len(stages) == 1
def test_cmd_repro(tmp_dir, copy_script, run_stage):
def test_cmd_repro(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = run_stage(
stage = dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
@@ -792,26 +847,29 @@ def test_repro_shell(tmp_dir, monkeypatch, dvc):
assert (tmp_dir / "shell.txt").read_text().rstrip() == shell
def test_repro_all_pipelines(mocker, dvc, run_stage):
def test_repro_all_pipelines(
mocker,
dvc,
):
stages = [
run_stage(
dvc.run(
outs=["start.txt"],
cmd="echo start > start.txt",
name="start",
),
run_stage(
dvc.run(
deps=["start.txt"],
outs=["middle.txt"],
cmd="echo middle > middle.txt",
name="middle",
),
run_stage(
dvc.run(
deps=["middle.txt"],
outs=["final.txt"],
cmd="echo final > final.txt",
name="final",
),
run_stage(
dvc.run(
outs=["disconnected.txt"],
cmd="echo other > disconnected.txt",
name="disconnected",
@@ -828,10 +886,14 @@ def test_repro_all_pipelines(mocker, dvc, run_stage):
assert mock_reproduce.call_count == 4
def test_repro_no_commit(tmp_dir, dvc, copy_script, run_stage):
def test_repro_no_commit(
tmp_dir,
dvc,
copy_script,
):
tmp_dir.gen("bar", "bar")
tmp_dir.dvc_gen("foo", "foo")
stage = run_stage(
stage = dvc.run(
outs=["file1"],
deps=["foo", "copy.py"],
cmd="python copy.py foo file1",
@@ -845,8 +907,11 @@ def test_repro_no_commit(tmp_dir, dvc, copy_script, run_stage):
class TestReproAlreadyCached:
def test(self, dvc, run_stage):
stage = run_stage(
def test(
self,
dvc,
):
stage = dvc.run(
always_changed=True,
deps=[],
outs=["datetime.txt"],
@@ -858,9 +923,13 @@ class TestReproAlreadyCached:
assert run_out.hash_info != repro_out.hash_info
def test_force_with_dependencies(self, tmp_dir, dvc, run_stage):
def test_force_with_dependencies(
self,
tmp_dir,
dvc,
):
tmp_dir.dvc_gen("foo", "foo")
stage = run_stage(
stage = dvc.run(
name="datetime",
deps=["foo"],
outs=["datetime.txt"],
+3 -16
View File
@@ -40,20 +40,6 @@ def test_run_no_exec(tmp_dir, dvc, run_copy):
}
def test_run_with_multistage_and_single_stage(tmp_dir, dvc, run_copy):
from dvc.stage import PipelineStage, Stage
tmp_dir.dvc_gen("foo", "foo")
stage1 = run_copy("foo", "foo1", single_stage=True)
stage2 = run_copy("foo1", "foo2", name="copy-foo1-foo2")
stage3 = run_copy("foo2", "foo3", single_stage=True)
assert isinstance(stage2, PipelineStage)
assert isinstance(stage1, Stage)
assert isinstance(stage3, Stage)
assert stage2.name == "copy-foo1-foo2"
def test_run_multi_stage_repeat(tmp_dir, dvc, run_copy):
from dvc.dvcfile import PROJECT_FILE, load_file
from dvc.stage import PipelineStage
@@ -61,14 +47,15 @@ def test_run_multi_stage_repeat(tmp_dir, dvc, run_copy):
tmp_dir.dvc_gen("foo", "foo")
run_copy("foo", "foo1", name="copy-foo-foo1")
run_copy("foo1", "foo2", name="copy-foo1-foo2")
run_copy("foo2", "foo3", single_stage=True)
run_copy("foo2", "foo3", name="copy-foo2-foo3")
stages = list(load_file(dvc, PROJECT_FILE).stages.values())
assert len(stages) == 2
assert len(stages) == 3
assert all(isinstance(stage, PipelineStage) for stage in stages)
assert {stage.name for stage in stages} == {
"copy-foo-foo1",
"copy-foo1-foo2",
"copy-foo2-foo3",
}
+13 -19
View File
@@ -89,21 +89,16 @@ def test_reload(tmp_dir, dvc):
def test_default_wdir_ignored_in_checksum(tmp_dir, dvc):
tmp_dir.gen("bar", "bar")
stage = dvc.run(
cmd="cp bar foo",
deps=["bar"],
outs=["foo"],
single_stage=True,
)
stage = dvc.run(cmd="cp bar foo", deps=["bar"], outs=["foo"], name="copy-foo-bar")
d = stage.dumpd()
assert Stage.PARAM_WDIR not in d.keys()
d = load_yaml(stage.relpath)
assert Stage.PARAM_WDIR not in d.keys()
d = load_yaml("dvc.yaml")
assert Stage.PARAM_WDIR not in d["stages"]["copy-foo-bar"]
with dvc.lock:
stage = SingleStageFile(dvc, stage.relpath).stage
stage = stage.reload()
assert not stage.changed()
@@ -117,7 +112,7 @@ def test_external_remote_output_resolution(tmp_dir, dvc, make_remote):
dvc.run(
cmd=f"echo file > {file_path}",
outs_no_cache=["remote://storage/file"],
single_stage=True,
name="gen-file",
)
assert os.path.exists(file_path)
@@ -245,16 +240,15 @@ def test_collect_symlink(tmp_dir, dvc, with_deps):
def test_stage_strings_representation(tmp_dir, dvc, run_copy):
tmp_dir.dvc_gen("foo", "foo")
stage1 = run_copy("foo", "bar", single_stage=True)
assert stage1.addressing == "bar.dvc"
assert repr(stage1) == "Stage: 'bar.dvc'"
assert str(stage1) == "stage: 'bar.dvc'"
(stage1,) = tmp_dir.dvc_gen("foo", "foo")
assert stage1.addressing == "foo.dvc"
assert repr(stage1) == "Stage: 'foo.dvc'"
assert str(stage1) == "stage: 'foo.dvc'"
stage2 = run_copy("bar", "baz", name="copy-bar-baz")
assert stage2.addressing == "copy-bar-baz"
assert repr(stage2) == "Stage: 'copy-bar-baz'"
assert str(stage2) == "stage: 'copy-bar-baz'"
stage2 = run_copy("foo", "bar", name="copy-foo-bar")
assert stage2.addressing == "copy-foo-bar"
assert repr(stage2) == "Stage: 'copy-foo-bar'"
assert str(stage2) == "stage: 'copy-foo-bar'"
folder = tmp_dir / "dir"
folder.mkdir()
+23 -15
View File
@@ -26,19 +26,19 @@ def test_collect(tmp_dir, scm, dvc, run_copy):
}
tmp_dir.dvc_gen("foo", "foo")
run_copy("foo", "bar", single_stage=True)
scm.add([".gitignore", "foo.dvc", "bar.dvc"])
run_copy("foo", "bar", name="copy-foo-bar")
scm.add([".gitignore", "foo.dvc", "dvc.yaml", "dvc.lock"])
scm.commit("Add foo and bar")
scm.checkout("new-branch", create_new=True)
run_copy("bar", "buzz", single_stage=True)
scm.add([".gitignore", "buzz.dvc"])
run_copy("bar", "buzz", name="copy-bar-buzz")
scm.add([".gitignore", "dvc.yaml", "dvc.lock"])
scm.commit("Add buzz")
assert collect_outs("bar.dvc", with_deps=True) == {"foo", "bar"}
assert collect_outs("buzz.dvc", with_deps=True) == {"foo", "bar", "buzz"}
assert collect_outs("buzz.dvc", with_deps=False) == {"buzz"}
assert collect_outs("copy-foo-bar", with_deps=True) == {"foo", "bar"}
assert collect_outs("copy-bar-buzz", with_deps=True) == {"foo", "bar", "buzz"}
assert collect_outs("copy-bar-buzz", with_deps=False) == {"buzz"}
run_copy("foo", "foobar", name="copy-foo-foobar")
assert collect_outs(":copy-foo-foobar") == {"foobar"}
@@ -52,16 +52,22 @@ def test_collect(tmp_dir, scm, dvc, run_copy):
assert collect_outs("copy-foo-foobar", recursive=True) == {"foobar"}
run_copy("foobar", "baz", name="copy-foobar-baz")
assert collect_outs("dvc.yaml") == {"foobar", "baz"}
assert collect_outs("dvc.yaml", with_deps=True) == {"foobar", "baz", "foo"}
assert collect_outs("dvc.yaml") == {"foobar", "baz", "bar", "buzz"}
assert collect_outs("dvc.yaml", with_deps=True) == {
"foobar",
"baz",
"bar",
"buzz",
"foo",
}
def test_collect_dir_recursive(tmp_dir, dvc, run_head):
tmp_dir.gen({"dir": {"foo": "foo"}})
(stage1,) = dvc.add("dir/*", glob=True)
with (tmp_dir / "dir").chdir():
stage2 = run_head("foo", name="copy-foo-bar")
stage3 = run_head("foo-1", single_stage=True)
stage2 = run_head("foo", name="head-foo")
stage3 = run_head("foo-1", name="head-foo1")
assert set(dvc.stage.collect("dir", recursive=True)) == {
stage1,
stage2,
@@ -101,7 +107,7 @@ def stages(tmp_dir, run_copy):
return {
"foo-generate": stage1,
"lorem-generate": stage2,
"copy-foo-bar": run_copy("foo", "bar", single_stage=True),
"copy-foo-bar": run_copy("foo", "bar", name="copy-foo-bar"),
"copy-bar-foobar": run_copy("bar", "foobar", name="copy-bar-foobar"),
"copy-lorem-ipsum": run_copy("lorem", "ipsum", name="copy-lorem-ipsum"),
}
@@ -147,7 +153,7 @@ def test_collect_generated(tmp_dir, dvc):
def test_collect_glob(tmp_dir, dvc, stages):
assert set(dvc.stage.collect("copy*", glob=True)) == {
stages[key] for key in ["copy-bar-foobar", "copy-lorem-ipsum"]
stages[key] for key in ["copy-bar-foobar", "copy-foo-bar", "copy-lorem-ipsum"]
}
assert set(dvc.stage.collect("copy-lorem*", glob=True, with_deps=True)) == {
stages[key] for key in ["copy-lorem-ipsum", "lorem-generate"]
@@ -162,12 +168,14 @@ def test_collect_granular_with_no_target(tmp_dir, dvc, stages):
def test_collect_granular_with_target(tmp_dir, dvc, stages):
assert dvc.stage.collect_granular("bar.dvc") == [(stages["copy-foo-bar"], None)]
assert dvc.stage.collect_granular("foo.dvc") == [(stages["foo-generate"], None)]
assert dvc.stage.collect_granular(PROJECT_FILE) == [
(stages["copy-foo-bar"], None),
(stages["copy-bar-foobar"], None),
(stages["copy-lorem-ipsum"], None),
]
assert dvc.stage.collect_granular(":") == [
(stages["copy-foo-bar"], None),
(stages["copy-bar-foobar"], None),
(stages["copy-lorem-ipsum"], None),
]
@@ -218,7 +226,7 @@ def test_collect_granular_with_deps(tmp_dir, dvc, stages):
assert set(
map(
itemgetter(0),
dvc.stage.collect_granular("bar.dvc", with_deps=True),
dvc.stage.collect_granular("copy-foo-bar", with_deps=True),
)
) == {stages["copy-foo-bar"], stages["foo-generate"]}
assert set(
+3 -3
View File
@@ -82,21 +82,21 @@ def test_skip_graph_checks(tmp_dir, dvc, mocker, run_copy):
# sanity check
tmp_dir.gen("foo", "foo text")
dvc.add("foo")
run_copy("foo", "bar", single_stage=True)
run_copy("foo", "bar", name="copy-foo-bar")
assert mock_build_graph.called
# check that our hack can be enabled
mock_build_graph.reset_mock()
dvc._skip_graph_checks = True
tmp_dir.gen("baz", "baz text")
run_copy("baz", "qux", single_stage=True)
run_copy("baz", "qux", name="copy-baz-qux")
assert not mock_build_graph.called
# check that our hack can be disabled
mock_build_graph.reset_mock()
dvc._skip_graph_checks = False
tmp_dir.gen("quux", "quux text")
run_copy("quux", "quuz", single_stage=True)
run_copy("quux", "quuz", name="copy-quux-quuz")
assert mock_build_graph.called
+4 -4
View File
@@ -9,10 +9,10 @@ def test_number_reproduces(tmp_dir, dvc, mocker):
)
tmp_dir.dvc_gen({"pre-foo": "pre-foo"})
dvc.run(single_stage=True, deps=["pre-foo"], outs=["foo"], cmd="echo foo > foo")
dvc.run(single_stage=True, deps=["foo"], outs=["bar"], cmd="echo bar > bar")
dvc.run(single_stage=True, deps=["foo"], outs=["baz"], cmd="echo baz > baz")
dvc.run(single_stage=True, deps=["bar"], outs=["boop"], cmd="echo boop > boop")
dvc.run(name="echo-foo", outs=["foo"], cmd="echo foo > foo")
dvc.run(name="echo-bar", deps=["foo"], outs=["bar"], cmd="echo bar > bar")
dvc.run(name="echo-baz", deps=["foo"], outs=["baz"], cmd="echo baz > baz")
dvc.run(name="echo-boop", deps=["bar"], outs=["boop"], cmd="echo boop > boop")
reproduce_stage_mock.reset_mock()
+9 -6
View File
@@ -15,14 +15,15 @@ def test_stage_cache(tmp_dir, dvc, mocker):
cmd="python script.py",
deps=["script.py", "dep"],
outs=["out"],
single_stage=True,
name="write-out",
)
with dvc.lock:
stage.remove(remove_outs=True, force=True)
assert not (tmp_dir / "out").exists()
assert not (tmp_dir / "out.dvc").exists()
assert not (tmp_dir / "dvc.yaml").exists()
assert not (tmp_dir / "dvc.lock").exists()
cache_dir = os.path.join(
dvc.stage_cache.cache_dir,
@@ -61,14 +62,15 @@ def test_stage_cache_params(tmp_dir, dvc, mocker):
cmd="python script.py",
params=["foo,bar", "myparams.yaml:baz,qux"],
outs=["out"],
single_stage=True,
name="write-out",
)
with dvc.lock:
stage.remove(remove_outs=True, force=True)
assert not (tmp_dir / "out").exists()
assert not (tmp_dir / "out.dvc").exists()
assert not (tmp_dir / "dvc.yaml").exists()
assert not (tmp_dir / "dvc.lock").exists()
cache_dir = os.path.join(
dvc.stage_cache.cache_dir,
@@ -107,7 +109,7 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker):
cmd="python ../script.py",
deps=["../script.py", "../dep"],
outs=["out"],
single_stage=True,
name="write-out",
wdir="wdir",
)
@@ -115,7 +117,8 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker):
stage.remove(remove_outs=True, force=True)
assert not (tmp_dir / "wdir" / "out").exists()
assert not (tmp_dir / "wdir" / "out.dvc").exists()
assert not (tmp_dir / "wdir" / "dvc.yaml").exists()
assert not (tmp_dir / "wdir" / "dvc.lock").exists()
cache_dir = os.path.join(
dvc.stage_cache.cache_dir,