From 26152292aa0804bfb5be196b944914e14944b86b Mon Sep 17 00:00:00 2001 From: skshetry <18718008+skshetry@users.noreply.github.com> Date: Tue, 6 Jun 2023 15:29:10 +0545 Subject: [PATCH] remove support for reading and generating Dvcfile (#9546) --- dvc/dvcfile.py | 6 +----- dvc/stage/utils.py | 8 +++----- tests/func/params/test_diff.py | 18 ++++++++--------- tests/func/params/test_show.py | 26 ++++++++---------------- tests/func/test_repro.py | 4 ++-- tests/func/test_run_single_stage.py | 31 ++++++++++++++++++++++------- tests/unit/stage/test_stage.py | 2 +- 7 files changed, 48 insertions(+), 47 deletions(-) diff --git a/dvc/dvcfile.py b/dvc/dvcfile.py index 326a92790..fa46694a2 100644 --- a/dvc/dvcfile.py +++ b/dvc/dvcfile.py @@ -35,7 +35,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) _T = TypeVar("_T") -DVC_FILE = "Dvcfile" DVC_FILE_SUFFIX = ".dvc" PROJECT_FILE = "dvc.yaml" LOCK_FILE = "dvc.lock" @@ -55,10 +54,7 @@ class ParametrizedDumpError(DvcException): def is_valid_filename(path): - return path.endswith(DVC_FILE_SUFFIX) or os.path.basename(path) in [ - DVC_FILE, - PROJECT_FILE, - ] + return path.endswith(DVC_FILE_SUFFIX) or os.path.basename(path) == PROJECT_FILE def is_dvc_file(path): diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index 568937f48..25dfc8190 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -225,7 +225,7 @@ def prepare_file_path(kwargs) -> str: Used in creating .dvc files. """ - from dvc.dvcfile import DVC_FILE, DVC_FILE_SUFFIX + from dvc.dvcfile import DVC_FILE_SUFFIX out = first( concat( @@ -239,10 +239,8 @@ def prepare_file_path(kwargs) -> str: kwargs.get("outs_persist_no_cache", []), ) ) - - return ( - os.path.basename(os.path.normpath(out)) + DVC_FILE_SUFFIX if out else DVC_FILE - ) + assert out + return os.path.basename(os.path.normpath(out)) + DVC_FILE_SUFFIX def check_stage_exists(repo: "Repo", stage: Union["Stage", "PipelineStage"], path: str): diff --git a/tests/func/params/test_diff.py b/tests/func/params/test_diff.py index a698665a6..1d0021b0c 100644 --- a/tests/func/params/test_diff.py +++ b/tests/func/params/test_diff.py @@ -11,7 +11,7 @@ def test_diff_no_params(tmp_dir, scm, dvc): def test_diff_no_changes(tmp_dir, scm, dvc): tmp_dir.gen("params.yaml", "foo: bar") - dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True) + dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params") scm.add(["params.yaml", "Dvcfile"]) scm.commit("bar") assert dvc.params.diff() == {} @@ -19,7 +19,7 @@ def test_diff_no_changes(tmp_dir, scm, dvc): def test_diff(tmp_dir, scm, dvc): tmp_dir.gen("params.yaml", "foo: bar") - dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True) + dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params") scm.add(["params.yaml", "Dvcfile"]) scm.commit("bar") @@ -33,7 +33,7 @@ def test_diff(tmp_dir, scm, dvc): def test_diff_dirty(tmp_dir, scm, dvc): tmp_dir.gen("params.yaml", "foo: bar") - dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True) + dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params") scm.add(["params.yaml", "Dvcfile"]) scm.commit("bar") @@ -45,14 +45,14 @@ def test_diff_dirty(tmp_dir, scm, dvc): def test_diff_new(tmp_dir, scm, dvc): tmp_dir.gen("params.yaml", "foo: bar") - dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True) + dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params") assert dvc.params.diff() == {"params.yaml": {"foo": {"old": None, "new": "bar"}}} def test_diff_deleted(tmp_dir, scm, dvc): tmp_dir.gen("params.yaml", "foo: bar") - dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True) + dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params") scm.add(["params.yaml", "Dvcfile"]) scm.commit("bar") @@ -63,7 +63,7 @@ def test_diff_deleted(tmp_dir, scm, dvc): def test_diff_list(tmp_dir, scm, dvc): tmp_dir.gen("params.yaml", "foo:\n- bar\n- baz") - dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True) + dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params") scm.add(["params.yaml", "Dvcfile"]) scm.commit("foo") @@ -78,7 +78,7 @@ def test_diff_list(tmp_dir, scm, dvc): def test_diff_dict(tmp_dir, scm, dvc): tmp_dir.gen("params.yaml", "foo:\n bar: baz") - dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True) + dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params") scm.add(["params.yaml", "Dvcfile"]) scm.commit("foo") @@ -91,7 +91,7 @@ def test_diff_dict(tmp_dir, scm, dvc): def test_diff_with_unchanged(tmp_dir, scm, dvc): tmp_dir.gen("params.yaml", "foo: bar\nxyz: val") - dvc.run(cmd="echo params.yaml", params=["foo,xyz"], single_stage=True) + dvc.run(cmd="echo params.yaml", params=["foo,xyz"], name="echo-params") scm.add(["params.yaml", "Dvcfile"]) scm.commit("bar") @@ -261,7 +261,7 @@ def test_diff_top_level_params(tmp_dir, dvc, scm, dvcfile, params_file): def test_diff_active_branch_no_changes(tmp_dir, scm, dvc): tmp_dir.gen("params.yaml", "foo: bar") - dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True) + dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params") scm.add(["params.yaml", "Dvcfile"]) scm.commit("bar") assert dvc.params.diff(a_rev=tmp_dir.scm.active_branch()) == {} diff --git a/tests/func/params/test_show.py b/tests/func/params/test_show.py index 4a5e1f482..dc8d849e0 100644 --- a/tests/func/params/test_show.py +++ b/tests/func/params/test_show.py @@ -14,7 +14,7 @@ def test_show_empty(dvc): def test_show(tmp_dir, dvc): tmp_dir.gen("params.yaml", "foo: bar") - dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True) + dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params") assert dvc.params.show() == { "": {"data": {"params.yaml": {"data": {"foo": "bar"}}}} } @@ -22,7 +22,7 @@ def test_show(tmp_dir, dvc): def test_show_targets(tmp_dir, dvc): tmp_dir.gen("params.yaml", "foo: bar") - dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True) + dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params") expected = {"": {"data": {"params.yaml": {"data": {"foo": "bar"}}}}} assert dvc.params.show(targets=["params.yaml"]) == expected assert dvc.params.show(targets=(tmp_dir / "params.yaml").fs_path) == expected @@ -30,7 +30,7 @@ def test_show_targets(tmp_dir, dvc): def test_show_toml(tmp_dir, dvc): tmp_dir.gen("params.toml", "[foo]\nbar = 42\nbaz = [1, 2]\n") - dvc.run(cmd="echo params.toml", params=["params.toml:foo"], single_stage=True) + dvc.run(cmd="echo params.toml", params=["params.toml:foo"], name="echo-params") assert dvc.params.show() == { "": {"data": {"params.toml": {"data": {"foo": {"bar": 42, "baz": [1, 2]}}}}} } @@ -44,7 +44,7 @@ def test_show_py(tmp_dir, dvc): dvc.run( cmd="echo params.py", params=["params.py:CONST,IS_DIR,Config.foo"], - single_stage=True, + name="echo-params", ) assert dvc.params.show() == { "": { @@ -59,18 +59,8 @@ def test_show_py(tmp_dir, dvc): def test_show_multiple(tmp_dir, dvc): tmp_dir.gen("params.yaml", "foo: bar\nbaz: qux\n") - dvc.run( - cmd="echo params.yaml", - fname="foo.dvc", - params=["foo"], - single_stage=True, - ) - dvc.run( - cmd="echo params.yaml", - fname="baz.dvc", - params=["baz"], - single_stage=True, - ) + dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params1") + dvc.run(cmd="echo params.yaml", params=["baz"], name="echo-params2") assert dvc.params.show() == { "": {"data": {"params.yaml": {"data": {"baz": "qux", "foo": "bar"}}}} } @@ -78,7 +68,7 @@ def test_show_multiple(tmp_dir, dvc): def test_show_list(tmp_dir, dvc): tmp_dir.gen("params.yaml", "foo:\n- bar\n- baz\n") - dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True) + dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params") assert dvc.params.show() == { "": {"data": {"params.yaml": {"data": {"foo": ["bar", "baz"]}}}} } @@ -86,7 +76,7 @@ def test_show_list(tmp_dir, dvc): def test_show_branch(tmp_dir, scm, dvc): tmp_dir.gen("params.yaml", "foo: bar") - dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True) + dvc.run(cmd="echo params.yaml", params=["foo"], name="echo-params") scm.add(["params.yaml", "Dvcfile"]) scm.commit("init") diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index f27836bea..939e59e53 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -7,7 +7,7 @@ from pathlib import Path import pytest from dvc.cli import main -from dvc.dvcfile import DVC_FILE, load_file +from dvc.dvcfile import load_file from dvc.exceptions import CyclicGraphError, ReproductionError, StagePathAsOutputError from dvc.fs import LocalFileSystem, system from dvc.output import Output @@ -943,7 +943,7 @@ def repro_dir(tmp_dir, dvc, run_copy): assert dir_file_copy.read_text() == "dir file content" stages["dir_file_copy"] = stage - last_stage = tmp_dir / "dir" / DVC_FILE + last_stage = tmp_dir / "dir" / "echo.dvc" deps = [os.fspath(origin_copy_2), os.fspath(dir_file_copy)] stage = dvc.run( cmd="echo {}".format(" ".join(deps)), diff --git a/tests/func/test_run_single_stage.py b/tests/func/test_run_single_stage.py index 4d9b9e0f8..76adceb1b 100644 --- a/tests/func/test_run_single_stage.py +++ b/tests/func/test_run_single_stage.py @@ -234,30 +234,40 @@ class TestRunStageInsideOutput: class TestRunBadCwd: def test(self, make_tmp_dir, dvc): with pytest.raises(StagePathOutsideError): - dvc.run(cmd="command", wdir=make_tmp_dir("tmp"), single_stage=True) + dvc.run( + cmd="command", + wdir=make_tmp_dir("tmp"), + fname="bad-cwd.dvc", + single_stage=True, + ) def test_same_prefix(self, tmp_dir, dvc): path = f"{tmp_dir}-{uuid.uuid4()}" os.mkdir(path) with pytest.raises(StagePathOutsideError): - dvc.run(cmd="command", wdir=path, single_stage=True) + dvc.run(cmd="command", wdir=path, fname="bad-cwd.dvc", single_stage=True) class TestRunBadWdir: def test(self, make_tmp_dir, dvc): with pytest.raises(StagePathOutsideError): - dvc.run(cmd="command", wdir=make_tmp_dir("tmp"), single_stage=True) + dvc.run( + cmd="command", + wdir=make_tmp_dir("tmp"), + fname="bad-wdir.dvc", + single_stage=True, + ) def test_same_prefix(self, tmp_dir, dvc): path = f"{tmp_dir}-{uuid.uuid4()}" os.mkdir(path) with pytest.raises(StagePathOutsideError): - dvc.run(cmd="command", wdir=path, single_stage=True) + dvc.run(cmd="command", wdir=path, fname="bad-wdir.dvc", single_stage=True) def test_not_found(self, tmp_dir, dvc): path = os.path.join(tmp_dir, str(uuid.uuid4())) with pytest.raises(StagePathNotFoundError): - dvc.run(cmd="command", wdir=path, single_stage=True) + dvc.run(cmd="command", wdir=path, fname="bad-wdir.dvc", single_stage=True) def test_not_dir(self, tmp_dir, dvc): path = tmp_dir / str(uuid.uuid4()) @@ -265,7 +275,12 @@ class TestRunBadWdir: path = path / str(uuid.uuid4()) path.touch() with pytest.raises(StagePathNotDirectoryError): - dvc.run(cmd="command", wdir=os.fspath(path), single_stage=True) + dvc.run( + cmd="command", + wdir=os.fspath(path), + fname="bad-wdir.dvc", + single_stage=True, + ) class TestRunBadName: @@ -372,7 +387,9 @@ def test_rerun_deterministic_ignore_cache(tmp_dir, run_copy, mocker): def test_rerun_callback(dvc): def run_callback(force=False): - return dvc.run(cmd="echo content > out", force=force, single_stage=True) + return dvc.run( + cmd="echo content > out", force=force, fname="echo.dvc", single_stage=True + ) assert run_callback() is not None with pytest.raises(StageFileAlreadyExistsError): diff --git a/tests/unit/stage/test_stage.py b/tests/unit/stage/test_stage.py index 2227cefda..35ce11ef0 100644 --- a/tests/unit/stage/test_stage.py +++ b/tests/unit/stage/test_stage.py @@ -90,7 +90,7 @@ def test_stage_run_ignore_sigint(dvc, mocker): popen = mocker.patch.object(subprocess, "Popen", return_value=proc) signal_mock = mocker.patch("signal.signal") - dvc.run(cmd="path", single_stage=True) + dvc.run(cmd="path", name="train") assert popen.called_once() assert communicate.called_once_with()