mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-10 15:28:34 +01:00
refactor: rename utils.stage to utils.yaml (#3923)
This commit is contained in:
+12
-16
@@ -17,11 +17,7 @@ from dvc.stage.exceptions import (
|
||||
from dvc.stage.loader import SingleStageLoader, StageLoader
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.collections import apply_diff
|
||||
from dvc.utils.stage import (
|
||||
dump_stage_file,
|
||||
parse_stage,
|
||||
parse_stage_for_update,
|
||||
)
|
||||
from dvc.utils.yaml import dump_yaml, parse_yaml, parse_yaml_for_update
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -101,7 +97,7 @@ class FileMixin:
|
||||
|
||||
with self.repo.tree.open(self.path) as fd:
|
||||
stage_text = fd.read()
|
||||
d = parse_stage(stage_text, self.path)
|
||||
d = parse_yaml(stage_text, self.path)
|
||||
self.validate(d, self.relpath)
|
||||
return d, stage_text
|
||||
|
||||
@@ -149,7 +145,7 @@ class SingleStageFile(FileMixin):
|
||||
logger.debug(
|
||||
"Saving information to '{file}'.".format(file=relpath(self.path))
|
||||
)
|
||||
dump_stage_file(self.path, serialize.to_single_stage_file(stage))
|
||||
dump_yaml(self.path, serialize.to_single_stage_file(stage))
|
||||
self.repo.scm.track_file(self.relpath)
|
||||
|
||||
def remove_with_prompt(self, force=False):
|
||||
@@ -198,7 +194,7 @@ class PipelineFile(FileMixin):
|
||||
data = {}
|
||||
if self.exists():
|
||||
with open(self.path) as fd:
|
||||
data = parse_stage_for_update(fd.read(), self.path)
|
||||
data = parse_yaml_for_update(fd.read(), self.path)
|
||||
else:
|
||||
logger.info("Creating '%s'", self.relpath)
|
||||
open(self.path, "w+").close()
|
||||
@@ -214,7 +210,7 @@ class PipelineFile(FileMixin):
|
||||
logger.info(
|
||||
"Adding stage '%s' to '%s'", stage.name, self.relpath,
|
||||
)
|
||||
dump_stage_file(self.path, data)
|
||||
dump_yaml(self.path, data)
|
||||
self.repo.scm.track_file(self.relpath)
|
||||
|
||||
@property
|
||||
@@ -243,7 +239,7 @@ class PipelineFile(FileMixin):
|
||||
return
|
||||
|
||||
with open(self.path, "r") as f:
|
||||
d = parse_stage_for_update(f.read(), self.path)
|
||||
d = parse_yaml_for_update(f.read(), self.path)
|
||||
|
||||
self.validate(d, self.path)
|
||||
if stage.name not in d.get("stages", {}):
|
||||
@@ -251,7 +247,7 @@ class PipelineFile(FileMixin):
|
||||
|
||||
logger.debug("Removing '%s' from '%s'", stage.name, self.path)
|
||||
del d["stages"][stage.name]
|
||||
dump_stage_file(self.path, d)
|
||||
dump_yaml(self.path, d)
|
||||
|
||||
|
||||
class Lockfile(FileMixin):
|
||||
@@ -261,7 +257,7 @@ class Lockfile(FileMixin):
|
||||
if not self.exists():
|
||||
return {}
|
||||
with self.repo.tree.open(self.path) as fd:
|
||||
data = parse_stage(fd.read(), self.path)
|
||||
data = parse_yaml(fd.read(), self.path)
|
||||
try:
|
||||
self.validate(data, fname=self.relpath)
|
||||
except StageFileFormatError:
|
||||
@@ -279,14 +275,14 @@ class Lockfile(FileMixin):
|
||||
open(self.path, "w+").close()
|
||||
else:
|
||||
with self.repo.tree.open(self.path, "r") as fd:
|
||||
data = parse_stage_for_update(fd.read(), self.path)
|
||||
data = parse_yaml_for_update(fd.read(), self.path)
|
||||
modified = data.get(stage.name, {}) != stage_data.get(
|
||||
stage.name, {}
|
||||
)
|
||||
if modified:
|
||||
logger.info("Updating lock file '%s'", self.relpath)
|
||||
data.update(stage_data)
|
||||
dump_stage_file(self.path, data)
|
||||
dump_yaml(self.path, data)
|
||||
if modified:
|
||||
self.repo.scm.track_file(self.relpath)
|
||||
|
||||
@@ -295,7 +291,7 @@ class Lockfile(FileMixin):
|
||||
return
|
||||
|
||||
with open(self.path) as f:
|
||||
d = parse_stage_for_update(f.read(), self.path)
|
||||
d = parse_yaml_for_update(f.read(), self.path)
|
||||
self.validate(d, self.path)
|
||||
|
||||
if stage.name not in d:
|
||||
@@ -304,7 +300,7 @@ class Lockfile(FileMixin):
|
||||
logger.debug("Removing '%s' from '%s'", stage.name, self.path)
|
||||
del d[stage.name]
|
||||
|
||||
dump_stage_file(self.path, d)
|
||||
dump_yaml(self.path, d)
|
||||
|
||||
|
||||
class Dvcfile:
|
||||
|
||||
+2
-3
@@ -178,12 +178,11 @@ class NoMetricsError(DvcException):
|
||||
)
|
||||
|
||||
|
||||
class StageFileCorruptedError(DvcException):
|
||||
class YAMLFileCorruptedError(DvcException):
|
||||
def __init__(self, path):
|
||||
path = relpath(path)
|
||||
super().__init__(
|
||||
"unable to read DVC-file: {} "
|
||||
"YAML file structure is corrupted".format(path)
|
||||
f"unable to read: '{path}', YAML file structure is corrupted"
|
||||
)
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -10,7 +10,7 @@ from dvc.output import BaseOutput
|
||||
from dvc.stage.params import StageParams
|
||||
from dvc.stage.utils import resolve_wdir
|
||||
from dvc.utils.collections import apply_diff
|
||||
from dvc.utils.stage import parse_stage_for_update
|
||||
from dvc.utils.yaml import parse_yaml_for_update
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.stage import PipelineStage, Stage
|
||||
@@ -165,7 +165,7 @@ def to_single_stage_file(stage: "Stage"):
|
||||
# - apply changes to a returned structure
|
||||
# - serialize it
|
||||
if stage._stage_text is not None:
|
||||
saved_state = parse_stage_for_update(stage._stage_text, stage.path)
|
||||
saved_state = parse_yaml_for_update(stage._stage_text, stage.path)
|
||||
# Stage doesn't work with meta in any way, so .dumpd() doesn't
|
||||
# have it. We simply copy it over.
|
||||
if "meta" in saved_state:
|
||||
|
||||
+2
-2
@@ -11,7 +11,7 @@ from dvc.serialize import to_single_stage_lockfile
|
||||
from dvc.stage.loader import StageLoader
|
||||
from dvc.utils import dict_sha256, relpath
|
||||
from dvc.utils.fs import makedirs
|
||||
from dvc.utils.stage import dump_stage_file
|
||||
from dvc.utils.yaml import dump_yaml
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -154,7 +154,7 @@ class StageCache:
|
||||
path = self._get_cache_path(cache_key, cache_value)
|
||||
dpath = os.path.dirname(path)
|
||||
makedirs(dpath, exist_ok=True)
|
||||
dump_stage_file(path, cache)
|
||||
dump_yaml(path, cache)
|
||||
|
||||
def is_cached(self, stage):
|
||||
return bool(self._load(stage))
|
||||
|
||||
@@ -4,7 +4,7 @@ import yaml
|
||||
from ruamel.yaml import YAML
|
||||
from ruamel.yaml.error import YAMLError
|
||||
|
||||
from dvc.exceptions import StageFileCorruptedError
|
||||
from dvc.exceptions import YAMLFileCorruptedError
|
||||
|
||||
try:
|
||||
from yaml import CSafeLoader as SafeLoader
|
||||
@@ -12,35 +12,35 @@ except ImportError:
|
||||
from yaml import SafeLoader
|
||||
|
||||
|
||||
def load_stage_file(path):
|
||||
def load_yaml(path):
|
||||
with open(path, encoding="utf-8") as fd:
|
||||
return parse_stage(fd.read(), path)
|
||||
return parse_yaml(fd.read(), path)
|
||||
|
||||
|
||||
def parse_stage(text, path):
|
||||
def parse_yaml(text, path):
|
||||
try:
|
||||
return yaml.load(text, Loader=SafeLoader) or {}
|
||||
except yaml.error.YAMLError as exc:
|
||||
raise StageFileCorruptedError(path) from exc
|
||||
raise YAMLFileCorruptedError(path) from exc
|
||||
|
||||
|
||||
def parse_stage_for_update(text, path):
|
||||
def parse_yaml_for_update(text, path):
|
||||
"""Parses text into Python structure.
|
||||
|
||||
Unlike `parse_stage()` this returns ordered dicts, values have special
|
||||
Unlike `parse_yaml()` this returns ordered dicts, values have special
|
||||
attributes to store comments and line breaks. This allows us to preserve
|
||||
all of those upon dump.
|
||||
|
||||
This one is, however, several times slower than simple `parse_stage()`.
|
||||
This one is, however, several times slower than simple `parse_yaml()`.
|
||||
"""
|
||||
try:
|
||||
yaml = YAML()
|
||||
return yaml.load(text) or {}
|
||||
except YAMLError as exc:
|
||||
raise StageFileCorruptedError(path) from exc
|
||||
raise YAMLFileCorruptedError(path) from exc
|
||||
|
||||
|
||||
def dump_stage_file(path, data):
|
||||
def dump_yaml(path, data):
|
||||
with open(path, "w", encoding="utf-8") as fd:
|
||||
yaml = YAML()
|
||||
yaml.default_flow_style = False
|
||||
@@ -17,7 +17,7 @@ from dvc.exceptions import (
|
||||
OutputDuplicationError,
|
||||
OverlappingOutputPathsError,
|
||||
RecursiveAddingWhileUsingFilename,
|
||||
StageFileCorruptedError,
|
||||
YAMLFileCorruptedError,
|
||||
)
|
||||
from dvc.main import main
|
||||
from dvc.output.base import OutputAlreadyTrackedError, OutputIsStageFileError
|
||||
@@ -27,7 +27,7 @@ from dvc.stage import Stage
|
||||
from dvc.system import System
|
||||
from dvc.utils import LARGE_DIR_SIZE, file_md5, relpath
|
||||
from dvc.utils.fs import path_isin
|
||||
from dvc.utils.stage import load_stage_file
|
||||
from dvc.utils.yaml import load_yaml
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.utils import get_gitignore_content
|
||||
|
||||
@@ -46,7 +46,7 @@ def test_add(tmp_dir, dvc):
|
||||
assert stage.outs[0].info["md5"] == md5
|
||||
assert stage.md5 is None
|
||||
|
||||
assert load_stage_file("foo.dvc") == {
|
||||
assert load_yaml("foo.dvc") == {
|
||||
"outs": [{"md5": "acbd18db4cc2f85cedef654fccc4a4d8", "path": "foo"}],
|
||||
}
|
||||
|
||||
@@ -220,14 +220,14 @@ class TestAddLocalRemoteFile(TestDvc):
|
||||
ret = main(["add", foo])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
d = load_stage_file("foo.dvc")
|
||||
d = load_yaml("foo.dvc")
|
||||
self.assertEqual(d["outs"][0]["path"], foo)
|
||||
|
||||
bar = os.path.join(cwd, self.BAR)
|
||||
ret = main(["add", bar])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
d = load_stage_file("bar.dvc")
|
||||
d = load_yaml("bar.dvc")
|
||||
self.assertEqual(d["outs"][0]["path"], self.BAR)
|
||||
|
||||
|
||||
@@ -371,7 +371,7 @@ class SymlinkAddTestBase(TestDvc):
|
||||
stage_file = self.data_file_name + DVC_FILE_SUFFIX
|
||||
self.assertTrue(os.path.exists(stage_file))
|
||||
|
||||
d = load_stage_file(stage_file)
|
||||
d = load_yaml(stage_file)
|
||||
relative_data_path = posixpath.join(
|
||||
self.link_name, self.data_file_name
|
||||
)
|
||||
@@ -436,8 +436,7 @@ class TestShouldThrowProperExceptionOnCorruptedStageFile(TestDvc):
|
||||
assert 1 == ret
|
||||
|
||||
expected_error = (
|
||||
"unable to read DVC-file: {} "
|
||||
"YAML file structure is corrupted".format(foo_stage)
|
||||
f"unable to read: '{foo_stage}', YAML file structure is corrupted"
|
||||
)
|
||||
|
||||
assert expected_error in self._caplog.text
|
||||
@@ -478,7 +477,7 @@ def test_failed_add_cleanup(tmp_dir, scm, dvc):
|
||||
dvc.add("foo")
|
||||
tmp_dir.gen("foo.dvc", "- broken\nyaml")
|
||||
|
||||
with pytest.raises(StageFileCorruptedError):
|
||||
with pytest.raises(YAMLFileCorruptedError):
|
||||
dvc.add("bar")
|
||||
|
||||
assert not os.path.exists("bar.dvc")
|
||||
|
||||
@@ -24,7 +24,7 @@ from dvc.stage.exceptions import StageFileDoesNotExistError
|
||||
from dvc.system import System
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import walk_files
|
||||
from dvc.utils.stage import dump_stage_file, load_stage_file
|
||||
from dvc.utils.yaml import dump_yaml, load_yaml
|
||||
from tests.basic_env import TestDvc, TestDvcGit
|
||||
from tests.func.test_repro import TestRepro
|
||||
from tests.remotes import S3
|
||||
@@ -221,7 +221,7 @@ class TestCheckoutSelectiveRemove(CheckoutBase):
|
||||
self.assertEqual(0, ret)
|
||||
|
||||
stage_path = self.DATA_DIR + DVC_FILE_SUFFIX
|
||||
stage = load_stage_file(stage_path)
|
||||
stage = load_yaml(stage_path)
|
||||
staged_files = self.outs_info(stage)
|
||||
|
||||
# move instead of remove, to lock inode assigned to stage_files[0].path
|
||||
@@ -304,10 +304,10 @@ class TestGitIgnoreWhenCheckout(CheckoutBase):
|
||||
|
||||
class TestCheckoutMissingMd5InStageFile(TestRepro):
|
||||
def test(self):
|
||||
d = load_stage_file(self.file1_stage)
|
||||
d = load_yaml(self.file1_stage)
|
||||
del d[Stage.PARAM_OUTS][0][LocalRemote.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_DEPS][0][LocalRemote.PARAM_CHECKSUM]
|
||||
dump_stage_file(self.file1_stage, d)
|
||||
dump_yaml(self.file1_stage, d)
|
||||
|
||||
with pytest.raises(CheckoutError):
|
||||
self.dvc.checkout(force=True)
|
||||
|
||||
@@ -2,7 +2,7 @@ import pytest
|
||||
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
from dvc.stage.exceptions import StageCommitError
|
||||
from dvc.utils.stage import dump_stage_file, load_stage_file
|
||||
from dvc.utils.yaml import dump_yaml, load_yaml
|
||||
|
||||
|
||||
def test_commit_recursive(tmp_dir, dvc):
|
||||
@@ -63,15 +63,15 @@ def test_commit_changed_md5(tmp_dir, dvc):
|
||||
tmp_dir.gen({"file": "file content"})
|
||||
(stage,) = dvc.add("file", no_commit=True)
|
||||
|
||||
stage_file_content = load_stage_file(stage.path)
|
||||
stage_file_content = load_yaml(stage.path)
|
||||
stage_file_content["md5"] = "1111111111"
|
||||
dump_stage_file(stage.path, stage_file_content)
|
||||
dump_yaml(stage.path, stage_file_content)
|
||||
|
||||
with pytest.raises(StageCommitError):
|
||||
dvc.commit(stage.path)
|
||||
|
||||
dvc.commit(stage.path, force=True)
|
||||
assert "md5" not in load_stage_file(stage.path)
|
||||
assert "md5" not in load_yaml(stage.path)
|
||||
|
||||
|
||||
def test_commit_no_exec(tmp_dir, dvc):
|
||||
|
||||
@@ -27,7 +27,7 @@ from dvc.remote.base import STATUS_DELETED, STATUS_NEW, STATUS_OK
|
||||
from dvc.stage.exceptions import StageNotFound
|
||||
from dvc.utils import file_md5
|
||||
from dvc.utils.fs import remove
|
||||
from dvc.utils.stage import dump_stage_file, load_stage_file
|
||||
from dvc.utils.yaml import dump_yaml, load_yaml
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.remotes import (
|
||||
GCP,
|
||||
@@ -523,9 +523,9 @@ class TestWarnOnOutdatedStage(TestDvc):
|
||||
self.main(["push"])
|
||||
|
||||
stage_file_path = stage.relpath
|
||||
content = load_stage_file(stage_file_path)
|
||||
content = load_yaml(stage_file_path)
|
||||
del content["outs"][0]["md5"]
|
||||
dump_stage_file(stage_file_path, content)
|
||||
dump_yaml(stage_file_path, content)
|
||||
|
||||
with self._caplog.at_level(logging.WARNING, logger="dvc"):
|
||||
self._caplog.clear()
|
||||
|
||||
@@ -8,7 +8,7 @@ from dvc.stage.exceptions import (
|
||||
StageFileFormatError,
|
||||
)
|
||||
from dvc.stage.loader import StageNotFound
|
||||
from dvc.utils.stage import dump_stage_file
|
||||
from dvc.utils.yaml import dump_yaml
|
||||
|
||||
|
||||
def test_run_load_one_for_multistage(tmp_dir, dvc):
|
||||
@@ -244,14 +244,14 @@ def test_remove_stage_on_lockfile_format_error(tmp_dir, dvc, run_copy):
|
||||
lock_data = lock_file.load()
|
||||
lock_data["gibberish"] = True
|
||||
data["gibberish"] = True
|
||||
dump_stage_file(lock_file.relpath, lock_data)
|
||||
dump_yaml(lock_file.relpath, lock_data)
|
||||
with pytest.raises(StageFileFormatError):
|
||||
dvc_file.remove_stage(stage)
|
||||
|
||||
lock_file.remove()
|
||||
dvc_file.dump(stage)
|
||||
|
||||
dump_stage_file(dvc_file.relpath, data)
|
||||
dump_yaml(dvc_file.relpath, data)
|
||||
with pytest.raises(StageFileFormatError):
|
||||
dvc_file.remove_stage(stage)
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import yaml
|
||||
from dvc.dvcfile import PIPELINE_LOCK
|
||||
from dvc.serialize import get_params_deps
|
||||
from dvc.utils.fs import remove
|
||||
from dvc.utils.stage import parse_stage_for_update
|
||||
from dvc.utils.yaml import parse_yaml_for_update
|
||||
from tests.func.test_run_multistage import supported_params
|
||||
|
||||
FS_STRUCTURE = {
|
||||
@@ -20,7 +20,7 @@ FS_STRUCTURE = {
|
||||
|
||||
def read_lock_file(file=PIPELINE_LOCK):
|
||||
with open(file) as f:
|
||||
data = parse_stage_for_update(f.read(), file)
|
||||
data = parse_yaml_for_update(f.read(), file)
|
||||
assert isinstance(data, OrderedDict)
|
||||
return data
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
from dvc.dvcfile import DVC_FILE_SUFFIX
|
||||
from dvc.exceptions import DvcException, MoveNotDataSourceError
|
||||
from dvc.main import main
|
||||
from dvc.utils.stage import load_stage_file
|
||||
from dvc.utils.yaml import load_yaml
|
||||
from tests.basic_env import TestDvc, TestDvcGit
|
||||
from tests.func.test_repro import TestRepro
|
||||
from tests.utils import cd
|
||||
@@ -117,7 +117,7 @@ class TestMoveFileToDirectoryWithoutSpecifiedTargetName(TestDvc):
|
||||
self.assertTrue(os.path.exists(target_foo_path))
|
||||
self.assertTrue(os.path.exists(target_foo_stage_file_path))
|
||||
|
||||
new_stage = load_stage_file(target_foo_stage_file_path)
|
||||
new_stage = load_yaml(target_foo_stage_file_path)
|
||||
self.assertEqual(self.FOO, new_stage["outs"][0]["path"])
|
||||
|
||||
|
||||
@@ -170,7 +170,7 @@ class TestMoveFileBetweenDirectories(TestDvc):
|
||||
self.assertTrue(os.path.exists(new_data_path))
|
||||
self.assertTrue(os.path.exists(new_data_stage_file))
|
||||
|
||||
new_stage_file = load_stage_file(new_data_stage_file)
|
||||
new_stage_file = load_yaml(new_data_stage_file)
|
||||
self.assertEqual(
|
||||
os.path.basename(self.DATA), new_stage_file["outs"][0]["path"]
|
||||
)
|
||||
|
||||
@@ -33,7 +33,7 @@ from dvc.stage.exceptions import StageFileDoesNotExistError
|
||||
from dvc.system import System
|
||||
from dvc.utils import file_md5, relpath
|
||||
from dvc.utils.fs import remove
|
||||
from dvc.utils.stage import dump_stage_file, load_stage_file
|
||||
from dvc.utils.yaml import dump_yaml, load_yaml
|
||||
from tests.basic_env import TestDvc
|
||||
from tests.remotes import (
|
||||
GCP,
|
||||
@@ -108,7 +108,7 @@ class TestReproCyclicGraph(SingleStageRun, TestDvc):
|
||||
"deps": [{"path": "baz.txt"}],
|
||||
"outs": [{"path": self.FOO}],
|
||||
}
|
||||
dump_stage_file("cycle.dvc", stage_dump)
|
||||
dump_yaml("cycle.dvc", stage_dump)
|
||||
|
||||
with self.assertRaises(CyclicGraphError):
|
||||
self.dvc.reproduce("cycle.dvc")
|
||||
@@ -149,7 +149,7 @@ class TestReproWorkingDirectoryAsOutput(TestDvc):
|
||||
"cmd": f"echo something > {output}",
|
||||
"outs": [{"path": output}],
|
||||
}
|
||||
dump_stage_file(faulty_stage_path, stage_dump)
|
||||
dump_yaml(faulty_stage_path, stage_dump)
|
||||
|
||||
with self.assertRaises(StagePathAsOutputError):
|
||||
self.dvc.reproduce(faulty_stage_path)
|
||||
@@ -188,7 +188,7 @@ class TestReproWorkingDirectoryAsOutput(TestDvc):
|
||||
"cmd": f"echo something > {output}",
|
||||
"outs": [{"path": output}],
|
||||
}
|
||||
dump_stage_file(error_stage_path, stage_dump)
|
||||
dump_yaml(error_stage_path, stage_dump)
|
||||
|
||||
# NOTE: os.walk() walks in a sorted order and we need dir2 subdirs to
|
||||
# be processed before dir1 to load error.dvc first.
|
||||
@@ -220,7 +220,7 @@ class TestReproWorkingDirectoryAsOutput(TestDvc):
|
||||
stage = os.path.join("something-1", "a.dvc")
|
||||
|
||||
stage_dump = {"cmd": "echo a > a", "outs": [{"path": "a"}]}
|
||||
dump_stage_file(stage, stage_dump)
|
||||
dump_yaml(stage, stage_dump)
|
||||
|
||||
try:
|
||||
self.dvc.reproduce(stage)
|
||||
@@ -775,10 +775,10 @@ class TestReproChangedDirData(SingleStageRun, TestDvc):
|
||||
|
||||
class TestReproMissingMd5InStageFile(TestRepro):
|
||||
def test(self):
|
||||
d = load_stage_file(self.file1_stage)
|
||||
d = load_yaml(self.file1_stage)
|
||||
del d[Stage.PARAM_OUTS][0][LocalRemote.PARAM_CHECKSUM]
|
||||
del d[Stage.PARAM_DEPS][0][LocalRemote.PARAM_CHECKSUM]
|
||||
dump_stage_file(self.file1_stage, d)
|
||||
dump_yaml(self.file1_stage, d)
|
||||
|
||||
stages = self.dvc.reproduce(self.file1_stage)
|
||||
self.assertEqual(len(stages), 1)
|
||||
@@ -1785,7 +1785,7 @@ def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy, mocker):
|
||||
|
||||
data = SingleStageFile(dvc, stage.path)._load()[0]
|
||||
data["cmd"] = " ".join(stage.cmd.split()) # change cmd spacing by two
|
||||
dump_stage_file(stage.path, data)
|
||||
dump_yaml(stage.path, data)
|
||||
|
||||
assert dvc.status([stage.addressing]) == {
|
||||
stage.addressing: ["changed checksum"]
|
||||
|
||||
@@ -10,7 +10,7 @@ from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
from dvc.exceptions import CyclicGraphError
|
||||
from dvc.main import main
|
||||
from dvc.stage import PipelineStage
|
||||
from dvc.utils.stage import dump_stage_file, parse_stage
|
||||
from dvc.utils.yaml import dump_yaml, parse_yaml
|
||||
from tests.func import test_repro
|
||||
|
||||
COPY_SCRIPT_FORMAT = dedent(
|
||||
@@ -343,7 +343,7 @@ def test_repro_when_new_deps_is_added_in_dvcfile(tmp_dir, dvc, run_copy):
|
||||
dvcfile = Dvcfile(dvc, stage.path)
|
||||
data, _ = dvcfile._load()
|
||||
data["stages"]["copy-file"]["deps"] += ["copy.py"]
|
||||
dump_stage_file(stage.path, data)
|
||||
dump_yaml(stage.path, data)
|
||||
|
||||
assert dvc.reproduce(target)[0] == stage
|
||||
|
||||
@@ -365,7 +365,7 @@ def test_repro_when_new_outs_is_added_in_dvcfile(tmp_dir, dvc):
|
||||
dvcfile = Dvcfile(dvc, stage.path)
|
||||
data, _ = dvcfile._load()
|
||||
data["stages"]["copy-file"]["outs"] = ["foobar"]
|
||||
dump_stage_file(stage.path, data)
|
||||
dump_yaml(stage.path, data)
|
||||
|
||||
assert dvc.reproduce(target)[0] == stage
|
||||
|
||||
@@ -392,7 +392,7 @@ def test_repro_when_new_deps_is_moved(tmp_dir, dvc):
|
||||
dvcfile = Dvcfile(dvc, stage.path)
|
||||
data, _ = dvcfile._load()
|
||||
data["stages"]["copy-file"]["deps"] = ["bar"]
|
||||
dump_stage_file(stage.path, data)
|
||||
dump_yaml(stage.path, data)
|
||||
|
||||
assert dvc.reproduce(target)[0] == stage
|
||||
|
||||
@@ -402,7 +402,7 @@ def test_repro_when_new_out_overlaps_others_stage_outs(tmp_dir, dvc):
|
||||
|
||||
tmp_dir.gen({"dir": {"file1": "file1"}, "foo": "foo"})
|
||||
dvc.add("dir")
|
||||
dump_stage_file(
|
||||
dump_yaml(
|
||||
PIPELINE_FILE,
|
||||
{
|
||||
"stages": {
|
||||
@@ -423,7 +423,7 @@ def test_repro_when_new_deps_added_does_not_exist(tmp_dir, dvc):
|
||||
|
||||
tmp_dir.gen("copy.py", COPY_SCRIPT)
|
||||
tmp_dir.gen("foo", "foo")
|
||||
dump_stage_file(
|
||||
dump_yaml(
|
||||
PIPELINE_FILE,
|
||||
{
|
||||
"stages": {
|
||||
@@ -444,7 +444,7 @@ def test_repro_when_new_outs_added_does_not_exist(tmp_dir, dvc):
|
||||
|
||||
tmp_dir.gen("copy.py", COPY_SCRIPT)
|
||||
tmp_dir.gen("foo", "foo")
|
||||
dump_stage_file(
|
||||
dump_yaml(
|
||||
PIPELINE_FILE,
|
||||
{
|
||||
"stages": {
|
||||
@@ -463,7 +463,7 @@ def test_repro_when_new_outs_added_does_not_exist(tmp_dir, dvc):
|
||||
def test_repro_when_lockfile_gets_deleted(tmp_dir, dvc):
|
||||
tmp_dir.gen("copy.py", COPY_SCRIPT)
|
||||
tmp_dir.gen("foo", "foo")
|
||||
dump_stage_file(
|
||||
dump_yaml(
|
||||
PIPELINE_FILE,
|
||||
{
|
||||
"stages": {
|
||||
@@ -495,13 +495,13 @@ def test_cyclic_graph_error(tmp_dir, dvc, run_copy):
|
||||
run_copy("baz", "foobar", name="copy-baz-foobar")
|
||||
|
||||
with open(PIPELINE_FILE) as f:
|
||||
data = parse_stage(f.read(), PIPELINE_FILE)
|
||||
data = parse_yaml(f.read(), PIPELINE_FILE)
|
||||
data["stages"]["copy-baz-foo"] = {
|
||||
"cmd": "echo baz > foo",
|
||||
"deps": ["baz"],
|
||||
"outs": ["foo"],
|
||||
}
|
||||
dump_stage_file(PIPELINE_FILE, data)
|
||||
dump_yaml(PIPELINE_FILE, data)
|
||||
with pytest.raises(CyclicGraphError):
|
||||
dvc.reproduce(":copy-baz-foo")
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ from dvc.stage.exceptions import (
|
||||
)
|
||||
from dvc.system import System
|
||||
from dvc.utils import file_md5
|
||||
from dvc.utils.stage import load_stage_file
|
||||
from dvc.utils.yaml import load_yaml
|
||||
from tests.basic_env import TestDvc, TestDvcGit
|
||||
|
||||
|
||||
@@ -629,13 +629,13 @@ class TestCmdRunWorkingDirectory(TestDvc):
|
||||
wdir=".",
|
||||
single_stage=True,
|
||||
)
|
||||
d = load_stage_file(stage.relpath)
|
||||
d = load_yaml(stage.relpath)
|
||||
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
|
||||
|
||||
stage = self.dvc.run(
|
||||
cmd=f"echo test > {self.BAR}", outs=[self.BAR], single_stage=True,
|
||||
)
|
||||
d = load_stage_file(stage.relpath)
|
||||
d = load_yaml(stage.relpath)
|
||||
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
|
||||
|
||||
def test_fname_changes_path_and_wdir(self):
|
||||
@@ -655,7 +655,7 @@ class TestCmdRunWorkingDirectory(TestDvc):
|
||||
)
|
||||
|
||||
# Check that it is dumped properly (relative to fname)
|
||||
d = load_stage_file(stage.relpath)
|
||||
d = load_yaml(stage.relpath)
|
||||
self.assertEqual(d[Stage.PARAM_WDIR], "..")
|
||||
|
||||
|
||||
@@ -775,7 +775,7 @@ class TestRunPersist(TestDvc):
|
||||
self.assertEqual(0, ret)
|
||||
|
||||
def stage_should_contain_persist_flag(self, stage_file):
|
||||
stage_file_content = load_stage_file(stage_file)
|
||||
stage_file_content = load_yaml(stage_file)
|
||||
self.assertEqual(
|
||||
True, stage_file_content["outs"][0][BaseOutput.PARAM_PERSIST]
|
||||
)
|
||||
|
||||
@@ -10,7 +10,7 @@ from dvc.remote.local import LocalRemote
|
||||
from dvc.repo import Repo
|
||||
from dvc.stage import PipelineStage, Stage
|
||||
from dvc.stage.exceptions import StageFileFormatError
|
||||
from dvc.utils.stage import dump_stage_file, load_stage_file
|
||||
from dvc.utils.yaml import dump_yaml, load_yaml
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
|
||||
@@ -74,12 +74,12 @@ class TestReload(TestDvc):
|
||||
stage = stages[0]
|
||||
self.assertTrue(stage is not None)
|
||||
|
||||
d = load_stage_file(stage.relpath)
|
||||
d = load_yaml(stage.relpath)
|
||||
|
||||
# NOTE: checking that reloaded stage didn't change its checksum
|
||||
md5 = "11111111111111111111111111111111"
|
||||
d[stage.PARAM_MD5] = md5
|
||||
dump_stage_file(stage.relpath, d)
|
||||
dump_yaml(stage.relpath, d)
|
||||
|
||||
dvcfile = SingleStageFile(self.dvc, stage.relpath)
|
||||
stage = dvcfile.stage
|
||||
@@ -87,7 +87,7 @@ class TestReload(TestDvc):
|
||||
self.assertTrue(stage is not None)
|
||||
dvcfile.dump(stage)
|
||||
|
||||
d = load_stage_file(stage.relpath)
|
||||
d = load_yaml(stage.relpath)
|
||||
self.assertEqual(d[stage.PARAM_MD5], md5)
|
||||
|
||||
|
||||
@@ -103,7 +103,7 @@ class TestDefaultWorkingDirectory(TestDvc):
|
||||
d = stage.dumpd()
|
||||
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
|
||||
|
||||
d = load_stage_file(stage.relpath)
|
||||
d = load_yaml(stage.relpath)
|
||||
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
|
||||
|
||||
with self.dvc.lock, self.dvc.state:
|
||||
@@ -167,16 +167,16 @@ def test_meta_is_preserved(tmp_dir, dvc):
|
||||
(stage,) = tmp_dir.dvc_gen("foo", "foo content")
|
||||
|
||||
# Add meta to DVC-file
|
||||
data = load_stage_file(stage.path)
|
||||
data = load_yaml(stage.path)
|
||||
data["meta"] = {"custom_key": 42}
|
||||
dump_stage_file(stage.path, data)
|
||||
dump_yaml(stage.path, data)
|
||||
|
||||
# Loading and dumping to test that it works and meta is retained
|
||||
dvcfile = SingleStageFile(dvc, stage.path)
|
||||
new_stage = dvcfile.stage
|
||||
dvcfile.dump(new_stage)
|
||||
|
||||
new_data = load_stage_file(stage.path)
|
||||
new_data = load_yaml(stage.path)
|
||||
assert new_data["meta"] == data["meta"]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user