From e31364979b8ddf202202d0ed8bd26ba9b6d1c0a1 Mon Sep 17 00:00:00 2001 From: Saugat Pachhai Date: Tue, 18 Aug 2020 07:16:16 +0000 Subject: [PATCH] Move toml/yaml serialization to dvc.utils.serialize (#4411) --- dvc/dvcfile.py | 2 +- dvc/repo/experiments/__init__.py | 8 ++++++-- dvc/stage/cache.py | 2 +- dvc/stage/serialize.py | 2 +- dvc/utils/serialize/__init__.py | 2 ++ dvc/utils/{toml.py => serialize/_toml.py} | 0 dvc/utils/{yaml.py => serialize/_yaml.py} | 0 tests/func/test_add.py | 2 +- tests/func/test_checkout.py | 2 +- tests/func/test_commit.py | 2 +- tests/func/test_data_cloud.py | 2 +- tests/func/test_dvcfile.py | 2 +- tests/func/test_lockfile.py | 2 +- tests/func/test_move.py | 2 +- tests/func/test_repro.py | 2 +- tests/func/test_repro_multistage.py | 2 +- tests/func/test_run_multistage.py | 2 +- tests/func/test_run_single_stage.py | 2 +- tests/func/test_stage.py | 2 +- tests/unit/dependency/test_params.py | 2 +- tests/unit/test_dvcfile.py | 2 +- 21 files changed, 25 insertions(+), 19 deletions(-) create mode 100644 dvc/utils/serialize/__init__.py rename dvc/utils/{toml.py => serialize/_toml.py} (100%) rename dvc/utils/{yaml.py => serialize/_yaml.py} (100%) diff --git a/dvc/dvcfile.py b/dvc/dvcfile.py index 5b0f2f38c..697b8fc42 100644 --- a/dvc/dvcfile.py +++ b/dvc/dvcfile.py @@ -16,7 +16,7 @@ from dvc.stage.exceptions import ( from dvc.stage.loader import SingleStageLoader, StageLoader from dvc.utils import relpath from dvc.utils.collections import apply_diff -from dvc.utils.yaml import dump_yaml, parse_yaml, parse_yaml_for_update +from dvc.utils.serialize import dump_yaml, parse_yaml, parse_yaml_for_update logger = logging.getLogger(__name__) diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index 4f6ec5aec..3275ab12b 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -189,8 +189,12 @@ class Experiments: def _update_params(self, params: dict): """Update experiment params files with the specified values.""" - from dvc.utils.toml import dump_toml, parse_toml_for_update - from dvc.utils.yaml import dump_yaml, parse_yaml_for_update + from dvc.utils.serialize import ( + dump_toml, + dump_yaml, + parse_toml_for_update, + parse_yaml_for_update, + ) logger.debug("Using experiment params '%s'", params) diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index d77a78e3b..9eceb465e 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -10,7 +10,7 @@ from dvc.cache.local import _log_exceptions from dvc.schema import COMPILED_LOCK_FILE_STAGE_SCHEMA from dvc.utils import dict_sha256, relpath from dvc.utils.fs import makedirs -from dvc.utils.yaml import dump_yaml +from dvc.utils.serialize import dump_yaml from .loader import StageLoader from .serialize import to_single_stage_lockfile diff --git a/dvc/stage/serialize.py b/dvc/stage/serialize.py index 44d3c4f3b..3b02fdbc4 100644 --- a/dvc/stage/serialize.py +++ b/dvc/stage/serialize.py @@ -8,7 +8,7 @@ from funcy import post_processing from dvc.dependency import ParamsDependency from dvc.output import BaseOutput from dvc.utils.collections import apply_diff -from dvc.utils.yaml import parse_yaml_for_update +from dvc.utils.serialize import parse_yaml_for_update from .params import StageParams from .utils import resolve_wdir, split_params_deps diff --git a/dvc/utils/serialize/__init__.py b/dvc/utils/serialize/__init__.py new file mode 100644 index 000000000..ed51128cc --- /dev/null +++ b/dvc/utils/serialize/__init__.py @@ -0,0 +1,2 @@ +from ._toml import * # noqa, pylint: disable=wildcard-import +from ._yaml import * # noqa, pylint: disable=wildcard-import diff --git a/dvc/utils/toml.py b/dvc/utils/serialize/_toml.py similarity index 100% rename from dvc/utils/toml.py rename to dvc/utils/serialize/_toml.py diff --git a/dvc/utils/yaml.py b/dvc/utils/serialize/_yaml.py similarity index 100% rename from dvc/utils/yaml.py rename to dvc/utils/serialize/_yaml.py diff --git a/tests/func/test_add.py b/tests/func/test_add.py index b712d744c..50bda72de 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -27,7 +27,7 @@ from dvc.system import System from dvc.tree.local import LocalTree from dvc.utils import LARGE_DIR_SIZE, file_md5, relpath from dvc.utils.fs import path_isin -from dvc.utils.yaml import load_yaml +from dvc.utils.serialize import load_yaml from tests.basic_env import TestDvc from tests.utils import get_gitignore_content diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index 04e73483a..4c65e932a 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -26,7 +26,7 @@ from dvc.tree.local import LocalTree from dvc.tree.s3 import S3Tree from dvc.utils import relpath from dvc.utils.fs import walk_files -from dvc.utils.yaml import dump_yaml, load_yaml +from dvc.utils.serialize import dump_yaml, load_yaml from tests.basic_env import TestDvc, TestDvcGit from tests.func.test_repro import TestRepro from tests.remotes import S3 diff --git a/tests/func/test_commit.py b/tests/func/test_commit.py index 958a29c53..3af50484c 100644 --- a/tests/func/test_commit.py +++ b/tests/func/test_commit.py @@ -6,7 +6,7 @@ from dvc.dependency.base import DependencyDoesNotExistError from dvc.dvcfile import PIPELINE_FILE from dvc.output.base import OutputDoesNotExistError from dvc.stage.exceptions import StageCommitError -from dvc.utils.yaml import dump_yaml, load_yaml +from dvc.utils.serialize import dump_yaml, load_yaml def test_commit_recursive(tmp_dir, dvc): diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 3023ef377..72809dba8 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -17,7 +17,7 @@ from dvc.main import main from dvc.stage.exceptions import StageNotFound from dvc.tree.local import LocalTree from dvc.utils.fs import move, remove -from dvc.utils.yaml import dump_yaml, load_yaml +from dvc.utils.serialize import dump_yaml, load_yaml from .test_api import all_clouds diff --git a/tests/func/test_dvcfile.py b/tests/func/test_dvcfile.py index 5ba42a093..2b029c20b 100644 --- a/tests/func/test_dvcfile.py +++ b/tests/func/test_dvcfile.py @@ -9,7 +9,7 @@ from dvc.stage.exceptions import ( StageFileFormatError, ) from dvc.stage.loader import StageNotFound -from dvc.utils.yaml import dump_yaml +from dvc.utils.serialize import dump_yaml def test_run_load_one_for_multistage(tmp_dir, dvc): diff --git a/tests/func/test_lockfile.py b/tests/func/test_lockfile.py index 2d2231368..b9a25ff18 100644 --- a/tests/func/test_lockfile.py +++ b/tests/func/test_lockfile.py @@ -6,7 +6,7 @@ import yaml from dvc.dvcfile import PIPELINE_LOCK from dvc.stage.utils import split_params_deps from dvc.utils.fs import remove -from dvc.utils.yaml import parse_yaml_for_update +from dvc.utils.serialize import parse_yaml_for_update from tests.func.test_run_multistage import supported_params FS_STRUCTURE = { diff --git a/tests/func/test_move.py b/tests/func/test_move.py index 61280c082..1059a3e05 100644 --- a/tests/func/test_move.py +++ b/tests/func/test_move.py @@ -3,7 +3,7 @@ import os from dvc.dvcfile import DVC_FILE_SUFFIX from dvc.exceptions import DvcException, MoveNotDataSourceError from dvc.main import main -from dvc.utils.yaml import load_yaml +from dvc.utils.serialize import load_yaml from tests.basic_env import TestDvc, TestDvcGit from tests.func.test_repro import TestRepro from tests.utils import cd diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index af0db3808..64744eac2 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -21,7 +21,7 @@ from dvc.system import System from dvc.tree.local import LocalTree from dvc.utils import file_md5, relpath from dvc.utils.fs import remove -from dvc.utils.yaml import dump_yaml, load_yaml +from dvc.utils.serialize import dump_yaml, load_yaml from tests.basic_env import TestDvc diff --git a/tests/func/test_repro_multistage.py b/tests/func/test_repro_multistage.py index c6875f270..979e06383 100644 --- a/tests/func/test_repro_multistage.py +++ b/tests/func/test_repro_multistage.py @@ -10,7 +10,7 @@ from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK from dvc.exceptions import CyclicGraphError from dvc.main import main from dvc.stage import PipelineStage -from dvc.utils.yaml import dump_yaml, parse_yaml +from dvc.utils.serialize import dump_yaml, parse_yaml from tests.func import test_repro COPY_SCRIPT_FORMAT = dedent( diff --git a/tests/func/test_run_multistage.py b/tests/func/test_run_multistage.py index 8be4c3a04..29799e99b 100644 --- a/tests/func/test_run_multistage.py +++ b/tests/func/test_run_multistage.py @@ -7,7 +7,7 @@ import yaml from dvc.exceptions import InvalidArgumentError from dvc.repo import Repo from dvc.stage.exceptions import DuplicateStageName, InvalidStageName -from dvc.utils.yaml import parse_yaml_for_update +from dvc.utils.serialize import parse_yaml_for_update def test_run_with_name(tmp_dir, dvc, run_copy): diff --git a/tests/func/test_run_single_stage.py b/tests/func/test_run_single_stage.py index 221fee116..b43e91437 100644 --- a/tests/func/test_run_single_stage.py +++ b/tests/func/test_run_single_stage.py @@ -32,7 +32,7 @@ from dvc.stage.exceptions import ( ) from dvc.system import System from dvc.utils import file_md5 -from dvc.utils.yaml import load_yaml +from dvc.utils.serialize import load_yaml from tests.basic_env import TestDvc, TestDvcGit diff --git a/tests/func/test_stage.py b/tests/func/test_stage.py index a87446cdb..e3914aa6d 100644 --- a/tests/func/test_stage.py +++ b/tests/func/test_stage.py @@ -10,7 +10,7 @@ from dvc.repo import Repo from dvc.stage import PipelineStage, Stage from dvc.stage.exceptions import StageFileFormatError from dvc.tree.local import LocalTree -from dvc.utils.yaml import dump_yaml, load_yaml +from dvc.utils.serialize import dump_yaml, load_yaml from tests.basic_env import TestDvc diff --git a/tests/unit/dependency/test_params.py b/tests/unit/dependency/test_params.py index d8992d747..3dacc1691 100644 --- a/tests/unit/dependency/test_params.py +++ b/tests/unit/dependency/test_params.py @@ -5,7 +5,7 @@ import yaml from dvc.dependency import ParamsDependency, loadd_from, loads_params from dvc.dependency.param import BadParamFileError, MissingParamsError from dvc.stage import Stage -from dvc.utils.yaml import load_yaml +from dvc.utils.serialize import load_yaml PARAMS = { "foo": 1, diff --git a/tests/unit/test_dvcfile.py b/tests/unit/test_dvcfile.py index 3668ee209..abb736f8f 100644 --- a/tests/unit/test_dvcfile.py +++ b/tests/unit/test_dvcfile.py @@ -13,7 +13,7 @@ from dvc.stage.exceptions import ( StageFileFormatError, StageFileIsNotDvcFileError, ) -from dvc.utils.yaml import dump_yaml +from dvc.utils.serialize import dump_yaml @pytest.mark.parametrize(