diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 5209c1f86..209259475 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -1,9 +1,6 @@ from __future__ import unicode_literals -import schema - import dvc.output as output -from .repo import DependencyREPO from dvc.dependency.gs import DependencyGS from dvc.dependency.hdfs import DependencyHDFS from dvc.dependency.http import DependencyHTTP @@ -14,6 +11,7 @@ from dvc.dependency.ssh import DependencySSH from dvc.output.base import OutputBase from dvc.remote import Remote from dvc.scheme import Schemes +from .repo import DependencyREPO DEPS = [ @@ -42,9 +40,9 @@ DEP_MAP = { # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not # output is a metric file and how to parse it, see `-M` flag for `dvc run`). SCHEMA = output.SCHEMA.copy() -del SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)] -del SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)] -SCHEMA[schema.Optional(DependencyREPO.PARAM_REPO)] = DependencyREPO.REPO_SCHEMA +del SCHEMA[OutputBase.PARAM_CACHE] +del SCHEMA[OutputBase.PARAM_METRIC] +SCHEMA[DependencyREPO.PARAM_REPO] = DependencyREPO.REPO_SCHEMA def _get(stage, p, info): diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 20439c024..f0ff9df06 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -4,7 +4,6 @@ import copy from contextlib import contextmanager from funcy import merge -from schema import Optional from .local import DependencyLOCAL from dvc.external_repo import external_repo @@ -17,11 +16,7 @@ class DependencyREPO(DependencyLOCAL): PARAM_REV = "rev" PARAM_REV_LOCK = "rev_lock" - REPO_SCHEMA = { - Optional(PARAM_URL): str, - Optional(PARAM_REV): str, - Optional(PARAM_REV_LOCK): str, - } + REPO_SCHEMA = {PARAM_URL: str, PARAM_REV: str, PARAM_REV_LOCK: str} def __init__(self, def_repo, stage, *args, **kwargs): self.def_repo = def_repo diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index b12ebf3af..d5ff96c99 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -import schema +from voluptuous import Any, Required from dvc.output.base import OutputBase from dvc.output.gs import OutputGS @@ -41,19 +41,19 @@ OUTS_MAP = { # so when a few types of outputs share the same name, we only need # specify it once. CHECKSUM_SCHEMA = { - schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None), - schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None), - schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None), + RemoteLOCAL.PARAM_CHECKSUM: Any(str, None), + RemoteS3.PARAM_CHECKSUM: Any(str, None), + RemoteHDFS.PARAM_CHECKSUM: Any(str, None), } -TAGS_SCHEMA = {schema.Optional(str): CHECKSUM_SCHEMA} +TAGS_SCHEMA = {str: CHECKSUM_SCHEMA} SCHEMA = CHECKSUM_SCHEMA.copy() -SCHEMA[OutputBase.PARAM_PATH] = str -SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)] = bool -SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)] = OutputBase.METRIC_SCHEMA -SCHEMA[schema.Optional(OutputBase.PARAM_TAGS)] = TAGS_SCHEMA -SCHEMA[schema.Optional(OutputBase.PARAM_PERSIST)] = bool +SCHEMA[Required(OutputBase.PARAM_PATH)] = str +SCHEMA[OutputBase.PARAM_CACHE] = bool +SCHEMA[OutputBase.PARAM_METRIC] = OutputBase.METRIC_SCHEMA +SCHEMA[OutputBase.PARAM_TAGS] = TAGS_SCHEMA +SCHEMA[OutputBase.PARAM_PERSIST] = bool def _get(stage, p, info, cache, metric, persist=False, tags=None): diff --git a/dvc/output/base.py b/dvc/output/base.py index 2a0f7bf24..120489d19 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -3,8 +3,7 @@ from __future__ import unicode_literals import logging from copy import copy -from schema import Optional -from schema import Or +from voluptuous import Any import dvc.prompt as prompt from dvc.cache import NamedCache @@ -55,12 +54,12 @@ class OutputBase(object): PARAM_METRIC_XPATH = "xpath" PARAM_PERSIST = "persist" - METRIC_SCHEMA = Or( + METRIC_SCHEMA = Any( None, bool, { - Optional(PARAM_METRIC_TYPE): Or(str, None), - Optional(PARAM_METRIC_XPATH): Or(str, None), + PARAM_METRIC_TYPE: Any(str, None), + PARAM_METRIC_XPATH: Any(str, None), }, ) diff --git a/dvc/stage.py b/dvc/stage.py index d408019dc..94b768ca3 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +from dvc.utils.compat import pathlib, str import copy import logging import os @@ -9,11 +10,7 @@ import subprocess import threading from itertools import chain -from schema import And -from schema import Optional -from schema import Or -from schema import Schema -from schema import SchemaError +from voluptuous import Any, Schema, MultipleInvalid import dvc.dependency as dependency import dvc.output as output @@ -23,8 +20,6 @@ from dvc.utils import dict_md5 from dvc.utils import fix_env from dvc.utils import relpath from dvc.utils.collections import apply_diff -from dvc.utils.compat import pathlib -from dvc.utils.compat import str from dvc.utils.fs import contains_symlink_up_to from dvc.utils.stage import dump_stage_file from dvc.utils.stage import load_stage_fd @@ -147,15 +142,16 @@ class Stage(object): PARAM_ALWAYS_CHANGED = "always_changed" SCHEMA = { - Optional(PARAM_MD5): Or(str, None), - Optional(PARAM_CMD): Or(str, None), - Optional(PARAM_WDIR): Or(str, None), - Optional(PARAM_DEPS): Or(And(list, Schema([dependency.SCHEMA])), None), - Optional(PARAM_OUTS): Or(And(list, Schema([output.SCHEMA])), None), - Optional(PARAM_LOCKED): bool, - Optional(PARAM_META): object, - Optional(PARAM_ALWAYS_CHANGED): bool, + PARAM_MD5: Any(str, None), + PARAM_CMD: Any(str, None), + PARAM_WDIR: Any(str, None), + PARAM_DEPS: Any([dependency.SCHEMA], None), + PARAM_OUTS: Any([output.SCHEMA], None), + PARAM_LOCKED: bool, + PARAM_META: object, + PARAM_ALWAYS_CHANGED: bool, } + COMPILED_SCHEMA = Schema(SCHEMA) TAG_REGEX = r"^(?P.*)@(?P[^\\/@:]*)$" @@ -362,8 +358,8 @@ class Stage(object): from dvc.utils import convert_to_unicode try: - Schema(Stage.SCHEMA).validate(convert_to_unicode(d)) - except SchemaError as exc: + Stage.COMPILED_SCHEMA(convert_to_unicode(d)) + except MultipleInvalid as exc: raise StageFileFormatError(fname, exc) @classmethod diff --git a/setup.py b/setup.py index 00bd03bf6..87bbf31d0 100644 --- a/setup.py +++ b/setup.py @@ -63,6 +63,8 @@ install_requires = [ "nanotime>=0.5.2", "pyasn1>=0.4.1", "schema>=0.6.7", + "schema>=0.6.7", + "voluptuous==0.11.7", "jsonpath-ng>=1.4.3", "requests>=2.22.0", "grandalf==0.6",