From 0eb4e7e15f9d964a462a497dc609a4f0dceff463 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Fri, 31 Jan 2020 00:02:28 +0700 Subject: [PATCH] api: move summon out Will be added to dvcx instead. --- dvc/api.py | 126 ----------------------------------------- tests/func/test_api.py | 60 +------------------- 2 files changed, 1 insertion(+), 185 deletions(-) diff --git a/dvc/api.py b/dvc/api.py index 081a2ba20..48a24c494 100644 --- a/dvc/api.py +++ b/dvc/api.py @@ -1,20 +1,11 @@ -from builtins import open as builtin_open import os from contextlib import contextmanager, _GeneratorContextManager as GCM -from funcy import cached_property, lmap -import ruamel.yaml -from voluptuous import Schema, Required, Invalid - from dvc.repo import Repo from dvc.exceptions import DvcException, NotDvcRepoError from dvc.external_repo import external_repo -class SummonError(DvcException): - pass - - class UrlNotDvcRepoError(DvcException): """Thrown if given url is not a DVC repository. @@ -92,123 +83,6 @@ def _make_repo(repo_url=None, rev=None): yield repo -class SummonFile(object): - DEFAULT_FILENAME = "dvcsummon.yaml" - SCHEMA = Schema( - { - Required("dvc-objects", default={}): { - str: { - "meta": dict, - Required("summon"): { - Required("type"): str, - "deps": [str], - str: object, - }, - } - } - } - ) - - def __init__(self, repo_obj, summon_file): - self.repo = repo_obj - self.filename = summon_file - self._path = os.path.join(self.repo.root_dir, summon_file) - - @staticmethod - @contextmanager - def prepare(repo=None, rev=None, summon_file=None): - """Does a couple of things every summon needs as a prerequisite: - clones the repo and parses the summon file. - - Calling code is expected to complete the summon logic following - instructions stated in "summon" dict of the object spec. - - Returns a SummonFile instance, which contains references to a Repo - object, named object specification and resolved paths to deps. - """ - summon_file = summon_file or SummonFile.DEFAULT_FILENAME - with _make_repo(repo, rev=rev) as _repo: - _require_dvc(_repo) - try: - yield SummonFile(_repo, summon_file) - except SummonError as exc: - raise SummonError( - str(exc) + " at '{}' in '{}'".format(summon_file, _repo) - ) from exc.__cause__ - - @cached_property - def objects(self): - return self._read_yaml()["dvc-objects"] - - def _read_yaml(self): - try: - with builtin_open(self._path, mode="r") as fd: - return self.SCHEMA(ruamel.yaml.safe_load(fd.read())) - except FileNotFoundError as exc: - raise SummonError("Summon file not found") from exc - except ruamel.yaml.YAMLError as exc: - raise SummonError("Failed to parse summon file") from exc - except Invalid as exc: - raise SummonError(str(exc)) from None - - def _write_yaml(self, objects): - try: - with builtin_open(self._path, "w") as fd: - content = self.SCHEMA({"dvc-objects": objects}) - ruamel.yaml.safe_dump(content, fd) - except Invalid as exc: - raise SummonError(str(exc)) from None - - def abs(self, path): - return os.path.join(self.repo.root_dir, path) - - def pull(self, targets): - self.repo.pull([self.abs(target) for target in targets]) - - def pull_deps(self, dobj): - self.pull(dobj["summon"].get("deps", [])) - - def get(self, name): - """ - Given a summonable object's name, search for it this file - and return its description. - """ - if name not in self.objects: - raise SummonError( - "No object with name '{}' in '{}'".format(name, self.filename) - ) - - return self.objects[name] - - def set(self, name, dobj, overwrite=True): - if not os.path.exists(self._path): - self.objects = self.SCHEMA({})["dvc-objects"] - - if name in self.objects and not overwrite: - raise SummonError( - "There is an existing summonable object named '{}' in '{}:{}'." - " Use SummonFile.set(..., overwrite=True) to" - " overwrite it.".format(name, self.repo.url, self.filename) - ) - - self.objects[name] = dobj - self._write_yaml(self.objects) - - # Add deps and push to remote - deps = dobj["summon"].get("deps", []) - stages = [] - if deps: - stages = self.repo.add( - lmap(self.abs, deps), fname=self.abs(name + ".dvc") - ) - self.repo.push() - - # Create commit and push - self.repo.scm.add([self._path] + [stage.path for stage in stages]) - self.repo.scm.commit("Add {} to {}".format(name, self.filename)) - self.repo.scm.push() - - def _require_dvc(repo): if not isinstance(repo, Repo): raise UrlNotDvcRepoError(repo.url) diff --git a/tests/func/test_api.py b/tests/func/test_api.py index 0855ebd7a..02d77866e 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -1,12 +1,10 @@ import os import shutil -import copy -import ruamel.yaml import pytest from dvc import api -from dvc.api import SummonFile, SummonError, UrlNotDvcRepoError +from dvc.api import UrlNotDvcRepoError from dvc.compat import fspath from dvc.exceptions import FileMissingError from dvc.main import main @@ -141,59 +139,3 @@ def test_open_not_cached(dvc): os.remove(metric_file) with pytest.raises(FileMissingError): api.read(metric_file) - - -def test_summon(tmp_dir, dvc, erepo_dir): - objects = { - SummonFile.DOBJ_SECTION: { - "sum": { - "meta": {"description": "Add to "}, - "summon": { - "type": "python", - "call": "calculator.add_to_num", - "args": {"x": 1}, - "deps": ["number"], - }, - } - } - } - - other_objects = copy.deepcopy(objects) - other_objects[SummonFile.DOBJ_SECTION]["sum"]["summon"]["args"]["x"] = 100 - - with erepo_dir.chdir(): - erepo_dir.dvc_gen("number", "100", commit="Add number.dvc") - erepo_dir.scm_gen(SummonFile.DEF_NAME, ruamel.yaml.dump(objects)) - erepo_dir.scm_gen("other.yaml", ruamel.yaml.dump(other_objects)) - erepo_dir.scm_gen("invalid.yaml", ruamel.yaml.dump({"name": "sum"})) - erepo_dir.scm_gen("not_yaml.yaml", "a: - this is not a YAML file") - erepo_dir.scm_gen( - "calculator.py", - "def add_to_num(x): return x + int(open('number').read())", - ) - erepo_dir.scm.commit("Add files") - - repo_url = "file://{}".format(erepo_dir) - - assert api.summon("sum", repo=repo_url) == 101 - assert api.summon("sum", repo=repo_url, args={"x": 2}) == 102 - assert api.summon("sum", repo=repo_url, summon_file="other.yaml") == 200 - - try: - api.summon("sum", repo=repo_url, summon_file="missing.yaml") - except SummonError as exc: - assert "Summon file not found" in str(exc) - assert "missing.yaml" in str(exc) - # Fails - # assert repo_url in str(exc) - else: - pytest.fail("Did not raise on missing summon file") - - with pytest.raises(SummonError, match=r"No object with name 'missing'"): - api.summon("missing", repo=repo_url) - - with pytest.raises(SummonError, match=r"extra keys not allowed"): - api.summon("sum", repo=repo_url, summon_file="invalid.yaml") - - with pytest.raises(SummonError, match=r"Failed to parse summon file"): - api.summon("sum", repo=repo_url, summon_file="not_yaml.yaml")