mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
api: move summon out
Will be added to dvcx instead.
This commit is contained in:
Alexander Schepanovski
-126
@@ -1,20 +1,11 @@
|
||||
from builtins import open as builtin_open
|
||||
import os
|
||||
from contextlib import contextmanager, _GeneratorContextManager as GCM
|
||||
|
||||
from funcy import cached_property, lmap
|
||||
import ruamel.yaml
|
||||
from voluptuous import Schema, Required, Invalid
|
||||
|
||||
from dvc.repo import Repo
|
||||
from dvc.exceptions import DvcException, NotDvcRepoError
|
||||
from dvc.external_repo import external_repo
|
||||
|
||||
|
||||
class SummonError(DvcException):
|
||||
pass
|
||||
|
||||
|
||||
class UrlNotDvcRepoError(DvcException):
|
||||
"""Thrown if given url is not a DVC repository.
|
||||
|
||||
@@ -92,123 +83,6 @@ def _make_repo(repo_url=None, rev=None):
|
||||
yield repo
|
||||
|
||||
|
||||
class SummonFile(object):
|
||||
DEFAULT_FILENAME = "dvcsummon.yaml"
|
||||
SCHEMA = Schema(
|
||||
{
|
||||
Required("dvc-objects", default={}): {
|
||||
str: {
|
||||
"meta": dict,
|
||||
Required("summon"): {
|
||||
Required("type"): str,
|
||||
"deps": [str],
|
||||
str: object,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, repo_obj, summon_file):
|
||||
self.repo = repo_obj
|
||||
self.filename = summon_file
|
||||
self._path = os.path.join(self.repo.root_dir, summon_file)
|
||||
|
||||
@staticmethod
|
||||
@contextmanager
|
||||
def prepare(repo=None, rev=None, summon_file=None):
|
||||
"""Does a couple of things every summon needs as a prerequisite:
|
||||
clones the repo and parses the summon file.
|
||||
|
||||
Calling code is expected to complete the summon logic following
|
||||
instructions stated in "summon" dict of the object spec.
|
||||
|
||||
Returns a SummonFile instance, which contains references to a Repo
|
||||
object, named object specification and resolved paths to deps.
|
||||
"""
|
||||
summon_file = summon_file or SummonFile.DEFAULT_FILENAME
|
||||
with _make_repo(repo, rev=rev) as _repo:
|
||||
_require_dvc(_repo)
|
||||
try:
|
||||
yield SummonFile(_repo, summon_file)
|
||||
except SummonError as exc:
|
||||
raise SummonError(
|
||||
str(exc) + " at '{}' in '{}'".format(summon_file, _repo)
|
||||
) from exc.__cause__
|
||||
|
||||
@cached_property
|
||||
def objects(self):
|
||||
return self._read_yaml()["dvc-objects"]
|
||||
|
||||
def _read_yaml(self):
|
||||
try:
|
||||
with builtin_open(self._path, mode="r") as fd:
|
||||
return self.SCHEMA(ruamel.yaml.safe_load(fd.read()))
|
||||
except FileNotFoundError as exc:
|
||||
raise SummonError("Summon file not found") from exc
|
||||
except ruamel.yaml.YAMLError as exc:
|
||||
raise SummonError("Failed to parse summon file") from exc
|
||||
except Invalid as exc:
|
||||
raise SummonError(str(exc)) from None
|
||||
|
||||
def _write_yaml(self, objects):
|
||||
try:
|
||||
with builtin_open(self._path, "w") as fd:
|
||||
content = self.SCHEMA({"dvc-objects": objects})
|
||||
ruamel.yaml.safe_dump(content, fd)
|
||||
except Invalid as exc:
|
||||
raise SummonError(str(exc)) from None
|
||||
|
||||
def abs(self, path):
|
||||
return os.path.join(self.repo.root_dir, path)
|
||||
|
||||
def pull(self, targets):
|
||||
self.repo.pull([self.abs(target) for target in targets])
|
||||
|
||||
def pull_deps(self, dobj):
|
||||
self.pull(dobj["summon"].get("deps", []))
|
||||
|
||||
def get(self, name):
|
||||
"""
|
||||
Given a summonable object's name, search for it this file
|
||||
and return its description.
|
||||
"""
|
||||
if name not in self.objects:
|
||||
raise SummonError(
|
||||
"No object with name '{}' in '{}'".format(name, self.filename)
|
||||
)
|
||||
|
||||
return self.objects[name]
|
||||
|
||||
def set(self, name, dobj, overwrite=True):
|
||||
if not os.path.exists(self._path):
|
||||
self.objects = self.SCHEMA({})["dvc-objects"]
|
||||
|
||||
if name in self.objects and not overwrite:
|
||||
raise SummonError(
|
||||
"There is an existing summonable object named '{}' in '{}:{}'."
|
||||
" Use SummonFile.set(..., overwrite=True) to"
|
||||
" overwrite it.".format(name, self.repo.url, self.filename)
|
||||
)
|
||||
|
||||
self.objects[name] = dobj
|
||||
self._write_yaml(self.objects)
|
||||
|
||||
# Add deps and push to remote
|
||||
deps = dobj["summon"].get("deps", [])
|
||||
stages = []
|
||||
if deps:
|
||||
stages = self.repo.add(
|
||||
lmap(self.abs, deps), fname=self.abs(name + ".dvc")
|
||||
)
|
||||
self.repo.push()
|
||||
|
||||
# Create commit and push
|
||||
self.repo.scm.add([self._path] + [stage.path for stage in stages])
|
||||
self.repo.scm.commit("Add {} to {}".format(name, self.filename))
|
||||
self.repo.scm.push()
|
||||
|
||||
|
||||
def _require_dvc(repo):
|
||||
if not isinstance(repo, Repo):
|
||||
raise UrlNotDvcRepoError(repo.url)
|
||||
|
||||
+1
-59
@@ -1,12 +1,10 @@
|
||||
import os
|
||||
import shutil
|
||||
import copy
|
||||
|
||||
import ruamel.yaml
|
||||
import pytest
|
||||
|
||||
from dvc import api
|
||||
from dvc.api import SummonFile, SummonError, UrlNotDvcRepoError
|
||||
from dvc.api import UrlNotDvcRepoError
|
||||
from dvc.compat import fspath
|
||||
from dvc.exceptions import FileMissingError
|
||||
from dvc.main import main
|
||||
@@ -141,59 +139,3 @@ def test_open_not_cached(dvc):
|
||||
os.remove(metric_file)
|
||||
with pytest.raises(FileMissingError):
|
||||
api.read(metric_file)
|
||||
|
||||
|
||||
def test_summon(tmp_dir, dvc, erepo_dir):
|
||||
objects = {
|
||||
SummonFile.DOBJ_SECTION: {
|
||||
"sum": {
|
||||
"meta": {"description": "Add <x> to <number>"},
|
||||
"summon": {
|
||||
"type": "python",
|
||||
"call": "calculator.add_to_num",
|
||||
"args": {"x": 1},
|
||||
"deps": ["number"],
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
other_objects = copy.deepcopy(objects)
|
||||
other_objects[SummonFile.DOBJ_SECTION]["sum"]["summon"]["args"]["x"] = 100
|
||||
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("number", "100", commit="Add number.dvc")
|
||||
erepo_dir.scm_gen(SummonFile.DEF_NAME, ruamel.yaml.dump(objects))
|
||||
erepo_dir.scm_gen("other.yaml", ruamel.yaml.dump(other_objects))
|
||||
erepo_dir.scm_gen("invalid.yaml", ruamel.yaml.dump({"name": "sum"}))
|
||||
erepo_dir.scm_gen("not_yaml.yaml", "a: - this is not a YAML file")
|
||||
erepo_dir.scm_gen(
|
||||
"calculator.py",
|
||||
"def add_to_num(x): return x + int(open('number').read())",
|
||||
)
|
||||
erepo_dir.scm.commit("Add files")
|
||||
|
||||
repo_url = "file://{}".format(erepo_dir)
|
||||
|
||||
assert api.summon("sum", repo=repo_url) == 101
|
||||
assert api.summon("sum", repo=repo_url, args={"x": 2}) == 102
|
||||
assert api.summon("sum", repo=repo_url, summon_file="other.yaml") == 200
|
||||
|
||||
try:
|
||||
api.summon("sum", repo=repo_url, summon_file="missing.yaml")
|
||||
except SummonError as exc:
|
||||
assert "Summon file not found" in str(exc)
|
||||
assert "missing.yaml" in str(exc)
|
||||
# Fails
|
||||
# assert repo_url in str(exc)
|
||||
else:
|
||||
pytest.fail("Did not raise on missing summon file")
|
||||
|
||||
with pytest.raises(SummonError, match=r"No object with name 'missing'"):
|
||||
api.summon("missing", repo=repo_url)
|
||||
|
||||
with pytest.raises(SummonError, match=r"extra keys not allowed"):
|
||||
api.summon("sum", repo=repo_url, summon_file="invalid.yaml")
|
||||
|
||||
with pytest.raises(SummonError, match=r"Failed to parse summon file"):
|
||||
api.summon("sum", repo=repo_url, summon_file="not_yaml.yaml")
|
||||
|
||||
Reference in New Issue
Block a user