mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
live integration: get rid of dvc from live (#5466)
* live integration: get rid of dvc from live * live: remove from api, add test for html generation during run * stage: run: convert monitors to class context managers * run: make monitors run in single thread * run: move monitor logic out of run
This commit is contained in:
Paweł Redzyński
@@ -1,273 +0,0 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from flaky.flaky_decorator import flaky
|
||||
from funcy import first, get_in
|
||||
|
||||
from dvc import api
|
||||
from dvc.exceptions import FileMissingError, OutputNotFoundError
|
||||
from dvc.path_info import URLInfo
|
||||
from dvc.utils.fs import remove
|
||||
from tests.unit.fs.test_repo import make_subrepo
|
||||
|
||||
cloud_names = [
|
||||
"s3",
|
||||
"gs",
|
||||
"azure",
|
||||
"gdrive",
|
||||
"oss",
|
||||
"ssh",
|
||||
"http",
|
||||
"hdfs",
|
||||
"webdav",
|
||||
"webhdfs",
|
||||
]
|
||||
clouds = [pytest.lazy_fixture(cloud) for cloud in cloud_names]
|
||||
all_clouds = [pytest.lazy_fixture("local_cloud")] + clouds
|
||||
|
||||
# `lazy_fixture` is confusing pylint, pylint: disable=unused-argument
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote", clouds, indirect=True)
|
||||
def test_get_url(tmp_dir, dvc, remote):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
|
||||
expected_url = URLInfo(remote.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
assert api.get_url("foo") == expected_url
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cloud", clouds)
|
||||
def test_get_url_external(tmp_dir, erepo_dir, cloud):
|
||||
erepo_dir.add_remote(config=cloud.config)
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("foo", "foo", commit="add foo")
|
||||
|
||||
# Using file url to force clone to tmp repo
|
||||
repo_url = f"file://{erepo_dir}"
|
||||
expected_url = URLInfo(cloud.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
assert api.get_url("foo", repo=repo_url) == expected_url
|
||||
|
||||
|
||||
def test_get_url_requires_dvc(tmp_dir, scm):
|
||||
tmp_dir.scm_gen({"foo": "foo"}, commit="initial")
|
||||
|
||||
with pytest.raises(OutputNotFoundError, match="output 'foo'"):
|
||||
api.get_url("foo", repo=os.fspath(tmp_dir))
|
||||
|
||||
with pytest.raises(OutputNotFoundError, match="output 'foo'"):
|
||||
api.get_url("foo", repo=f"file://{tmp_dir}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote", all_clouds, indirect=True)
|
||||
def test_open(tmp_dir, dvc, remote):
|
||||
tmp_dir.dvc_gen("foo", "foo-text")
|
||||
dvc.push()
|
||||
|
||||
# Remove cache to force download
|
||||
remove(dvc.odb.local.cache_dir)
|
||||
|
||||
with api.open("foo") as fd:
|
||||
assert fd.read() == "foo-text"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cloud",
|
||||
[
|
||||
pytest.lazy_fixture(cloud)
|
||||
for cloud in [
|
||||
"s3",
|
||||
"gs",
|
||||
"azure",
|
||||
"gdrive",
|
||||
"oss",
|
||||
"http",
|
||||
"hdfs",
|
||||
"ssh",
|
||||
"webdav",
|
||||
]
|
||||
],
|
||||
)
|
||||
def test_open_external(tmp_dir, erepo_dir, cloud):
|
||||
erepo_dir.add_remote(config=cloud.config)
|
||||
|
||||
with erepo_dir.chdir():
|
||||
erepo_dir.dvc_gen("version", "master", commit="add version")
|
||||
|
||||
with erepo_dir.branch("branch", new="True"):
|
||||
# NOTE: need file to be other size for Mac
|
||||
erepo_dir.dvc_gen("version", "branchver", commit="add version")
|
||||
|
||||
erepo_dir.dvc.push(all_branches=True)
|
||||
|
||||
# Remove cache to force download
|
||||
remove(erepo_dir.dvc.odb.local.cache_dir)
|
||||
|
||||
# Using file url to force clone to tmp repo
|
||||
repo_url = f"file://{erepo_dir}"
|
||||
with api.open("version", repo=repo_url) as fd:
|
||||
assert fd.read() == "master"
|
||||
|
||||
assert api.read("version", repo=repo_url, rev="branch") == "branchver"
|
||||
|
||||
|
||||
@flaky(max_runs=3, min_passes=1)
|
||||
@pytest.mark.parametrize("remote", all_clouds, indirect=True)
|
||||
def test_open_granular(tmp_dir, dvc, remote):
|
||||
tmp_dir.dvc_gen({"dir": {"foo": "foo-text"}})
|
||||
dvc.push()
|
||||
|
||||
# Remove cache to force download
|
||||
remove(dvc.odb.local.cache_dir)
|
||||
|
||||
with api.open("dir/foo") as fd:
|
||||
assert fd.read() == "foo-text"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"remote",
|
||||
[
|
||||
pytest.lazy_fixture(cloud)
|
||||
for cloud in [
|
||||
"s3",
|
||||
"gs",
|
||||
"azure",
|
||||
"gdrive",
|
||||
"oss",
|
||||
"ssh",
|
||||
"http",
|
||||
"hdfs",
|
||||
"webdav",
|
||||
]
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
def test_missing(tmp_dir, dvc, remote):
|
||||
tmp_dir.dvc_gen("foo", "foo")
|
||||
|
||||
# Remove cache to make foo missing
|
||||
remove(dvc.odb.local.cache_dir)
|
||||
|
||||
api.read("foo")
|
||||
|
||||
remove("foo")
|
||||
|
||||
with pytest.raises(FileMissingError):
|
||||
api.read("foo")
|
||||
|
||||
|
||||
def test_open_scm_controlled(tmp_dir, erepo_dir):
|
||||
erepo_dir.scm_gen({"scm_controlled": "file content"}, commit="create file")
|
||||
|
||||
with api.open("scm_controlled", repo=os.fspath(erepo_dir)) as fd:
|
||||
assert fd.read() == "file content"
|
||||
|
||||
|
||||
def test_open_not_cached(dvc):
|
||||
metric_file = "metric.txt"
|
||||
metric_content = "0.6"
|
||||
metric_code = "open('{}', 'w').write('{}')".format(
|
||||
metric_file, metric_content
|
||||
)
|
||||
dvc.run(
|
||||
single_stage=True,
|
||||
metrics_no_cache=[metric_file],
|
||||
cmd=(f'python -c "{metric_code}"'),
|
||||
)
|
||||
|
||||
with api.open(metric_file) as fd:
|
||||
assert fd.read() == metric_content
|
||||
|
||||
os.remove(metric_file)
|
||||
with pytest.raises(FileMissingError):
|
||||
api.read(metric_file)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("as_external", [True, False])
|
||||
@pytest.mark.parametrize("remote", [pytest.lazy_fixture("ssh")], indirect=True)
|
||||
@pytest.mark.parametrize(
|
||||
"files, to_read",
|
||||
[
|
||||
({"foo": "foo"}, "foo"),
|
||||
({"dir": {"foo": "foo", "bar": "bar"}}, os.path.join("dir", "foo")),
|
||||
],
|
||||
ids=["file", "inside-dir"],
|
||||
)
|
||||
def test_api_missing_local_cache_exists_on_remote(
|
||||
tmp_dir, scm, dvc, as_external, remote, files, to_read,
|
||||
):
|
||||
tmp_dir.dvc_gen(files, commit="DVC track files")
|
||||
dvc.push()
|
||||
|
||||
# Remove cache to make foo missing
|
||||
remove(dvc.odb.local.cache_dir)
|
||||
remove(first(files))
|
||||
|
||||
repo_url = f"file://{tmp_dir}" if as_external else None
|
||||
file_content = get_in(files, to_read.split(os.sep))
|
||||
assert api.read(to_read, repo=repo_url) == file_content
|
||||
|
||||
|
||||
@pytest.mark.parametrize("local_repo", [False, True])
|
||||
def test_read_with_subrepos(tmp_dir, scm, local_cloud, local_repo):
|
||||
tmp_dir.scm_gen("foo.txt", "foo.txt", commit="add foo.txt")
|
||||
subrepo = tmp_dir / "dir" / "subrepo"
|
||||
make_subrepo(subrepo, scm, config=local_cloud.config)
|
||||
with subrepo.chdir():
|
||||
subrepo.scm_gen({"lorem": "lorem"}, commit="add lorem")
|
||||
subrepo.dvc_gen({"dir": {"file.txt": "file.txt"}}, commit="add dir")
|
||||
subrepo.dvc_gen("dvc-file", "dvc-file", commit="add dir")
|
||||
subrepo.dvc.push()
|
||||
|
||||
repo_path = None if local_repo else f"file:///{tmp_dir}"
|
||||
subrepo_path = os.path.join("dir", "subrepo")
|
||||
|
||||
assert api.read("foo.txt", repo=repo_path) == "foo.txt"
|
||||
assert (
|
||||
api.read(os.path.join(subrepo_path, "lorem"), repo=repo_path)
|
||||
== "lorem"
|
||||
)
|
||||
assert (
|
||||
api.read(os.path.join(subrepo_path, "dvc-file"), repo=repo_path)
|
||||
== "dvc-file"
|
||||
)
|
||||
assert (
|
||||
api.read(os.path.join(subrepo_path, "dir", "file.txt"), repo=repo_path)
|
||||
== "file.txt"
|
||||
)
|
||||
|
||||
|
||||
def test_get_url_granular(tmp_dir, dvc, s3):
|
||||
tmp_dir.add_remote(config=s3.config)
|
||||
tmp_dir.dvc_gen(
|
||||
{"dir": {"foo": "foo", "bar": "bar", "nested": {"file": "file"}}}
|
||||
)
|
||||
|
||||
expected_url = URLInfo(s3.url) / "5f/c28ea78987408341668eba6525ebd1.dir"
|
||||
assert api.get_url("dir") == expected_url
|
||||
|
||||
expected_url = URLInfo(s3.url) / "ac/bd18db4cc2f85cedef654fccc4a4d8"
|
||||
assert api.get_url("dir/foo") == expected_url
|
||||
|
||||
expected_url = URLInfo(s3.url) / "37/b51d194a7513e45b56f6524f2d51f2"
|
||||
assert api.get_url("dir/bar") == expected_url
|
||||
|
||||
expected_url = URLInfo(s3.url) / "8c/7dd922ad47494fc02c388e12c00eac"
|
||||
assert api.get_url(os.path.join("dir", "nested", "file")) == expected_url
|
||||
|
||||
|
||||
def test_get_url_subrepos(tmp_dir, scm, local_cloud):
|
||||
subrepo = tmp_dir / "subrepo"
|
||||
make_subrepo(subrepo, scm, config=local_cloud.config)
|
||||
with subrepo.chdir():
|
||||
subrepo.dvc_gen(
|
||||
{"dir": {"foo": "foo"}, "bar": "bar"}, commit="add files"
|
||||
)
|
||||
subrepo.dvc.push()
|
||||
|
||||
path = os.path.relpath(local_cloud.config["url"])
|
||||
|
||||
expected_url = os.path.join(path, "ac", "bd18db4cc2f85cedef654fccc4a4d8")
|
||||
assert api.get_url(os.path.join("subrepo", "dir", "foo")) == expected_url
|
||||
|
||||
expected_url = os.path.join(path, "37", "b51d194a7513e45b56f6524f2d51f2")
|
||||
assert api.get_url("subrepo/bar") == expected_url
|
||||
@@ -1,55 +0,0 @@
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
from typing import Dict, List
|
||||
|
||||
import pytest
|
||||
from funcy import last
|
||||
|
||||
from dvc.api.live import summary
|
||||
from tests.utils import dump_sv
|
||||
|
||||
|
||||
def _dumps_tsv(metrics: List[Dict]):
|
||||
stream = io.StringIO()
|
||||
dump_sv(stream, metrics)
|
||||
stream.seek(0)
|
||||
return stream.read()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_results(tmp_dir):
|
||||
def make(path="logs"):
|
||||
datapoints = [{"metric": 0.0, "step": 0}, {"metric": 0.5, "step": 1}]
|
||||
tmp_dir.gen(
|
||||
{
|
||||
(tmp_dir / path).with_suffix(".json"): json.dumps(
|
||||
last(datapoints)
|
||||
),
|
||||
(tmp_dir / path / "metric.tsv"): _dumps_tsv(datapoints),
|
||||
}
|
||||
)
|
||||
|
||||
yield make
|
||||
|
||||
|
||||
def test_live_summary_no_repo(tmp_dir, live_results, caplog):
|
||||
live_results("logs")
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="dvc"):
|
||||
summary("logs")
|
||||
|
||||
summary_path = tmp_dir / "logs.html"
|
||||
assert summary_path.exists()
|
||||
assert f"file://{str(summary_path)}" in caplog.text
|
||||
|
||||
|
||||
def test_live_summary(tmp_dir, dvc, live_results, caplog):
|
||||
live_results("logs")
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="dvc"):
|
||||
summary("logs")
|
||||
|
||||
summary_path = tmp_dir / "logs.html"
|
||||
assert summary_path.exists()
|
||||
assert f"file://{str(summary_path)}" in caplog.text
|
||||
Reference in New Issue
Block a user