mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 23:11:44 +01:00
* plots: allow dir plots * first scenario * separate dvclive tests * rename index to step * another test * refactor * its ok to have step in data * sort plots * introduce --logs * add logs to run commands * variable name refactoring * fix logs option * fixup * cmd: run: fix unit tests * plots: modify: fix tests * improve dir handling * fixup * dvclive: logs summary * api: introduce logs summarization method * handle no repo case * fix granular dir plots * api: dvclive: simplify * dvclive: roll back logs flag * cmd: introduce logs * api: adjust to logs * test: dvclive fix * rename logs to dvclive * live: command: add uninitialized * outs: dvclive type * dvclive: support providing config via evn variables * dvclive: treat as metrics * dvclive integration: add support for summary on/off * dvc integration: add temporary test for configuration export * remove requirements * cmd: introduce logs * rename logs to dvclive * cmd: introduce logs * remove logs leftowvers * dvclive: env: adjust to chekpoints env variables * command: live: remove unnecessary help * fix linter suggestions * type hinting * review refactor * replace dvclive with live * stage: loading: move live loading to separate method * stage: serialize: allow only single live output * live: add show/diff * stage: serialize: roll back single live enforcing * fixup * rebase fixups * output: live: force single instance * live: command: add unit tests * Update dvc/output/base.py Co-authored-by: Ruslan Kuprieiev <[email protected]> * Update dvc/stage/__init__.py Co-authored-by: Ruslan Kuprieiev <[email protected]> * Update dvc/stage/serialize.py Co-authored-by: Ruslan Kuprieiev <[email protected]> * review v2 * live: fix summary tracking by experiments * naming fixups * api: live: add tests * tests: api: separate live tests Co-authored-by: Ruslan Kuprieiev <[email protected]>
267 lines
6.5 KiB
Python
267 lines
6.5 KiB
Python
from collections import defaultdict
|
|
from urllib.parse import urlparse
|
|
|
|
from funcy import collecting, project
|
|
from voluptuous import And, Any, Coerce, Length, Lower, Required, SetTo
|
|
|
|
from dvc.hash_info import HashInfo
|
|
from dvc.output.base import BaseOutput
|
|
from dvc.output.gs import GSOutput
|
|
from dvc.output.hdfs import HDFSOutput
|
|
from dvc.output.local import LocalOutput
|
|
from dvc.output.s3 import S3Output
|
|
from dvc.output.ssh import SSHOutput
|
|
from dvc.output.webhdfs import WebHDFSOutput
|
|
from dvc.scheme import Schemes
|
|
|
|
from ..tree import get_cloud_tree
|
|
from ..tree.hdfs import HDFSTree
|
|
from ..tree.local import LocalTree
|
|
from ..tree.s3 import S3Tree
|
|
from ..tree.webhdfs import WebHDFSTree
|
|
|
|
OUTS = [
|
|
HDFSOutput,
|
|
S3Output,
|
|
GSOutput,
|
|
SSHOutput,
|
|
WebHDFSOutput,
|
|
# NOTE: LocalOutput is the default choice
|
|
]
|
|
|
|
OUTS_MAP = {
|
|
Schemes.HDFS: HDFSOutput,
|
|
Schemes.S3: S3Output,
|
|
Schemes.GS: GSOutput,
|
|
Schemes.SSH: SSHOutput,
|
|
Schemes.LOCAL: LocalOutput,
|
|
Schemes.WEBHDFS: WebHDFSOutput,
|
|
}
|
|
|
|
CHECKSUM_SCHEMA = Any(
|
|
None,
|
|
And(str, Length(max=0), SetTo(None)),
|
|
And(Any(str, And(int, Coerce(str))), Length(min=3), Lower),
|
|
)
|
|
|
|
# NOTE: currently there are only 3 possible checksum names:
|
|
#
|
|
# 1) md5 (LOCAL, SSH, GS);
|
|
# 2) etag (S3);
|
|
# 3) checksum (HDFS);
|
|
#
|
|
# so when a few types of outputs share the same name, we only need
|
|
# specify it once.
|
|
CHECKSUMS_SCHEMA = {
|
|
LocalTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
|
S3Tree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
|
HDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
|
WebHDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
|
|
}
|
|
|
|
SCHEMA = CHECKSUMS_SCHEMA.copy()
|
|
SCHEMA[Required(BaseOutput.PARAM_PATH)] = str
|
|
SCHEMA[BaseOutput.PARAM_CACHE] = bool
|
|
SCHEMA[BaseOutput.PARAM_METRIC] = BaseOutput.METRIC_SCHEMA
|
|
SCHEMA[BaseOutput.PARAM_PLOT] = bool
|
|
SCHEMA[BaseOutput.PARAM_PERSIST] = bool
|
|
SCHEMA[BaseOutput.PARAM_CHECKPOINT] = bool
|
|
SCHEMA[HashInfo.PARAM_SIZE] = int
|
|
SCHEMA[HashInfo.PARAM_NFILES] = int
|
|
SCHEMA[BaseOutput.PARAM_DESC] = str
|
|
SCHEMA[BaseOutput.PARAM_ISEXEC] = bool
|
|
|
|
|
|
def _get(
|
|
stage,
|
|
p,
|
|
info=None,
|
|
cache=True,
|
|
metric=False,
|
|
plot=False,
|
|
persist=False,
|
|
checkpoint=False,
|
|
live=False,
|
|
desc=None,
|
|
isexec=False,
|
|
):
|
|
parsed = urlparse(p)
|
|
|
|
if parsed.scheme == "remote":
|
|
tree = get_cloud_tree(stage.repo, name=parsed.netloc)
|
|
return OUTS_MAP[tree.scheme](
|
|
stage,
|
|
p,
|
|
info,
|
|
cache=cache,
|
|
tree=tree,
|
|
metric=metric,
|
|
plot=plot,
|
|
persist=persist,
|
|
checkpoint=checkpoint,
|
|
live=live,
|
|
desc=desc,
|
|
isexec=isexec,
|
|
)
|
|
|
|
for o in OUTS:
|
|
if o.supported(p):
|
|
return o(
|
|
stage,
|
|
p,
|
|
info,
|
|
cache=cache,
|
|
tree=None,
|
|
metric=metric,
|
|
plot=plot,
|
|
persist=persist,
|
|
checkpoint=checkpoint,
|
|
live=live,
|
|
desc=desc,
|
|
isexec=isexec,
|
|
)
|
|
return LocalOutput(
|
|
stage,
|
|
p,
|
|
info,
|
|
cache=cache,
|
|
tree=None,
|
|
metric=metric,
|
|
plot=plot,
|
|
persist=persist,
|
|
checkpoint=checkpoint,
|
|
live=live,
|
|
desc=desc,
|
|
isexec=isexec,
|
|
)
|
|
|
|
|
|
def loadd_from(stage, d_list):
|
|
ret = []
|
|
for d in d_list:
|
|
p = d.pop(BaseOutput.PARAM_PATH)
|
|
cache = d.pop(BaseOutput.PARAM_CACHE, True)
|
|
metric = d.pop(BaseOutput.PARAM_METRIC, False)
|
|
plot = d.pop(BaseOutput.PARAM_PLOT, False)
|
|
persist = d.pop(BaseOutput.PARAM_PERSIST, False)
|
|
checkpoint = d.pop(BaseOutput.PARAM_CHECKPOINT, False)
|
|
desc = d.pop(BaseOutput.PARAM_DESC, False)
|
|
isexec = d.pop(BaseOutput.PARAM_ISEXEC, False)
|
|
live = d.pop(BaseOutput.PARAM_LIVE, False)
|
|
ret.append(
|
|
_get(
|
|
stage,
|
|
p,
|
|
info=d,
|
|
cache=cache,
|
|
metric=metric,
|
|
plot=plot,
|
|
persist=persist,
|
|
checkpoint=checkpoint,
|
|
desc=desc,
|
|
isexec=isexec,
|
|
live=live,
|
|
)
|
|
)
|
|
return ret
|
|
|
|
|
|
def loads_from(
|
|
stage,
|
|
s_list,
|
|
use_cache=True,
|
|
metric=False,
|
|
plot=False,
|
|
persist=False,
|
|
checkpoint=False,
|
|
isexec=False,
|
|
live=False,
|
|
):
|
|
return [
|
|
_get(
|
|
stage,
|
|
s,
|
|
info={},
|
|
cache=use_cache,
|
|
metric=metric,
|
|
plot=plot,
|
|
persist=persist,
|
|
checkpoint=checkpoint,
|
|
isexec=isexec,
|
|
live=live,
|
|
)
|
|
for s in s_list
|
|
]
|
|
|
|
|
|
def _split_dict(d, keys):
|
|
return project(d, keys), project(d, d.keys() - keys)
|
|
|
|
|
|
def _merge_data(s_list):
|
|
d = defaultdict(dict)
|
|
for key in s_list:
|
|
if isinstance(key, str):
|
|
d[key].update({})
|
|
continue
|
|
if not isinstance(key, dict):
|
|
raise ValueError(f"'{type(key).__name__}' not supported.")
|
|
|
|
for k, flags in key.items():
|
|
if not isinstance(flags, dict):
|
|
raise ValueError(
|
|
f"Expected dict for '{k}', got: '{type(flags).__name__}'"
|
|
)
|
|
d[k].update(flags)
|
|
return d
|
|
|
|
|
|
@collecting
|
|
def load_from_pipeline(stage, data, typ="outs"):
|
|
if typ not in (
|
|
stage.PARAM_OUTS,
|
|
stage.PARAM_METRICS,
|
|
stage.PARAM_PLOTS,
|
|
stage.PARAM_LIVE,
|
|
):
|
|
raise ValueError(f"'{typ}' key is not allowed for pipeline files.")
|
|
|
|
metric = typ == stage.PARAM_METRICS
|
|
plot = typ == stage.PARAM_PLOTS
|
|
live = typ == stage.PARAM_LIVE
|
|
|
|
if live:
|
|
# `live` is single object
|
|
data = [data]
|
|
|
|
d = _merge_data(data)
|
|
|
|
for path, flags in d.items():
|
|
plt_d, live_d = {}, {}
|
|
if plot:
|
|
from dvc.schema import PLOT_PROPS
|
|
|
|
plt_d, flags = _split_dict(flags, keys=PLOT_PROPS.keys())
|
|
if live:
|
|
from dvc.schema import LIVE_PROPS
|
|
|
|
live_d, flags = _split_dict(flags, keys=LIVE_PROPS.keys())
|
|
extra = project(
|
|
flags,
|
|
[
|
|
BaseOutput.PARAM_CACHE,
|
|
BaseOutput.PARAM_PERSIST,
|
|
BaseOutput.PARAM_CHECKPOINT,
|
|
],
|
|
)
|
|
|
|
yield _get(
|
|
stage,
|
|
path,
|
|
{},
|
|
plot=plt_d or plot,
|
|
metric=metric,
|
|
live=live_d or live,
|
|
**extra,
|
|
)
|