Files
dvc/dvc/output/__init__.py
T
Paweł RedzyńskiandRuslan Kuprieiev 862ab8de62 live logs initial (#4902)
* plots: allow dir plots

* first scenario

* separate dvclive tests

* rename index to step

* another test

* refactor

* its ok to have step in data

* sort plots

* introduce --logs

* add logs to run commands

* variable name refactoring

* fix logs option

* fixup

* cmd: run: fix unit tests

* plots: modify: fix tests

* improve dir handling

* fixup

* dvclive: logs summary

* api: introduce logs summarization method

* handle no repo case

* fix granular dir plots

* api: dvclive: simplify

* dvclive: roll back logs flag

* cmd: introduce logs

* api: adjust to logs

* test: dvclive fix

* rename logs to dvclive

* live: command: add uninitialized

* outs: dvclive type

* dvclive: support providing config via evn variables

* dvclive: treat as metrics

* dvclive integration: add support for summary on/off

* dvc integration: add temporary test for configuration export

* remove requirements

* cmd: introduce logs

* rename logs to dvclive

* cmd: introduce logs

* remove logs leftowvers

* dvclive: env: adjust to chekpoints env variables

* command: live: remove unnecessary help

* fix linter suggestions

* type hinting

* review refactor

* replace dvclive with live

* stage: loading: move live loading to separate method

* stage: serialize: allow only single live output

* live: add show/diff

* stage: serialize: roll back single live enforcing

* fixup

* rebase fixups

* output: live: force single instance

* live: command: add unit tests

* Update dvc/output/base.py

Co-authored-by: Ruslan Kuprieiev <[email protected]>

* Update dvc/stage/__init__.py

Co-authored-by: Ruslan Kuprieiev <[email protected]>

* Update dvc/stage/serialize.py

Co-authored-by: Ruslan Kuprieiev <[email protected]>

* review v2

* live: fix summary tracking by experiments

* naming fixups

* api: live: add tests

* tests: api: separate live tests

Co-authored-by: Ruslan Kuprieiev <[email protected]>
2021-01-04 11:29:41 +02:00

267 lines
6.5 KiB
Python

from collections import defaultdict
from urllib.parse import urlparse
from funcy import collecting, project
from voluptuous import And, Any, Coerce, Length, Lower, Required, SetTo
from dvc.hash_info import HashInfo
from dvc.output.base import BaseOutput
from dvc.output.gs import GSOutput
from dvc.output.hdfs import HDFSOutput
from dvc.output.local import LocalOutput
from dvc.output.s3 import S3Output
from dvc.output.ssh import SSHOutput
from dvc.output.webhdfs import WebHDFSOutput
from dvc.scheme import Schemes
from ..tree import get_cloud_tree
from ..tree.hdfs import HDFSTree
from ..tree.local import LocalTree
from ..tree.s3 import S3Tree
from ..tree.webhdfs import WebHDFSTree
OUTS = [
HDFSOutput,
S3Output,
GSOutput,
SSHOutput,
WebHDFSOutput,
# NOTE: LocalOutput is the default choice
]
OUTS_MAP = {
Schemes.HDFS: HDFSOutput,
Schemes.S3: S3Output,
Schemes.GS: GSOutput,
Schemes.SSH: SSHOutput,
Schemes.LOCAL: LocalOutput,
Schemes.WEBHDFS: WebHDFSOutput,
}
CHECKSUM_SCHEMA = Any(
None,
And(str, Length(max=0), SetTo(None)),
And(Any(str, And(int, Coerce(str))), Length(min=3), Lower),
)
# NOTE: currently there are only 3 possible checksum names:
#
# 1) md5 (LOCAL, SSH, GS);
# 2) etag (S3);
# 3) checksum (HDFS);
#
# so when a few types of outputs share the same name, we only need
# specify it once.
CHECKSUMS_SCHEMA = {
LocalTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
S3Tree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
HDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
WebHDFSTree.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
}
SCHEMA = CHECKSUMS_SCHEMA.copy()
SCHEMA[Required(BaseOutput.PARAM_PATH)] = str
SCHEMA[BaseOutput.PARAM_CACHE] = bool
SCHEMA[BaseOutput.PARAM_METRIC] = BaseOutput.METRIC_SCHEMA
SCHEMA[BaseOutput.PARAM_PLOT] = bool
SCHEMA[BaseOutput.PARAM_PERSIST] = bool
SCHEMA[BaseOutput.PARAM_CHECKPOINT] = bool
SCHEMA[HashInfo.PARAM_SIZE] = int
SCHEMA[HashInfo.PARAM_NFILES] = int
SCHEMA[BaseOutput.PARAM_DESC] = str
SCHEMA[BaseOutput.PARAM_ISEXEC] = bool
def _get(
stage,
p,
info=None,
cache=True,
metric=False,
plot=False,
persist=False,
checkpoint=False,
live=False,
desc=None,
isexec=False,
):
parsed = urlparse(p)
if parsed.scheme == "remote":
tree = get_cloud_tree(stage.repo, name=parsed.netloc)
return OUTS_MAP[tree.scheme](
stage,
p,
info,
cache=cache,
tree=tree,
metric=metric,
plot=plot,
persist=persist,
checkpoint=checkpoint,
live=live,
desc=desc,
isexec=isexec,
)
for o in OUTS:
if o.supported(p):
return o(
stage,
p,
info,
cache=cache,
tree=None,
metric=metric,
plot=plot,
persist=persist,
checkpoint=checkpoint,
live=live,
desc=desc,
isexec=isexec,
)
return LocalOutput(
stage,
p,
info,
cache=cache,
tree=None,
metric=metric,
plot=plot,
persist=persist,
checkpoint=checkpoint,
live=live,
desc=desc,
isexec=isexec,
)
def loadd_from(stage, d_list):
ret = []
for d in d_list:
p = d.pop(BaseOutput.PARAM_PATH)
cache = d.pop(BaseOutput.PARAM_CACHE, True)
metric = d.pop(BaseOutput.PARAM_METRIC, False)
plot = d.pop(BaseOutput.PARAM_PLOT, False)
persist = d.pop(BaseOutput.PARAM_PERSIST, False)
checkpoint = d.pop(BaseOutput.PARAM_CHECKPOINT, False)
desc = d.pop(BaseOutput.PARAM_DESC, False)
isexec = d.pop(BaseOutput.PARAM_ISEXEC, False)
live = d.pop(BaseOutput.PARAM_LIVE, False)
ret.append(
_get(
stage,
p,
info=d,
cache=cache,
metric=metric,
plot=plot,
persist=persist,
checkpoint=checkpoint,
desc=desc,
isexec=isexec,
live=live,
)
)
return ret
def loads_from(
stage,
s_list,
use_cache=True,
metric=False,
plot=False,
persist=False,
checkpoint=False,
isexec=False,
live=False,
):
return [
_get(
stage,
s,
info={},
cache=use_cache,
metric=metric,
plot=plot,
persist=persist,
checkpoint=checkpoint,
isexec=isexec,
live=live,
)
for s in s_list
]
def _split_dict(d, keys):
return project(d, keys), project(d, d.keys() - keys)
def _merge_data(s_list):
d = defaultdict(dict)
for key in s_list:
if isinstance(key, str):
d[key].update({})
continue
if not isinstance(key, dict):
raise ValueError(f"'{type(key).__name__}' not supported.")
for k, flags in key.items():
if not isinstance(flags, dict):
raise ValueError(
f"Expected dict for '{k}', got: '{type(flags).__name__}'"
)
d[k].update(flags)
return d
@collecting
def load_from_pipeline(stage, data, typ="outs"):
if typ not in (
stage.PARAM_OUTS,
stage.PARAM_METRICS,
stage.PARAM_PLOTS,
stage.PARAM_LIVE,
):
raise ValueError(f"'{typ}' key is not allowed for pipeline files.")
metric = typ == stage.PARAM_METRICS
plot = typ == stage.PARAM_PLOTS
live = typ == stage.PARAM_LIVE
if live:
# `live` is single object
data = [data]
d = _merge_data(data)
for path, flags in d.items():
plt_d, live_d = {}, {}
if plot:
from dvc.schema import PLOT_PROPS
plt_d, flags = _split_dict(flags, keys=PLOT_PROPS.keys())
if live:
from dvc.schema import LIVE_PROPS
live_d, flags = _split_dict(flags, keys=LIVE_PROPS.keys())
extra = project(
flags,
[
BaseOutput.PARAM_CACHE,
BaseOutput.PARAM_PERSIST,
BaseOutput.PARAM_CHECKPOINT,
],
)
yield _get(
stage,
path,
{},
plot=plt_d or plot,
metric=metric,
live=live_d or live,
**extra,
)