Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-08 15:28:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1600 from iterative/cached-metrics

Browse Source 
Cached metrics
This commit is contained in:
Ruslan Kuprieiev
2019-02-10 21:33:04 +02:00
committed by GitHub
parent fbd5393a2d 41befea210
commit 9bfff1f957
10 changed files with 156 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.pre-commit-config.yaml
+2 -2
View File
@@ -3,9 +3,9 @@ repos:
rev: stable
hooks:
- id: black
language_version: python3.6
language_version: python3
- repo: https://gitlab.com/pycqa/flake8
rev: master
hooks:
- id: flake8
language_version: python3.6
language_version: python3
dvc/command/run.py
+16 -5
View File
@@ -17,12 +17,14 @@ class CmdRun(CmdBase):
self.args.deps,
self.args.outs,
self.args.outs_no_cache,
self.args.metrics,
self.args.metrics_no_cache,
self.args.command,
]
): # pragma: no cover
logger.error(
"too few arguments. Specify at least one: '-d', "
"'-o', '-O', 'command'."
"'-o', '-O', '-m', '-M', 'command'."
)
return 1
@@ -31,6 +33,7 @@ class CmdRun(CmdBase):
cmd=self._parsed_cmd(),
outs=self.args.outs,
outs_no_cache=self.args.outs_no_cache,
metrics=self.args.metrics,
metrics_no_cache=self.args.metrics_no_cache,
deps=self.args.deps,
fname=self.args.file,
@@ -88,22 +91,30 @@ def add_parser(subparsers, parent_parser):
"--outs",
action="append",
default=[],
help="Declare output data file or data directory.",
help="Declare output file or directory.",
)
run_parser.add_argument(
"-O",
"--outs-no-cache",
action="append",
default=[],
help="Declare output regular file or directory "
"(sync to Git, not DVC cache).",
help="Declare output file or directory "
"(do not put into DVC cache).",
)
run_parser.add_argument(
"-m",
"--metrics",
action="append",
default=[],
help="Declare output metric file or directory.",
)
run_parser.add_argument(
"-M",
"--metrics-no-cache",
action="append",
default=[],
help="Declare output metric file or " "directory (not cached by DVC).",
help="Declare output metric file or directory "
"(do not put into DVC cache).",
)
run_parser.add_argument(
"-f",
dvc/exceptions.py
+2 -2
View File
@@ -11,7 +11,7 @@ class DvcException(Exception):
"""Base class for all dvc exceptions.
Args:
msg (str): message for this exception.
msg (unicode): message for this exception.
cause (Exception): optional cause exception.
"""
@@ -33,7 +33,7 @@ class OutputDuplicationError(DvcException):
stage.
Args:
output (str): path to the file/directory.
output (unicode): path to the file/directory.
stages (list): list of paths to stages.
"""
dvc/logger.py
+8 -3
View File
@@ -2,6 +2,7 @@
from __future__ import unicode_literals
from dvc.exceptions import DvcException
from dvc.utils.compat import str
import re
@@ -63,8 +64,8 @@ def box(message, border_color=None):
"""Prints a message in a box.
Args:
message (str): message to print.
border_color (str): name of a color to outline the box with.
message (unicode): message to print.
border_color (unicode): name of a color to outline the box with.
"""
lines = message.split("\n")
max_width = max(_visual_width(line) for line in lines)
@@ -230,7 +231,7 @@ def _walk_exc(exc):
tb_list.insert(0, str(exc.cause_tb))
exc = exc.cause
return (exc_list, tb_list)
return exc_list, tb_list
def _parse_exc():
@@ -260,6 +261,10 @@ def _description(message, exception):
description = "{exception}"
elif message:
description = "{message}"
else:
raise DvcException(
"Unexpected error - either exception or message must be provided"
)
return description.format(message=message, exception=exception)
dvc/project.py
+33 -19
View File
@@ -370,10 +370,11 @@ class Project(object):
def run(
self,
cmd=None,
deps=[],
outs=[],
outs_no_cache=[],
metrics_no_cache=[],
deps=None,
outs=None,
outs_no_cache=None,
metrics=None,
metrics_no_cache=None,
fname=None,
cwd=os.curdir,
no_exec=False,
@@ -383,6 +384,17 @@ class Project(object):
):
from dvc.stage import Stage
if outs is None:
outs = []
if deps is None:
deps = []
if outs_no_cache is None:
outs_no_cache = []
if metrics is None:
metrics = []
if metrics_no_cache is None:
metrics_no_cache = []
with self.state:
stage = Stage.create(
project=self,
@@ -391,6 +403,7 @@ class Project(object):
cwd=cwd,
outs=outs,
outs_no_cache=outs_no_cache,
metrics=metrics,
metrics_no_cache=metrics_no_cache,
deps=deps,
overwrite=overwrite,
@@ -1017,14 +1030,16 @@ class Project(object):
)
return self._local_status(target, with_deps=with_deps)
def _read_metric_json(self, fd, json_path):
@staticmethod
def _read_metric_json(fd, json_path):
import json
from jsonpath_rw import parse
parser = parse(json_path)
return [x.value for x in parser.find(json.load(fd))]
def _do_read_metric_xsv(self, reader, row, col):
@staticmethod
def _do_read_metric_xsv(reader, row, col):
if col is not None and row is not None:
return [reader[row][col]]
elif col is not None:
@@ -1033,22 +1048,24 @@ class Project(object):
return reader[row]
return None
def _read_metric_hxsv(self, fd, hxsv_path, delimiter):
@classmethod
def _read_metric_hxsv(cls, fd, hxsv_path, delimiter):
import csv
col, row = hxsv_path.split(",")
row = int(row)
reader = list(csv.DictReader(fd, delimiter=builtin_str(delimiter)))
return self._do_read_metric_xsv(reader, row, col)
return cls._do_read_metric_xsv(reader, row, col)
def _read_metric_xsv(self, fd, xsv_path, delimiter):
@classmethod
def _read_metric_xsv(cls, fd, xsv_path, delimiter):
import csv
col, row = xsv_path.split(",")
row = int(row)
col = int(col)
reader = list(csv.reader(fd, delimiter=builtin_str(delimiter)))
return self._do_read_metric_xsv(reader, row, col)
return cls._do_read_metric_xsv(reader, row, col)
def _read_metric(self, path, typ=None, xpath=None):
ret = None
@@ -1137,7 +1154,7 @@ class Project(object):
if os.path.isdir(path):
logger.warning(
"Path '{path}' is a directory. "
"Consider runnig with '-R'.".format(path=path)
"Consider running with '-R'.".format(path=path)
)
return {}
@@ -1182,11 +1199,10 @@ class Project(object):
return res
if path and os.path.isdir(path):
msg = "file '{}' does not exist".format(path)
return res
if path:
msg = "file '{}' does not exist".format(path)
msg = "file '{}' does not exist or malformed".format(path)
else:
msg = (
"no metric files in this repository."
@@ -1215,10 +1231,6 @@ class Project(object):
msg = "output '{}' scheme '{}' is not supported for metrics"
raise DvcException(msg.format(out.path, out.path_info["scheme"]))
if out.use_cache:
msg = "cached output '{}' is not supported for metrics"
raise DvcException(msg.format(out.rel_path))
if typ:
if not isinstance(out.metric, dict):
out.metric = {}
@@ -1366,7 +1378,8 @@ class Project(object):
stages.extend(list(nx.get_node_attributes(G, "stage").values()))
return stages
def _welcome_message(self):
@staticmethod
def _welcome_message():
import colorama
logger.box(
@@ -1390,7 +1403,8 @@ class Project(object):
)
)
def _expand_target_path(self, from_path, to_path):
@staticmethod
def _expand_target_path(from_path, to_path):
if os.path.isdir(to_path) and not os.path.isdir(from_path):
return os.path.join(to_path, os.path.basename(from_path))
return to_path
dvc/prompt.py
+1 -1
View File
@@ -36,7 +36,7 @@ def confirm(statement):
"""Ask the user for confirmation about the specified statement.
Args:
statement (str): statement to ask the user confirmation about.
statement (unicode): statement to ask the user confirmation about.
Returns:
bool: whether or not specified statement was confirmed.
dvc/stage.py
+35 -13
View File
@@ -116,11 +116,16 @@ class Stage(object):
path=None,
cmd=None,
cwd=os.curdir,
deps=[],
outs=[],
deps=None,
outs=None,
md5=None,
locked=False,
):
if deps is None:
deps = []
if outs is None:
outs = []
self.project = project
self.path = path
self.cmd = cmd
@@ -342,10 +347,11 @@ class Stage(object):
def create(
project=None,
cmd=None,
deps=[],
outs=[],
outs_no_cache=[],
metrics_no_cache=[],
deps=None,
outs=None,
outs_no_cache=None,
metrics=None,
metrics_no_cache=None,
fname=None,
cwd=os.curdir,
locked=False,
@@ -355,9 +361,23 @@ class Stage(object):
remove_outs=False,
):
if outs is None:
outs = []
if deps is None:
deps = []
if outs_no_cache is None:
outs_no_cache = []
if metrics is None:
metrics = []
if metrics_no_cache is None:
metrics_no_cache = []
stage = Stage(project=project, cwd=cwd, cmd=cmd, locked=locked)
stage.outs = output.loads_from(stage, outs, use_cache=True)
stage.outs += output.loads_from(
stage, metrics, use_cache=True, metric=True
)
stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False)
stage.outs += output.loads_from(
stage, metrics_no_cache, use_cache=False, metric=True
@@ -512,7 +532,8 @@ class Stage(object):
if any(missing):
raise MissingDep(missing)
def _warn_if_fish(self, executable): # pragma: no cover
@staticmethod
def _warn_if_fish(executable): # pragma: no cover
if (
executable is None
or os.path.basename(os.path.realpath(executable)) != "fish"
@@ -530,8 +551,8 @@ class Stage(object):
def _check_circular_dependency(self):
from dvc.exceptions import CircularDependencyError
circular_dependencies = set(file.path for file in self.deps) & set(
file.path for file in self.outs
circular_dependencies = set(d.path for d in self.deps) & set(
o.path for o in self.outs
)
if circular_dependencies:
@@ -541,11 +562,11 @@ class Stage(object):
from dvc.exceptions import ArgumentDuplicationError
from collections import Counter
count = Counter(file.path for file in self.deps + self.outs)
path_counts = Counter(edge.path for edge in self.deps + self.outs)
for file, occurrence in count.items():
for path, occurrence in path_counts.items():
if occurrence > 1:
raise ArgumentDuplicationError(file)
raise ArgumentDuplicationError(path)
def _run(self):
self._check_missing_deps()
@@ -619,7 +640,8 @@ class Stage(object):
for out in self.outs:
out.checkout(force=force)
def _status(self, entries, name):
@staticmethod
def _status(entries, name):
ret = {}
for entry in entries:
tests/__main__.py
+9 -1
View File
@@ -1,6 +1,8 @@
import os
import sys
from subprocess import check_call
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
os.chdir(REPO_ROOT)
@@ -10,8 +12,14 @@ os.putenv(
os.putenv("DVC_HOME", REPO_ROOT)
os.putenv("DVC_TEST", "true")
if len(sys.argv) == 1:
scope = "--all-modules"
else:
scope = " ".join(sys.argv[1:])
cmd = (
"nosetests -v --processes=-1 --process-timeout=500 --cover-inclusive "
"--cover-erase --cover-package=dvc --with-coverage --all-modules --with-flaky"
"--cover-erase --cover-package=dvc --with-coverage --with-flaky "
"{scope} ".format(scope=scope)
)
check_call(cmd, shell=True)
tests/test_metrics.py
+37 -6
View File
@@ -359,7 +359,7 @@ class TestNoMetrics(TestDvc):
class TestCachedMetrics(TestDvc):
def _do_write(self, branch):
def _do_add(self, branch):
self.dvc.scm.checkout(branch)
self.dvc.checkout(force=True)
@@ -369,20 +369,45 @@ class TestCachedMetrics(TestDvc):
stages = self.dvc.add("metrics.json")
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertNotEqual(stage, None)
self.assertIsNotNone(stage)
self.dvc.scm.add([".gitignore", "metrics.json.dvc"])
self.dvc.scm.commit(branch)
def test(self):
def _do_run(self, branch):
self.dvc.scm.checkout(branch)
self.dvc.checkout(force=True)
with open("code.py", "w+") as fobj:
fobj.write("import sys\n")
fobj.write("import os\n")
fobj.write("import json\n")
fobj.write(
'print(json.dumps({{"metrics": "{branch}"}}))\n'.format(
branch=branch
)
)
stage = self.dvc.run(
deps=["code.py"],
metrics=["metrics.json"],
cmd="python code.py metrics.json > metrics.json",
)
self.assertIsNotNone(stage)
self.assertEqual(stage.relpath, "metrics.json.dvc")
self.dvc.scm.add([".gitignore", "metrics.json.dvc"])
self.dvc.scm.commit(branch)
def _test_metrics(self, func):
self.dvc.scm.commit("init")
self.dvc.scm.branch("one")
self.dvc.scm.branch("two")
self._do_write("master")
self._do_write("one")
self._do_write("two")
func("master")
func("one")
func("two")
self.dvc = Project(".")
@@ -398,3 +423,9 @@ class TestCachedMetrics(TestDvc):
"two": {"metrics.json": ["two"]},
},
)
def test_add(self):
self._test_metrics(self._do_add)
def test_run(self):
self._test_metrics(self._do_run)
tests/test_run.py
+13 -1
View File
@@ -50,7 +50,7 @@ class TestRun(TestDvc):
self.assertTrue(stage.path, fname)
with self.assertRaises(OutputDuplicationError):
stage = self.dvc.run(
self.dvc.run(
cmd=cmd,
deps=deps,
outs=outs,
@@ -530,6 +530,18 @@ class TestCmdRunOverwrite(TestDvc):
self.assertNotEqual(stage_mtime, os.path.getmtime("out.dvc"))
class TestCmdRunCliMetrics(TestDvc):
def test_cached(self):
ret = main(["run", "-m", "metrics.txt", "echo test > metrics.txt"])
self.assertEqual(ret, 0)
self.assertEqual(open("metrics.txt", "r").read().rstrip(), "test")
def test_not_cached(self):
ret = main(["run", "-M", "metrics.txt", "echo test > metrics.txt"])
self.assertEqual(ret, 0)
self.assertEqual(open("metrics.txt", "r").read().rstrip(), "test")
class TestRunDeterministicBase(TestDvc):
def setUp(self):
super(TestRunDeterministicBase, self).setUp()