diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3eff137df..62f0c4c83 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,9 +3,9 @@ repos: rev: stable hooks: - id: black - language_version: python3.6 + language_version: python3 - repo: https://gitlab.com/pycqa/flake8 rev: master hooks: - id: flake8 - language_version: python3.6 + language_version: python3 diff --git a/dvc/command/run.py b/dvc/command/run.py index b516dba49..c070f2708 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -17,12 +17,14 @@ class CmdRun(CmdBase): self.args.deps, self.args.outs, self.args.outs_no_cache, + self.args.metrics, + self.args.metrics_no_cache, self.args.command, ] ): # pragma: no cover logger.error( "too few arguments. Specify at least one: '-d', " - "'-o', '-O', 'command'." + "'-o', '-O', '-m', '-M', 'command'." ) return 1 @@ -31,6 +33,7 @@ class CmdRun(CmdBase): cmd=self._parsed_cmd(), outs=self.args.outs, outs_no_cache=self.args.outs_no_cache, + metrics=self.args.metrics, metrics_no_cache=self.args.metrics_no_cache, deps=self.args.deps, fname=self.args.file, @@ -88,22 +91,30 @@ def add_parser(subparsers, parent_parser): "--outs", action="append", default=[], - help="Declare output data file or data directory.", + help="Declare output file or directory.", ) run_parser.add_argument( "-O", "--outs-no-cache", action="append", default=[], - help="Declare output regular file or directory " - "(sync to Git, not DVC cache).", + help="Declare output file or directory " + "(do not put into DVC cache).", + ) + run_parser.add_argument( + "-m", + "--metrics", + action="append", + default=[], + help="Declare output metric file or directory.", ) run_parser.add_argument( "-M", "--metrics-no-cache", action="append", default=[], - help="Declare output metric file or " "directory (not cached by DVC).", + help="Declare output metric file or directory " + "(do not put into DVC cache).", ) run_parser.add_argument( "-f", diff --git a/dvc/exceptions.py b/dvc/exceptions.py index aecd8249b..a152355fd 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -11,7 +11,7 @@ class DvcException(Exception): """Base class for all dvc exceptions. Args: - msg (str): message for this exception. + msg (unicode): message for this exception. cause (Exception): optional cause exception. """ @@ -33,7 +33,7 @@ class OutputDuplicationError(DvcException): stage. Args: - output (str): path to the file/directory. + output (unicode): path to the file/directory. stages (list): list of paths to stages. """ diff --git a/dvc/logger.py b/dvc/logger.py index 824436a50..fcc0663fd 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals +from dvc.exceptions import DvcException from dvc.utils.compat import str import re @@ -63,8 +64,8 @@ def box(message, border_color=None): """Prints a message in a box. Args: - message (str): message to print. - border_color (str): name of a color to outline the box with. + message (unicode): message to print. + border_color (unicode): name of a color to outline the box with. """ lines = message.split("\n") max_width = max(_visual_width(line) for line in lines) @@ -230,7 +231,7 @@ def _walk_exc(exc): tb_list.insert(0, str(exc.cause_tb)) exc = exc.cause - return (exc_list, tb_list) + return exc_list, tb_list def _parse_exc(): @@ -260,6 +261,10 @@ def _description(message, exception): description = "{exception}" elif message: description = "{message}" + else: + raise DvcException( + "Unexpected error - either exception or message must be provided" + ) return description.format(message=message, exception=exception) diff --git a/dvc/project.py b/dvc/project.py index 363c6b275..a81309e7f 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -370,10 +370,11 @@ class Project(object): def run( self, cmd=None, - deps=[], - outs=[], - outs_no_cache=[], - metrics_no_cache=[], + deps=None, + outs=None, + outs_no_cache=None, + metrics=None, + metrics_no_cache=None, fname=None, cwd=os.curdir, no_exec=False, @@ -383,6 +384,17 @@ class Project(object): ): from dvc.stage import Stage + if outs is None: + outs = [] + if deps is None: + deps = [] + if outs_no_cache is None: + outs_no_cache = [] + if metrics is None: + metrics = [] + if metrics_no_cache is None: + metrics_no_cache = [] + with self.state: stage = Stage.create( project=self, @@ -391,6 +403,7 @@ class Project(object): cwd=cwd, outs=outs, outs_no_cache=outs_no_cache, + metrics=metrics, metrics_no_cache=metrics_no_cache, deps=deps, overwrite=overwrite, @@ -1017,14 +1030,16 @@ class Project(object): ) return self._local_status(target, with_deps=with_deps) - def _read_metric_json(self, fd, json_path): + @staticmethod + def _read_metric_json(fd, json_path): import json from jsonpath_rw import parse parser = parse(json_path) return [x.value for x in parser.find(json.load(fd))] - def _do_read_metric_xsv(self, reader, row, col): + @staticmethod + def _do_read_metric_xsv(reader, row, col): if col is not None and row is not None: return [reader[row][col]] elif col is not None: @@ -1033,22 +1048,24 @@ class Project(object): return reader[row] return None - def _read_metric_hxsv(self, fd, hxsv_path, delimiter): + @classmethod + def _read_metric_hxsv(cls, fd, hxsv_path, delimiter): import csv col, row = hxsv_path.split(",") row = int(row) reader = list(csv.DictReader(fd, delimiter=builtin_str(delimiter))) - return self._do_read_metric_xsv(reader, row, col) + return cls._do_read_metric_xsv(reader, row, col) - def _read_metric_xsv(self, fd, xsv_path, delimiter): + @classmethod + def _read_metric_xsv(cls, fd, xsv_path, delimiter): import csv col, row = xsv_path.split(",") row = int(row) col = int(col) reader = list(csv.reader(fd, delimiter=builtin_str(delimiter))) - return self._do_read_metric_xsv(reader, row, col) + return cls._do_read_metric_xsv(reader, row, col) def _read_metric(self, path, typ=None, xpath=None): ret = None @@ -1137,7 +1154,7 @@ class Project(object): if os.path.isdir(path): logger.warning( "Path '{path}' is a directory. " - "Consider runnig with '-R'.".format(path=path) + "Consider running with '-R'.".format(path=path) ) return {} @@ -1182,11 +1199,10 @@ class Project(object): return res if path and os.path.isdir(path): - msg = "file '{}' does not exist".format(path) return res if path: - msg = "file '{}' does not exist".format(path) + msg = "file '{}' does not exist or malformed".format(path) else: msg = ( "no metric files in this repository." @@ -1215,10 +1231,6 @@ class Project(object): msg = "output '{}' scheme '{}' is not supported for metrics" raise DvcException(msg.format(out.path, out.path_info["scheme"])) - if out.use_cache: - msg = "cached output '{}' is not supported for metrics" - raise DvcException(msg.format(out.rel_path)) - if typ: if not isinstance(out.metric, dict): out.metric = {} @@ -1366,7 +1378,8 @@ class Project(object): stages.extend(list(nx.get_node_attributes(G, "stage").values())) return stages - def _welcome_message(self): + @staticmethod + def _welcome_message(): import colorama logger.box( @@ -1390,7 +1403,8 @@ class Project(object): ) ) - def _expand_target_path(self, from_path, to_path): + @staticmethod + def _expand_target_path(from_path, to_path): if os.path.isdir(to_path) and not os.path.isdir(from_path): return os.path.join(to_path, os.path.basename(from_path)) return to_path diff --git a/dvc/prompt.py b/dvc/prompt.py index 6853ce40b..898eb921b 100644 --- a/dvc/prompt.py +++ b/dvc/prompt.py @@ -36,7 +36,7 @@ def confirm(statement): """Ask the user for confirmation about the specified statement. Args: - statement (str): statement to ask the user confirmation about. + statement (unicode): statement to ask the user confirmation about. Returns: bool: whether or not specified statement was confirmed. diff --git a/dvc/stage.py b/dvc/stage.py index 8f20d0bd3..8e885befc 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -116,11 +116,16 @@ class Stage(object): path=None, cmd=None, cwd=os.curdir, - deps=[], - outs=[], + deps=None, + outs=None, md5=None, locked=False, ): + if deps is None: + deps = [] + if outs is None: + outs = [] + self.project = project self.path = path self.cmd = cmd @@ -342,10 +347,11 @@ class Stage(object): def create( project=None, cmd=None, - deps=[], - outs=[], - outs_no_cache=[], - metrics_no_cache=[], + deps=None, + outs=None, + outs_no_cache=None, + metrics=None, + metrics_no_cache=None, fname=None, cwd=os.curdir, locked=False, @@ -355,9 +361,23 @@ class Stage(object): remove_outs=False, ): + if outs is None: + outs = [] + if deps is None: + deps = [] + if outs_no_cache is None: + outs_no_cache = [] + if metrics is None: + metrics = [] + if metrics_no_cache is None: + metrics_no_cache = [] + stage = Stage(project=project, cwd=cwd, cmd=cmd, locked=locked) stage.outs = output.loads_from(stage, outs, use_cache=True) + stage.outs += output.loads_from( + stage, metrics, use_cache=True, metric=True + ) stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False) stage.outs += output.loads_from( stage, metrics_no_cache, use_cache=False, metric=True @@ -512,7 +532,8 @@ class Stage(object): if any(missing): raise MissingDep(missing) - def _warn_if_fish(self, executable): # pragma: no cover + @staticmethod + def _warn_if_fish(executable): # pragma: no cover if ( executable is None or os.path.basename(os.path.realpath(executable)) != "fish" @@ -530,8 +551,8 @@ class Stage(object): def _check_circular_dependency(self): from dvc.exceptions import CircularDependencyError - circular_dependencies = set(file.path for file in self.deps) & set( - file.path for file in self.outs + circular_dependencies = set(d.path for d in self.deps) & set( + o.path for o in self.outs ) if circular_dependencies: @@ -541,11 +562,11 @@ class Stage(object): from dvc.exceptions import ArgumentDuplicationError from collections import Counter - count = Counter(file.path for file in self.deps + self.outs) + path_counts = Counter(edge.path for edge in self.deps + self.outs) - for file, occurrence in count.items(): + for path, occurrence in path_counts.items(): if occurrence > 1: - raise ArgumentDuplicationError(file) + raise ArgumentDuplicationError(path) def _run(self): self._check_missing_deps() @@ -619,7 +640,8 @@ class Stage(object): for out in self.outs: out.checkout(force=force) - def _status(self, entries, name): + @staticmethod + def _status(entries, name): ret = {} for entry in entries: diff --git a/tests/__main__.py b/tests/__main__.py index 9612fa198..ab9e891e0 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -1,6 +1,8 @@ import os +import sys from subprocess import check_call + REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) os.chdir(REPO_ROOT) @@ -10,8 +12,14 @@ os.putenv( os.putenv("DVC_HOME", REPO_ROOT) os.putenv("DVC_TEST", "true") +if len(sys.argv) == 1: + scope = "--all-modules" +else: + scope = " ".join(sys.argv[1:]) + cmd = ( "nosetests -v --processes=-1 --process-timeout=500 --cover-inclusive " - "--cover-erase --cover-package=dvc --with-coverage --all-modules --with-flaky" + "--cover-erase --cover-package=dvc --with-coverage --with-flaky " + "{scope} ".format(scope=scope) ) check_call(cmd, shell=True) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index c76704518..77ff4e355 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -359,7 +359,7 @@ class TestNoMetrics(TestDvc): class TestCachedMetrics(TestDvc): - def _do_write(self, branch): + def _do_add(self, branch): self.dvc.scm.checkout(branch) self.dvc.checkout(force=True) @@ -369,20 +369,45 @@ class TestCachedMetrics(TestDvc): stages = self.dvc.add("metrics.json") self.assertEqual(len(stages), 1) stage = stages[0] - self.assertNotEqual(stage, None) + self.assertIsNotNone(stage) self.dvc.scm.add([".gitignore", "metrics.json.dvc"]) self.dvc.scm.commit(branch) - def test(self): + def _do_run(self, branch): + self.dvc.scm.checkout(branch) + self.dvc.checkout(force=True) + + with open("code.py", "w+") as fobj: + fobj.write("import sys\n") + fobj.write("import os\n") + fobj.write("import json\n") + fobj.write( + 'print(json.dumps({{"metrics": "{branch}"}}))\n'.format( + branch=branch + ) + ) + + stage = self.dvc.run( + deps=["code.py"], + metrics=["metrics.json"], + cmd="python code.py metrics.json > metrics.json", + ) + self.assertIsNotNone(stage) + self.assertEqual(stage.relpath, "metrics.json.dvc") + + self.dvc.scm.add([".gitignore", "metrics.json.dvc"]) + self.dvc.scm.commit(branch) + + def _test_metrics(self, func): self.dvc.scm.commit("init") self.dvc.scm.branch("one") self.dvc.scm.branch("two") - self._do_write("master") - self._do_write("one") - self._do_write("two") + func("master") + func("one") + func("two") self.dvc = Project(".") @@ -398,3 +423,9 @@ class TestCachedMetrics(TestDvc): "two": {"metrics.json": ["two"]}, }, ) + + def test_add(self): + self._test_metrics(self._do_add) + + def test_run(self): + self._test_metrics(self._do_run) diff --git a/tests/test_run.py b/tests/test_run.py index d68ce88de..97d979f28 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -50,7 +50,7 @@ class TestRun(TestDvc): self.assertTrue(stage.path, fname) with self.assertRaises(OutputDuplicationError): - stage = self.dvc.run( + self.dvc.run( cmd=cmd, deps=deps, outs=outs, @@ -530,6 +530,18 @@ class TestCmdRunOverwrite(TestDvc): self.assertNotEqual(stage_mtime, os.path.getmtime("out.dvc")) +class TestCmdRunCliMetrics(TestDvc): + def test_cached(self): + ret = main(["run", "-m", "metrics.txt", "echo test > metrics.txt"]) + self.assertEqual(ret, 0) + self.assertEqual(open("metrics.txt", "r").read().rstrip(), "test") + + def test_not_cached(self): + ret = main(["run", "-M", "metrics.txt", "echo test > metrics.txt"]) + self.assertEqual(ret, 0) + self.assertEqual(open("metrics.txt", "r").read().rstrip(), "test") + + class TestRunDeterministicBase(TestDvc): def setUp(self): super(TestRunDeterministicBase, self).setUp()