From 16570bad039d05df1cb8c94b086746479f784428 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 9 Feb 2019 13:48:18 -0800 Subject: [PATCH 01/12] add cached metrics, cleanup warnings --- dvc/command/run.py | 10 ++++++++- dvc/exceptions.py | 4 ++-- dvc/logger.py | 13 +++++++----- dvc/project.py | 48 +++++++++++++++++++++++++++--------------- dvc/prompt.py | 2 +- dvc/stage.py | 52 ++++++++++++++++++++++++++++++++-------------- 6 files changed, 87 insertions(+), 42 deletions(-) diff --git a/dvc/command/run.py b/dvc/command/run.py index b516dba49..c60115721 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -31,6 +31,7 @@ class CmdRun(CmdBase): cmd=self._parsed_cmd(), outs=self.args.outs, outs_no_cache=self.args.outs_no_cache, + metrics=self.args.metrics, metrics_no_cache=self.args.metrics_no_cache, deps=self.args.deps, fname=self.args.file, @@ -95,9 +96,16 @@ def add_parser(subparsers, parent_parser): "--outs-no-cache", action="append", default=[], - help="Declare output regular file or directory " + help="Declare output data file or directory " "(sync to Git, not DVC cache).", ) + run_parser.add_argument( + "-m", + "--metrics", + action="append", + default=[], + help="Declare output metric file or directory.", + ) run_parser.add_argument( "-M", "--metrics-no-cache", diff --git a/dvc/exceptions.py b/dvc/exceptions.py index aecd8249b..a152355fd 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -11,7 +11,7 @@ class DvcException(Exception): """Base class for all dvc exceptions. Args: - msg (str): message for this exception. + msg (unicode): message for this exception. cause (Exception): optional cause exception. """ @@ -33,7 +33,7 @@ class OutputDuplicationError(DvcException): stage. Args: - output (str): path to the file/directory. + output (unicode): path to the file/directory. stages (list): list of paths to stages. """ diff --git a/dvc/logger.py b/dvc/logger.py index 824436a50..e51e854ed 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals +from dvc.exceptions import DvcException from dvc.utils.compat import str import re @@ -63,8 +64,8 @@ def box(message, border_color=None): """Prints a message in a box. Args: - message (str): message to print. - border_color (str): name of a color to outline the box with. + message (unicode): message to print. + border_color (unicode): name of a color to outline the box with. """ lines = message.split("\n") max_width = max(_visual_width(line) for line in lines) @@ -230,13 +231,13 @@ def _walk_exc(exc): tb_list.insert(0, str(exc.cause_tb)) exc = exc.cause - return (exc_list, tb_list) + return exc_list, tb_list def _parse_exc(): exc = sys.exc_info()[1] if not exc: - return (None, "") + return None, "" exc_list, tb_list = _walk_exc(exc) @@ -250,7 +251,7 @@ def _parse_exc(): else: stack_trace = "" - return (exception, stack_trace) + return exception, stack_trace def _description(message, exception): @@ -260,6 +261,8 @@ def _description(message, exception): description = "{exception}" elif message: description = "{message}" + else: + raise DvcException('Unexpected error - either exception or message must be provided') return description.format(message=message, exception=exception) diff --git a/dvc/project.py b/dvc/project.py index 363c6b275..864928b1f 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -370,10 +370,11 @@ class Project(object): def run( self, cmd=None, - deps=[], - outs=[], - outs_no_cache=[], - metrics_no_cache=[], + deps=None, + outs=None, + outs_no_cache=None, + metrics=None, + metrics_no_cache=None, fname=None, cwd=os.curdir, no_exec=False, @@ -383,6 +384,17 @@ class Project(object): ): from dvc.stage import Stage + if outs is None: + outs = [] + if deps is None: + deps = [] + if outs_no_cache is None: + outs_no_cache = [] + if metrics is None: + metrics = [] + if metrics_no_cache is None: + metrics_no_cache = [] + with self.state: stage = Stage.create( project=self, @@ -391,6 +403,7 @@ class Project(object): cwd=cwd, outs=outs, outs_no_cache=outs_no_cache, + metrics=metrics, metrics_no_cache=metrics_no_cache, deps=deps, overwrite=overwrite, @@ -1017,14 +1030,16 @@ class Project(object): ) return self._local_status(target, with_deps=with_deps) - def _read_metric_json(self, fd, json_path): + @staticmethod + def _read_metric_json(fd, json_path): import json from jsonpath_rw import parse parser = parse(json_path) return [x.value for x in parser.find(json.load(fd))] - def _do_read_metric_xsv(self, reader, row, col): + @staticmethod + def _do_read_metric_xsv(reader, row, col): if col is not None and row is not None: return [reader[row][col]] elif col is not None: @@ -1033,22 +1048,24 @@ class Project(object): return reader[row] return None - def _read_metric_hxsv(self, fd, hxsv_path, delimiter): + @classmethod + def _read_metric_hxsv(cls, fd, hxsv_path, delimiter): import csv col, row = hxsv_path.split(",") row = int(row) reader = list(csv.DictReader(fd, delimiter=builtin_str(delimiter))) - return self._do_read_metric_xsv(reader, row, col) + return cls._do_read_metric_xsv(reader, row, col) - def _read_metric_xsv(self, fd, xsv_path, delimiter): + @classmethod + def _read_metric_xsv(cls, fd, xsv_path, delimiter): import csv col, row = xsv_path.split(",") row = int(row) col = int(col) reader = list(csv.reader(fd, delimiter=builtin_str(delimiter))) - return self._do_read_metric_xsv(reader, row, col) + return cls._do_read_metric_xsv(reader, row, col) def _read_metric(self, path, typ=None, xpath=None): ret = None @@ -1182,7 +1199,6 @@ class Project(object): return res if path and os.path.isdir(path): - msg = "file '{}' does not exist".format(path) return res if path: @@ -1215,10 +1231,6 @@ class Project(object): msg = "output '{}' scheme '{}' is not supported for metrics" raise DvcException(msg.format(out.path, out.path_info["scheme"])) - if out.use_cache: - msg = "cached output '{}' is not supported for metrics" - raise DvcException(msg.format(out.rel_path)) - if typ: if not isinstance(out.metric, dict): out.metric = {} @@ -1366,7 +1378,8 @@ class Project(object): stages.extend(list(nx.get_node_attributes(G, "stage").values())) return stages - def _welcome_message(self): + @staticmethod + def _welcome_message(): import colorama logger.box( @@ -1390,7 +1403,8 @@ class Project(object): ) ) - def _expand_target_path(self, from_path, to_path): + @staticmethod + def _expand_target_path(from_path, to_path): if os.path.isdir(to_path) and not os.path.isdir(from_path): return os.path.join(to_path, os.path.basename(from_path)) return to_path diff --git a/dvc/prompt.py b/dvc/prompt.py index 6853ce40b..898eb921b 100644 --- a/dvc/prompt.py +++ b/dvc/prompt.py @@ -36,7 +36,7 @@ def confirm(statement): """Ask the user for confirmation about the specified statement. Args: - statement (str): statement to ask the user confirmation about. + statement (unicode): statement to ask the user confirmation about. Returns: bool: whether or not specified statement was confirmed. diff --git a/dvc/stage.py b/dvc/stage.py index 8f20d0bd3..1f41384ce 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -116,11 +116,16 @@ class Stage(object): path=None, cmd=None, cwd=os.curdir, - deps=[], - outs=[], + deps=None, + outs=None, md5=None, locked=False, ): + if deps is None: + deps = [] + if outs is None: + outs = [] + self.project = project self.path = path self.cmd = cmd @@ -281,10 +286,10 @@ class Stage(object): @classmethod def _stage_fname_cwd(cls, fname, cwd, outs, add): if fname and cwd: - return (fname, cwd) + return fname, cwd if not outs: - return (cls.STAGE_FILE, cwd if cwd else os.getcwd()) + return cls.STAGE_FILE, cwd if cwd else os.getcwd() out = outs[0] if out.scheme == "local": @@ -298,7 +303,7 @@ class Stage(object): if not cwd or (add and out.is_local): cwd = path.dirname(out.path) - return (fname, cwd) + return fname, cwd @staticmethod def _check_inside_project(project, cwd): @@ -342,10 +347,11 @@ class Stage(object): def create( project=None, cmd=None, - deps=[], - outs=[], - outs_no_cache=[], - metrics_no_cache=[], + deps=None, + outs=None, + outs_no_cache=None, + metrics=None, + metrics_no_cache=None, fname=None, cwd=os.curdir, locked=False, @@ -355,9 +361,21 @@ class Stage(object): remove_outs=False, ): + if outs is None: + outs = [] + if deps is None: + deps = [] + if outs_no_cache is None: + outs_no_cache = [] + if metrics is None: + metrics = [] + if metrics_no_cache is None: + metrics_no_cache = [] + stage = Stage(project=project, cwd=cwd, cmd=cmd, locked=locked) stage.outs = output.loads_from(stage, outs, use_cache=True) + stage.outs = output.loads_from(stage, metrics, use_cache=True, metric=True) stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False) stage.outs += output.loads_from( stage, metrics_no_cache, use_cache=False, metric=True @@ -512,7 +530,8 @@ class Stage(object): if any(missing): raise MissingDep(missing) - def _warn_if_fish(self, executable): # pragma: no cover + @staticmethod + def _warn_if_fish(executable): # pragma: no cover if ( executable is None or os.path.basename(os.path.realpath(executable)) != "fish" @@ -530,8 +549,8 @@ class Stage(object): def _check_circular_dependency(self): from dvc.exceptions import CircularDependencyError - circular_dependencies = set(file.path for file in self.deps) & set( - file.path for file in self.outs + circular_dependencies = set(f.path for f in self.deps) & set( + f.path for f in self.outs ) if circular_dependencies: @@ -541,11 +560,11 @@ class Stage(object): from dvc.exceptions import ArgumentDuplicationError from collections import Counter - count = Counter(file.path for file in self.deps + self.outs) + count = Counter(f.path for f in self.deps + self.outs) - for file, occurrence in count.items(): + for f, occurrence in count.items(): if occurrence > 1: - raise ArgumentDuplicationError(file) + raise ArgumentDuplicationError(f) def _run(self): self._check_missing_deps() @@ -619,7 +638,8 @@ class Stage(object): for out in self.outs: out.checkout(force=force) - def _status(self, entries, name): + @staticmethod + def _status(entries, name): ret = {} for entry in entries: From 2419f94ef35521ab08490e8d7bf0eabfa0f0f0c4 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 9 Feb 2019 15:00:16 -0800 Subject: [PATCH 02/12] use python3 in hook --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3eff137df..62f0c4c83 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,9 +3,9 @@ repos: rev: stable hooks: - id: black - language_version: python3.6 + language_version: python3 - repo: https://gitlab.com/pycqa/flake8 rev: master hooks: - id: flake8 - language_version: python3.6 + language_version: python3 From 0ed44cf151531c642b269a7cc9ae9bea84db1d57 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 9 Feb 2019 15:44:12 -0800 Subject: [PATCH 03/12] fix formatting --- dvc/logger.py | 4 +++- dvc/stage.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dvc/logger.py b/dvc/logger.py index e51e854ed..108571706 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -262,7 +262,9 @@ def _description(message, exception): elif message: description = "{message}" else: - raise DvcException('Unexpected error - either exception or message must be provided') + raise DvcException( + "Unexpected error - either exception or message must be provided" + ) return description.format(message=message, exception=exception) diff --git a/dvc/stage.py b/dvc/stage.py index 1f41384ce..574d3e678 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -375,7 +375,9 @@ class Stage(object): stage = Stage(project=project, cwd=cwd, cmd=cmd, locked=locked) stage.outs = output.loads_from(stage, outs, use_cache=True) - stage.outs = output.loads_from(stage, metrics, use_cache=True, metric=True) + stage.outs += output.loads_from( + stage, metrics, use_cache=True, metric=True + ) stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False) stage.outs += output.loads_from( stage, metrics_no_cache, use_cache=False, metric=True From 3287c3f40762e9cb99927b3f89828f50942eb48f Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 9 Feb 2019 18:52:46 -0800 Subject: [PATCH 04/12] add ability run tests from specified file(s) --- tests/__main__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/__main__.py b/tests/__main__.py index 9612fa198..ab9e891e0 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -1,6 +1,8 @@ import os +import sys from subprocess import check_call + REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) os.chdir(REPO_ROOT) @@ -10,8 +12,14 @@ os.putenv( os.putenv("DVC_HOME", REPO_ROOT) os.putenv("DVC_TEST", "true") +if len(sys.argv) == 1: + scope = "--all-modules" +else: + scope = " ".join(sys.argv[1:]) + cmd = ( "nosetests -v --processes=-1 --process-timeout=500 --cover-inclusive " - "--cover-erase --cover-package=dvc --with-coverage --all-modules --with-flaky" + "--cover-erase --cover-package=dvc --with-coverage --with-flaky " + "{scope} ".format(scope=scope) ) check_call(cmd, shell=True) From dab347f42456a55a1b6f7c356511c883e1488f35 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 9 Feb 2019 18:54:54 -0800 Subject: [PATCH 05/12] fix few checks and messages in dvc run --- dvc/command/run.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dvc/command/run.py b/dvc/command/run.py index c60115721..a978b8daf 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -17,12 +17,14 @@ class CmdRun(CmdBase): self.args.deps, self.args.outs, self.args.outs_no_cache, + self.args.metrics, + self.args.metrics_no_cache, self.args.command, ] ): # pragma: no cover logger.error( "too few arguments. Specify at least one: '-d', " - "'-o', '-O', 'command'." + "'-o', '-O', '-m', '-M', 'command'." ) return 1 From 17c6858ad38359addf6c68928a4cd0b3858f0d6a Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 9 Feb 2019 18:56:17 -0800 Subject: [PATCH 06/12] minor: better variable names in stage class --- dvc/stage.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dvc/stage.py b/dvc/stage.py index 574d3e678..cc1888d33 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -551,8 +551,8 @@ class Stage(object): def _check_circular_dependency(self): from dvc.exceptions import CircularDependencyError - circular_dependencies = set(f.path for f in self.deps) & set( - f.path for f in self.outs + circular_dependencies = set(d.path for d in self.deps) & set( + o.path for o in self.outs ) if circular_dependencies: @@ -562,11 +562,11 @@ class Stage(object): from dvc.exceptions import ArgumentDuplicationError from collections import Counter - count = Counter(f.path for f in self.deps + self.outs) + path_counts = Counter(edge.path for edge in self.deps + self.outs) - for f, occurrence in count.items(): + for path, occurrence in path_counts.items(): if occurrence > 1: - raise ArgumentDuplicationError(f) + raise ArgumentDuplicationError(path) def _run(self): self._check_missing_deps() From dc1242a107b35d777b7366d4411e8f2208ac33bc Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 9 Feb 2019 18:57:15 -0800 Subject: [PATCH 07/12] fix error messages in metrics show --- dvc/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dvc/project.py b/dvc/project.py index 864928b1f..a81309e7f 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -1154,7 +1154,7 @@ class Project(object): if os.path.isdir(path): logger.warning( "Path '{path}' is a directory. " - "Consider runnig with '-R'.".format(path=path) + "Consider running with '-R'.".format(path=path) ) return {} @@ -1202,7 +1202,7 @@ class Project(object): return res if path: - msg = "file '{}' does not exist".format(path) + msg = "file '{}' does not exist or malformed".format(path) else: msg = ( "no metric files in this repository." From 36c4a3a5af38121bc02911419683105d828e4ba3 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 9 Feb 2019 19:00:40 -0800 Subject: [PATCH 08/12] add metrics tests --- tests/test_metrics.py | 33 +++++++++++++++++++++++++++------ tests/test_run.py | 18 +++++++++++++++++- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index c76704518..ce3d088f1 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -359,7 +359,7 @@ class TestNoMetrics(TestDvc): class TestCachedMetrics(TestDvc): - def _do_write(self, branch): + def _do_add(self, branch): self.dvc.scm.checkout(branch) self.dvc.checkout(force=True) @@ -369,20 +369,35 @@ class TestCachedMetrics(TestDvc): stages = self.dvc.add("metrics.json") self.assertEqual(len(stages), 1) stage = stages[0] - self.assertNotEqual(stage, None) + self.assertIsNotNone(stage) self.dvc.scm.add([".gitignore", "metrics.json.dvc"]) self.dvc.scm.commit(branch) - def test(self): + def _do_run(self, branch): + self.dvc.scm.checkout(branch) + self.dvc.checkout(force=True) + + content = json.dumps({"metrics": branch}) + stage = self.dvc.run( + metrics=["metrics.json"], + cmd="echo '{content}' > metrics.json".format(content=content), + ) + self.assertIsNotNone(stage) + self.assertEqual(stage.relpath, "metrics.json.dvc") + + self.dvc.scm.add([".gitignore", "metrics.json.dvc"]) + self.dvc.scm.commit(branch) + + def _test_metrics(self, func): self.dvc.scm.commit("init") self.dvc.scm.branch("one") self.dvc.scm.branch("two") - self._do_write("master") - self._do_write("one") - self._do_write("two") + func("master") + func("one") + func("two") self.dvc = Project(".") @@ -398,3 +413,9 @@ class TestCachedMetrics(TestDvc): "two": {"metrics.json": ["two"]}, }, ) + + def test_add(self): + self._test_metrics(self._do_add) + + def test_run(self): + self._test_metrics(self._do_run) diff --git a/tests/test_run.py b/tests/test_run.py index d68ce88de..5e6355500 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -50,7 +50,7 @@ class TestRun(TestDvc): self.assertTrue(stage.path, fname) with self.assertRaises(OutputDuplicationError): - stage = self.dvc.run( + self.dvc.run( cmd=cmd, deps=deps, outs=outs, @@ -530,6 +530,22 @@ class TestCmdRunOverwrite(TestDvc): self.assertNotEqual(stage_mtime, os.path.getmtime("out.dvc")) +class TestCmdRunCliMetrics(TestDvc): + def test_cached(self): + ret = main( + ["run", "-m", "metrics.txt", "echo -n 'test' > metrics.txt"] + ) + self.assertEqual(ret, 0) + self.assertEqual(open("metrics.txt", "r").read(), "test") + + def test_not_cached(self): + ret = main( + ["run", "-M", "metrics.txt", "echo -n 'test' > metrics.txt"] + ) + self.assertEqual(ret, 0) + self.assertEqual(open("metrics.txt", "r").read(), "test") + + class TestRunDeterministicBase(TestDvc): def setUp(self): super(TestRunDeterministicBase, self).setUp() From 5563f13273e41a375394ba9854643a2a7e50d92c Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 9 Feb 2019 19:57:41 -0800 Subject: [PATCH 09/12] fix metrics tests on windows --- tests/test_metrics.py | 20 ++++++++++++++++++-- tests/test_run.py | 12 ++++-------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index ce3d088f1..5ebf11e1e 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -378,10 +378,26 @@ class TestCachedMetrics(TestDvc): self.dvc.scm.checkout(branch) self.dvc.checkout(force=True) - content = json.dumps({"metrics": branch}) + print( + 'print(json.dumps({{"metrics": "{branch}"}})\n'.format( + branch=branch + ) + ) + + with open("code.py", "w+") as fobj: + fobj.write("import sys\n") + fobj.write("import os\n") + fobj.write("import json\n") + fobj.write( + 'print(json.dumps({{"metrics": "{branch}"}}))\n'.format( + branch=branch + ) + ) + stage = self.dvc.run( + deps=["code.py"], metrics=["metrics.json"], - cmd="echo '{content}' > metrics.json".format(content=content), + cmd="python code.py metrics.json > metrics.json", ) self.assertIsNotNone(stage) self.assertEqual(stage.relpath, "metrics.json.dvc") diff --git a/tests/test_run.py b/tests/test_run.py index 5e6355500..046f5158c 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -532,18 +532,14 @@ class TestCmdRunOverwrite(TestDvc): class TestCmdRunCliMetrics(TestDvc): def test_cached(self): - ret = main( - ["run", "-m", "metrics.txt", "echo -n 'test' > metrics.txt"] - ) + ret = main(["run", "-m", "metrics.txt", "echo 'test' > metrics.txt"]) self.assertEqual(ret, 0) - self.assertEqual(open("metrics.txt", "r").read(), "test") + self.assertEqual(open("metrics.txt", "r").read().rstrip(), "test") def test_not_cached(self): - ret = main( - ["run", "-M", "metrics.txt", "echo -n 'test' > metrics.txt"] - ) + ret = main(["run", "-M", "metrics.txt", "echo 'test' > metrics.txt"]) self.assertEqual(ret, 0) - self.assertEqual(open("metrics.txt", "r").read(), "test") + self.assertEqual(open("metrics.txt", "r").read().rstrip(), "test") class TestRunDeterministicBase(TestDvc): From bd28b711f99b744488ddf0485108b63c5a99fb7c Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 9 Feb 2019 19:58:10 -0800 Subject: [PATCH 10/12] minor: remove debug print --- tests/test_metrics.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 5ebf11e1e..77ff4e355 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -378,12 +378,6 @@ class TestCachedMetrics(TestDvc): self.dvc.scm.checkout(branch) self.dvc.checkout(force=True) - print( - 'print(json.dumps({{"metrics": "{branch}"}})\n'.format( - branch=branch - ) - ) - with open("code.py", "w+") as fobj: fobj.write("import sys\n") fobj.write("import os\n") From 8817864e7a72d6249d298bcdabd2258f2eef75f3 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 9 Feb 2019 20:04:35 -0800 Subject: [PATCH 11/12] fix one more test on windows --- tests/test_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_run.py b/tests/test_run.py index 046f5158c..97d979f28 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -532,12 +532,12 @@ class TestCmdRunOverwrite(TestDvc): class TestCmdRunCliMetrics(TestDvc): def test_cached(self): - ret = main(["run", "-m", "metrics.txt", "echo 'test' > metrics.txt"]) + ret = main(["run", "-m", "metrics.txt", "echo test > metrics.txt"]) self.assertEqual(ret, 0) self.assertEqual(open("metrics.txt", "r").read().rstrip(), "test") def test_not_cached(self): - ret = main(["run", "-M", "metrics.txt", "echo 'test' > metrics.txt"]) + ret = main(["run", "-M", "metrics.txt", "echo test > metrics.txt"]) self.assertEqual(ret, 0) self.assertEqual(open("metrics.txt", "r").read().rstrip(), "test") From 41befea210989c8bbbf3d87be13b506a861d915a Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sun, 10 Feb 2019 11:06:23 -0800 Subject: [PATCH 12/12] address PR review comments --- dvc/command/run.py | 9 +++++---- dvc/logger.py | 4 ++-- dvc/stage.py | 6 +++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/dvc/command/run.py b/dvc/command/run.py index a978b8daf..c070f2708 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -91,15 +91,15 @@ def add_parser(subparsers, parent_parser): "--outs", action="append", default=[], - help="Declare output data file or data directory.", + help="Declare output file or directory.", ) run_parser.add_argument( "-O", "--outs-no-cache", action="append", default=[], - help="Declare output data file or directory " - "(sync to Git, not DVC cache).", + help="Declare output file or directory " + "(do not put into DVC cache).", ) run_parser.add_argument( "-m", @@ -113,7 +113,8 @@ def add_parser(subparsers, parent_parser): "--metrics-no-cache", action="append", default=[], - help="Declare output metric file or " "directory (not cached by DVC).", + help="Declare output metric file or directory " + "(do not put into DVC cache).", ) run_parser.add_argument( "-f", diff --git a/dvc/logger.py b/dvc/logger.py index 108571706..fcc0663fd 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -237,7 +237,7 @@ def _walk_exc(exc): def _parse_exc(): exc = sys.exc_info()[1] if not exc: - return None, "" + return (None, "") exc_list, tb_list = _walk_exc(exc) @@ -251,7 +251,7 @@ def _parse_exc(): else: stack_trace = "" - return exception, stack_trace + return (exception, stack_trace) def _description(message, exception): diff --git a/dvc/stage.py b/dvc/stage.py index cc1888d33..8e885befc 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -286,10 +286,10 @@ class Stage(object): @classmethod def _stage_fname_cwd(cls, fname, cwd, outs, add): if fname and cwd: - return fname, cwd + return (fname, cwd) if not outs: - return cls.STAGE_FILE, cwd if cwd else os.getcwd() + return (cls.STAGE_FILE, cwd if cwd else os.getcwd()) out = outs[0] if out.scheme == "local": @@ -303,7 +303,7 @@ class Stage(object): if not cwd or (add and out.is_local): cwd = path.dirname(out.path) - return fname, cwd + return (fname, cwd) @staticmethod def _check_inside_project(project, cwd):