Merge pull request #1944 from brbarkley/feature_jsonpath-ng

dvc metrics show: implement jsonpath-ng

dvc/repo/metrics/show.py

@@ -4,7 +4,7 @@ import os
 import csv
 import json
 import logging
-from jsonpath_rw import parse
+from jsonpath_ng.ext import parse
 
 from dvc.exceptions import OutputNotFoundError, BadMetricError, NoMetricsError
 from dvc.utils.compat import builtin_str, open, StringIO, csv_reader

requirements.txt

@@ -15,7 +15,7 @@ nanotime>=0.5.2
 pyasn1>=0.4.1
 schema>=0.6.7
 paramiko>=2.4.1
-jsonpath-rw==1.4.0
+jsonpath-ng>=1.4.3
 requests>=2.18.4
 wheel>=0.31.1
 futures>=3.2.0; python_version == "2.7"

setup.py

@@ -48,7 +48,7 @@ install_requires = [
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",
     "schema>=0.6.7",
-    "jsonpath-rw==1.4.0",
+    "jsonpath-ng>=1.4.3",
     "requests>=2.18.4",
     "grandalf==0.6",
     "asciimatics>=1.10.0",

tests/func/test_metrics.py

@@ -37,6 +37,35 @@ class TestMetrics(TestDvc):
                 fd.write("branch\n")
                 fd.write(branch)
 
+            if branch == "foo":
+                deviation_mse_train = 0.173461
+            else:
+                deviation_mse_train = 0.356245
+
+            with open("metric_json_ext", "w+") as fd:
+                json.dump(
+                    {
+                        "metrics": [
+                            {
+                                "dataset": "train",
+                                "deviation_mse": deviation_mse_train,
+                                "value_mse": 0.421601,
+                            },
+                            {
+                                "dataset": "testing",
+                                "deviation_mse": 0.289545,
+                                "value_mse": 0.297848,
+                            },
+                            {
+                                "dataset": "validation",
+                                "deviation_mse": 0.67528,
+                                "value_mse": 0.671502,
+                            },
+                        ]
+                    },
+                    fd,
+                )
+
             files = [
                 "metric",
                 "metric_json",
@@ -44,6 +73,7 @@ class TestMetrics(TestDvc):
                 "metric_htsv",
                 "metric_csv",
                 "metric_hcsv",
+                "metric_json_ext",
             ]
 
             self.dvc.run(metrics_no_cache=files, overwrite=True)
@@ -101,6 +131,26 @@ class TestMetrics(TestDvc):
         self.assertSequenceEqual(ret["bar"]["metric_hcsv"], ["bar"])
         self.assertSequenceEqual(ret["baz"]["metric_hcsv"], ["baz"])
 
+        ret = self.dvc.metrics.show(
+            "metric_json_ext",
+            typ="json",
+            xpath="$.metrics[?(@.deviation_mse<0.30) & (@.value_mse>0.4)]",
+            all_branches=True,
+        )
+        self.assertEqual(len(ret), 1)
+        self.assertSequenceEqual(
+            ret["foo"]["metric_json_ext"],
+            [
+                {
+                    "dataset": "train",
+                    "deviation_mse": 0.173461,
+                    "value_mse": 0.421601,
+                }
+            ],
+        )
+        self.assertRaises(KeyError, lambda: ret["bar"])
+        self.assertRaises(KeyError, lambda: ret["baz"])
+
     def test_unknown_type_ignored(self):
         ret = self.dvc.metrics.show(
             "metric_hcsv", typ="unknown", xpath="0,branch", all_branches=True
@@ -293,6 +343,7 @@ class TestMetrics(TestDvc):
         metric_htsv_path = os.path.join(root_relpath, "metric_htsv")
         metric_csv_path = os.path.join(root_relpath, "metric_csv")
         metric_hcsv_path = os.path.join(root_relpath, "metric_hcsv")
+        metric_json_ext_path = os.path.join(root_relpath, "metric_json_ext")
         for branch in ["bar", "baz", "foo"]:
             self.assertEqual(
                 set(metrics[branch].keys()),
@@ -303,6 +354,7 @@ class TestMetrics(TestDvc):
                     metric_htsv_path,
                     metric_csv_path,
                     metric_hcsv_path,
+                    metric_json_ext_path,
                 },
             )