import os import json from dvc.project import Project from dvc.main import main from dvc.exceptions import DvcException from tests.basic_env import TestDvc class TestMetrics(TestDvc): def setUp(self): super(TestMetrics, self).setUp() self.dvc.scm.commit('init') for branch in ['foo', 'bar', 'baz']: self.dvc.scm.checkout(branch, create_new=True) with open('metric', 'w+') as fd: fd.write(branch) with open('metric_json', 'w+') as fd: json.dump({'branch': branch}, fd) with open('metric_csv', 'w+') as fd: fd.write(branch) with open('metric_hcsv', 'w+') as fd: fd.write('branch\n') fd.write(branch) with open('metric_tsv', 'w+') as fd: fd.write(branch) with open('metric_htsv', 'w+') as fd: fd.write('branch\n') fd.write(branch) self.dvc.scm.add(['metric', 'metric_json', 'metric_tsv', 'metric_htsv', 'metric_csv', 'metric_hcsv']) self.dvc.scm.commit('metric') self.dvc.scm.checkout('master') def test(self): ret = self.dvc.metrics_show('metric', all_branches=True) self.assertEqual(len(ret), 3) self.assertTrue(ret['foo']['metric'] == 'foo') self.assertTrue(ret['bar']['metric'] == 'bar') self.assertTrue(ret['baz']['metric'] == 'baz') ret = self.dvc.metrics_show('metric_json', typ='json', xpath='branch', all_branches=True) self.assertEqual(len(ret), 3) self.assertTrue(ret['foo']['metric_json'] == ['foo']) self.assertTrue(ret['bar']['metric_json'] == ['bar']) self.assertTrue(ret['baz']['metric_json'] == ['baz']) ret = self.dvc.metrics_show('metric_tsv', typ='tsv', xpath='0,0', all_branches=True) self.assertEqual(len(ret), 3) self.assertTrue(ret['foo']['metric_tsv'] == ['foo']) self.assertTrue(ret['bar']['metric_tsv'] == ['bar']) self.assertTrue(ret['baz']['metric_tsv'] == ['baz']) ret = self.dvc.metrics_show('metric_htsv', typ='htsv', xpath='branch,0', all_branches=True) self.assertEqual(len(ret), 3) self.assertTrue(ret['foo']['metric_htsv'] == ['foo']) self.assertTrue(ret['bar']['metric_htsv'] == ['bar']) self.assertTrue(ret['baz']['metric_htsv'] == ['baz']) ret = self.dvc.metrics_show('metric_csv', typ='csv', xpath='0,0', all_branches=True) self.assertEqual(len(ret), 3) self.assertTrue(ret['foo']['metric_csv'] == ['foo']) self.assertTrue(ret['bar']['metric_csv'] == ['bar']) self.assertTrue(ret['baz']['metric_csv'] == ['baz']) ret = self.dvc.metrics_show('metric_hcsv', typ='hcsv', xpath='branch,0', all_branches=True) self.assertEqual(len(ret), 3) self.assertTrue(ret['foo']['metric_hcsv'] == ['foo']) self.assertTrue(ret['bar']['metric_hcsv'] == ['bar']) self.assertTrue(ret['baz']['metric_hcsv'] == ['baz']) class TestMetricsReproCLI(TestDvc): def test(self): stage = self.dvc.run(metrics_no_cache=['metrics'], cmd='python {} {} {}'.format(self.CODE, self.FOO, 'metrics')) ret = main(['repro', '-m', stage.path]) self.assertEqual(ret, 0) ret = main(['metrics', 'remove', 'metrics']) self.assertEqual(ret, 0) ret = main(['repro', '-f', '-m', stage.path]) self.assertNotEqual(ret, 0) ret = main(['metrics', 'add', 'metrics']) self.assertEqual(ret, 0) ret = main(['metrics', 'modify', '-t', 'csv', '-x', '0,0', 'metrics']) self.assertEqual(ret, 0) ret = main(['repro', '-f', '-m', stage.path]) self.assertEqual(ret, 0) def test_dir(self): os.mkdir('metrics_dir') with self.assertRaises(DvcException): self.dvc.run(metrics_no_cache=['metrics_dir']) def test_binary(self): with open('metrics_bin', 'w+') as fd: fd.write('\0\0\0\0\0\0\0\0') with self.assertRaises(DvcException): self.dvc.run(metrics_no_cache=['metrics_bin']) class TestMetricsCLICompat(TestMetrics): def test(self): #FIXME check output ret = main(['metrics', 'show', '--all-branches', 'metric', '-v']) self.assertEqual(ret, 0) ret = main(['metrics', 'show', '--all-branches', 'metric_json', '--json-path', 'branch']) self.assertEqual(ret, 0) ret = main(['metrics', 'show', '--all-branches', 'metric_tsv', '--tsv-path', '0,0']) self.assertEqual(ret, 0) ret = main(['metrics', 'show', '--all-branches', 'metric_htsv', '--htsv-path', 'branch,0']) self.assertEqual(ret, 0) ret = main(['metrics', 'show', '--all-branches', 'metric_csv', '--csv-path', '0,0']) self.assertEqual(ret, 0) ret = main(['metrics', 'show', '--all-branches', 'metric_hcsv', '--hcsv-path', 'branch,0']) self.assertEqual(ret, 0) class TestMetricsCLI(TestMetrics): def test(self): #FIXME check output ret = main(['metrics', 'show', '-a', 'metric', '-v']) self.assertEqual(ret, 0) ret = main(['metrics', 'show', '-a', 'metric_json', '-t', 'json', '-x', 'branch']) self.assertEqual(ret, 0) ret = main(['metrics', 'show', '-a', 'metric_tsv', '-t', 'tsv', '-x', '0,0']) self.assertEqual(ret, 0) ret = main(['metrics', 'show', '-a', 'metric_htsv', '-t', 'htsv', '-x', 'branch,0']) self.assertEqual(ret, 0) ret = main(['metrics', 'show', '-a', 'metric_csv', '-t', 'csv', '-x', '0,0']) self.assertEqual(ret, 0) ret = main(['metrics', 'show', '-a', 'metric_hcsv', '-t', 'hcsv', '-x', 'branch,0']) self.assertEqual(ret, 0) def test_dir(self): os.mkdir('metrics_dir') with self.assertRaises(DvcException): self.dvc.run(outs_no_cache=['metrics_dir']) self.dvc.metrics_add('metrics_dir') def test_binary(self): with open('metrics_bin', 'w+') as fd: fd.write('\0\0\0\0\0\0\0\0') with self.assertRaises(DvcException): self.dvc.run(outs_no_cache=['metrics_bin']) self.dvc.metrics_add('metrics_bin') def test_non_existing(self): ret = main(['metrics', 'add', 'non-existing']) self.assertNotEqual(ret, 0) ret = main(['metrics', 'modify', 'non-existing']) self.assertNotEqual(ret, 0) ret = main(['metrics', 'remove', 'non-existing']) self.assertNotEqual(ret, 0) class TestNoMetrics(TestDvc): def test(self): ret = main(['metrics', 'show']) self.assertNotEqual(ret, 0) class TestCachedMetrics(TestDvc): def _do_write(self, branch): self.dvc.scm.checkout(branch) self.dvc.checkout() with open('metrics.json', 'w+') as fd: json.dump({'metrics': branch}, fd) stages = self.dvc.add('metrics.json') self.assertEqual(len(stages), 1) stage = stages[0] self.assertNotEqual(stage, None) self.dvc.scm.add(['.gitignore', 'metrics.json.dvc']) self.dvc.scm.commit(branch) def test(self): self.dvc.scm.commit('init') self.dvc.scm.branch('one') self.dvc.scm.branch('two') self._do_write('master') self._do_write('one') self._do_write('two') self.dvc = Project('.') res = self.dvc.metrics_show('metrics.json', all_branches=True, typ='json', xpath='metrics') self.assertEqual(res, {"master": {"metrics.json": ["master"]}, "one": {"metrics.json": ["one"]}, "two": {"metrics.json": ["two"]}})