diff --git a/dvc/command/metrics.py b/dvc/command/metrics.py index d1e019521..48c0bac52 100644 --- a/dvc/command/metrics.py +++ b/dvc/command/metrics.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import argparse import logging -from dvc.exceptions import DvcException +from dvc.exceptions import DvcException, BadMetricError from dvc.command.base import CmdBase, fix_subparsers, append_doc_link @@ -16,6 +16,10 @@ def show_metrics(metrics, all_branches=False, all_tags=False): metrics (list): Where each element is either a `list` if an xpath was specified, otherwise a `str` """ + # When `metrics` contains a `None` key, it means that some files + # specified as `targets` in `repo.metrics.show` didn't contain any metrics. + missing = metrics.pop(None, None) + for branch, val in metrics.items(): if all_branches or all_tags: logger.info("{branch}:".format(branch=branch)) @@ -32,16 +36,17 @@ def show_metrics(metrics, all_branches=False, all_tags=False): else: logger.info("\t{}: {}".format(fname, metric)) + if missing: + raise BadMetricError(missing) + class CmdMetricsShow(CmdBase): def run(self): - typ = self.args.type - xpath = self.args.xpath try: metrics = self.repo.metrics.show( - self.args.path, - typ=typ, - xpath=xpath, + self.args.targets, + typ=self.args.type, + xpath=self.args.xpath, all_branches=self.args.all_branches, all_tags=self.args.all_tags, recursive=self.args.recursive, @@ -121,7 +126,10 @@ def add_parser(subparsers, parent_parser): formatter_class=argparse.RawDescriptionHelpFormatter, ) metrics_show_parser.add_argument( - "path", nargs="?", help="Path to a metric file or a directory." + "targets", + nargs="*", + help="Metric files or directories (see -R) to show " + "(leave empty to display all)", ) metrics_show_parser.add_argument( "-t", @@ -155,8 +163,8 @@ def add_parser(subparsers, parent_parser): action="store_true", default=False, help=( - "If path is a directory, recursively search and process metric " - "files in path." + "If any target is a directory, recursively search and process " + "metric files." ), ) metrics_show_parser.set_defaults(func=CmdMetricsShow) diff --git a/dvc/command/version.py b/dvc/command/version.py index 770d9ad3c..ad041b4dd 100644 --- a/dvc/command/version.py +++ b/dvc/command/version.py @@ -109,7 +109,7 @@ class CmdVersion(CmdBaseNoRepo): def add_parser(subparsers, parent_parser): - VERSION_HELP = "Show DVC version and system/environment informaion." + VERSION_HELP = "Show DVC version and system/environment information." version_parser = subparsers.add_parser( "version", diff --git a/dvc/exceptions.py b/dvc/exceptions.py index da96fd0f9..bddf83ec9 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -196,10 +196,11 @@ class ReproductionError(DvcException): class BadMetricError(DvcException): - def __init__(self, path): + def __init__(self, paths): super(BadMetricError, self).__init__( - "'{}' does not exist, not a metric or is malformed".format( - relpath(path) + "the following metrics do not exists, " + "are not metric files or are malformed: {paths}".format( + paths=", ".join("'{}'".format(path) for path in paths) ) ) diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 5f8839d1e..c8dcd9777 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -6,6 +6,8 @@ import re import logging from datetime import datetime, timedelta +from funcy import cached_property + from dvc.scheme import Schemes try: @@ -19,7 +21,6 @@ from dvc.progress import Tqdm from dvc.config import Config from dvc.remote.base import RemoteBASE from dvc.path_info import CloudURLInfo -from dvc.utils.http import open_url logger = logging.getLogger(__name__) @@ -63,29 +64,27 @@ class RemoteAZURE(RemoteBASE): if not self.connection_string: raise ValueError("azure storage connection string missing") - self.__blob_service = None self.path_info = ( self.path_cls(url) if path else self.path_cls.from_parts(scheme=self.scheme, netloc=bucket) ) - @property + @cached_property def blob_service(self): - if self.__blob_service is None: - logger.debug("URL {}".format(self.path_info)) - logger.debug("Connection string {}".format(self.connection_string)) - self.__blob_service = BlockBlobService( - connection_string=self.connection_string + logger.debug("URL {}".format(self.path_info)) + logger.debug("Connection string {}".format(self.connection_string)) + blob_service = BlockBlobService( + connection_string=self.connection_string + ) + logger.debug("Container name {}".format(self.path_info.bucket)) + try: # verify that container exists + blob_service.list_blobs( + self.path_info.bucket, delimiter="/", num_results=1 ) - logger.debug("Container name {}".format(self.path_info.bucket)) - try: # verify that container exists - self.__blob_service.list_blobs( - self.path_info.bucket, delimiter="/", num_results=1 - ) - except AzureMissingResourceHttpError: - self.__blob_service.create_container(self.path_info.bucket) - return self.__blob_service + except AzureMissingResourceHttpError: + blob_service.create_container(self.path_info.bucket) + return blob_service def remove(self, path_info): if path_info.scheme != self.scheme: @@ -135,9 +134,9 @@ class RemoteAZURE(RemoteBASE): progress_callback=pbar.update_to, ) - def open(self, path_info, mode="r", encoding=None): - get_url = lambda: self._generate_download_url(path_info) # noqa: E731 - return open_url(get_url, mode=mode, encoding=encoding) + def exists(self, path_info): + paths = self._list_paths(path_info.bucket, path_info.path) + return any(path_info.path == path for path in paths) def _generate_download_url(self, path_info, expires=3600): expires_at = datetime.utcnow() + timedelta(seconds=expires) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 9a2aeea35..396c06696 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -10,7 +10,7 @@ import tempfile import itertools from operator import itemgetter from multiprocessing import cpu_count -import functools +from functools import partial import dvc.prompt as prompt from dvc.config import Config @@ -20,16 +20,10 @@ from dvc.exceptions import ( DvcIgnoreInCollectedDirError, ) from dvc.progress import Tqdm, TqdmThreadPoolExecutor -from dvc.utils import ( - LARGE_DIR_SIZE, - tmp_fname, - to_chunks, - move, - relpath, - makedirs, -) +from dvc.utils import LARGE_DIR_SIZE, tmp_fname, move, relpath, makedirs from dvc.state import StateNoop from dvc.path_info import PathInfo, URLInfo +from dvc.utils.http import open_url logger = logging.getLogger(__name__) @@ -503,6 +497,10 @@ class RemoteBASE(object): return 0 def open(self, path_info, mode="r", encoding=None): + if hasattr(self, "_generate_download_url"): + get_url = partial(self._generate_download_url, path_info) + return open_url(get_url, mode=mode, encoding=encoding) + raise RemoteActionNotImplemented("open", self.scheme) def remove(self, path_info): @@ -630,25 +628,20 @@ class RemoteBASE(object): Returns: A list with checksums that were found in the remote """ - if self.no_traverse and hasattr(self, "batch_exists"): - with Tqdm(total=len(checksums)) as pbar: - exists_with_progress = functools.partial( - self.batch_exists, callback=pbar.update_desc - ) + if not self.no_traverse: + return list(set(checksums) & set(self.all())) - with TqdmThreadPoolExecutor( - max_workers=jobs or self.JOBS - ) as executor: - path_infos = [ - self.checksum_to_path_info(x) for x in checksums - ] - chunks = to_chunks(path_infos, num_chunks=self.JOBS) - results = executor.map(exists_with_progress, chunks) - in_remote = itertools.chain.from_iterable(results) - ret = list(itertools.compress(checksums, in_remote)) - return ret + with Tqdm(total=len(checksums)) as pbar: + def exists_with_progress(path_info): + ret = self.exists(path_info) + pbar.update_desc(str(path_info)) + return ret - return list(set(checksums) & set(self.all())) + with TqdmThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor: + path_infos = [self.checksum_to_path_info(x) for x in checksums] + in_remote = executor.map(exists_with_progress, path_infos) + ret = list(itertools.compress(checksums, in_remote)) + return ret def already_cached(self, path_info): current = self.get_checksum(path_info) diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index 3bb495fd8..e64a41e42 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals import logging -import itertools +from datetime import timedelta from funcy import cached_property try: @@ -89,17 +89,6 @@ class RemoteGS(RemoteBASE): paths = set(self._list_paths(path_info.bucket, path_info.path)) return any(path_info.path == path for path in paths) - def batch_exists(self, path_infos, callback): - paths = [] - - for path_info in path_infos: - paths.append(self._list_paths(path_info.bucket, path_info.path)) - callback(str(path_info)) - - paths = set(itertools.chain.from_iterable(paths)) - - return [path_info.path in paths for path_info in path_infos] - def _upload(self, from_file, to_info, **_kwargs): bucket = self.gs.bucket(to_info.bucket) blob = bucket.blob(to_info.path) @@ -109,3 +98,10 @@ class RemoteGS(RemoteBASE): bucket = self.gs.bucket(from_info.bucket) blob = bucket.get_blob(from_info.path) blob.download_to_filename(to_file) + + def _generate_download_url(self, path_info, expires=3600): + expiration = timedelta(seconds=int(expires)) + + bucket = self.gs.bucket(path_info.bucket) + blob = bucket.get_blob(path_info.path) + return blob.generate_signed_url(expiration=expiration) diff --git a/dvc/remote/http.py b/dvc/remote/http.py index 9c6aafea2..64891bd16 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -4,6 +4,7 @@ from dvc.scheme import Schemes from dvc.utils import LARGE_FILE_SIZE from dvc.utils.compat import open +import threading import requests import logging @@ -50,15 +51,6 @@ class RemoteHTTP(RemoteBASE): def exists(self, path_info): return bool(self._request("HEAD", path_info.url)) - def batch_exists(self, path_infos, callback): - results = [] - - for path_info in path_infos: - results.append(self.exists(path_info)) - callback(str(path_info)) - - return results - def _content_length(self, url_or_request): headers = getattr( url_or_request, diff --git a/dvc/remote/oss.py b/dvc/remote/oss.py index 6e493cc07..108c1bc49 100644 --- a/dvc/remote/oss.py +++ b/dvc/remote/oss.py @@ -119,3 +119,12 @@ class RemoteOSS(RemoteBASE): self.oss_service.get_object_to_file( from_info.path, to_file, progress_callback=pbar.update_to ) + + def _generate_download_url(self, path_info, expires=3600): + assert path_info.bucket == self.path_info.bucket + + return self.oss_service.sign_url("GET", path_info.path, expires) + + def exists(self, path_info): + paths = self._list_paths(path_info.path) + return any(path_info.path == path for path in paths) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index f86d0277c..547a33969 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -1,8 +1,8 @@ from __future__ import unicode_literals import os +import threading import logging -import itertools from funcy import cached_property try: @@ -202,16 +202,6 @@ class RemoteS3(RemoteBASE): paths = self._list_paths(path_info.bucket, path_info.path) return any(path_info.path == path for path in paths) - def batch_exists(self, path_infos, callback): - paths = [] - - for path_info in path_infos: - paths.append(self._list_paths(path_info.bucket, path_info.path)) - callback(str(path_info)) - - paths = set(itertools.chain.from_iterable(paths)) - return [path_info.path in paths for path_info in path_infos] - def _upload(self, from_file, to_info, name=None, no_progress_bar=False): total = os.path.getsize(from_file) with Tqdm( @@ -244,3 +234,9 @@ class RemoteS3(RemoteBASE): self.s3.download_file( from_info.bucket, from_info.path, to_file, Callback=pbar.update ) + + def _generate_download_url(self, path_info, expires=3600): + params = {"Bucket": path_info.bucket, "Key": path_info.path} + return self.s3.generate_presigned_url( + ClientMethod="get_object", Params=params, ExpiresIn=int(expires) + ) diff --git a/dvc/remote/ssh/__init__.py b/dvc/remote/ssh/__init__.py index c817eabaa..fe76eb6bf 100644 --- a/dvc/remote/ssh/__init__.py +++ b/dvc/remote/ssh/__init__.py @@ -1,11 +1,12 @@ from __future__ import unicode_literals +import errno +import itertools import os import getpass import logging -import itertools -import errno import threading +from concurrent.futures import ThreadPoolExecutor try: import paramiko @@ -19,7 +20,7 @@ from dvc.utils.compat import urlparse, StringIO from dvc.remote.base import RemoteBASE from dvc.scheme import Schemes from dvc.remote.pool import get_connection -from dvc.progress import TqdmThreadPoolExecutor +from dvc.progress import Tqdm from .connection import SSHConnection @@ -148,34 +149,6 @@ class RemoteSSH(RemoteBASE): with self.ssh(path_info) as ssh: return ssh.exists(path_info.path) - def batch_exists(self, path_infos, callback): - def _exists(chunk_and_channel): - chunk, channel = chunk_and_channel - ret = [] - for path in chunk: - try: - channel.stat(path) - ret.append(True) - except IOError as exc: - if exc.errno != errno.ENOENT: - raise - ret.append(False) - callback(path) - return ret - - with self.ssh(path_infos[0]) as ssh: - channels = ssh.open_max_sftp_channels() - max_workers = len(channels) - - with TqdmThreadPoolExecutor(max_workers=max_workers) as executor: - paths = [path_info.path for path_info in path_infos] - chunks = to_chunks(paths, num_chunks=max_workers) - chunks_and_channels = zip(chunks, channels) - outcome = executor.map(_exists, chunks_and_channels) - results = list(itertools.chain.from_iterable(outcome)) - - return results - def get_file_checksum(self, path_info): if path_info.scheme != self.scheme: raise NotImplementedError @@ -240,3 +213,52 @@ class RemoteSSH(RemoteBASE): def makedirs(self, path_info): with self.ssh(path_info) as ssh: ssh.makedirs(path_info.path) + + def batch_exists(self, path_infos, callback): + def _exists(chunk_and_channel): + chunk, channel = chunk_and_channel + ret = [] + for path in chunk: + try: + channel.stat(path) + ret.append(True) + except IOError as exc: + if exc.errno != errno.ENOENT: + raise + ret.append(False) + callback(path) + return ret + + with self.ssh(path_infos[0]) as ssh: + channels = ssh.open_max_sftp_channels() + max_workers = len(channels) + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + paths = [path_info.path for path_info in path_infos] + chunks = to_chunks(paths, num_chunks=max_workers) + chunks_and_channels = zip(chunks, channels) + outcome = executor.map(_exists, chunks_and_channels) + results = list(itertools.chain.from_iterable(outcome)) + + return results + + def cache_exists(self, checksums, jobs=None): + """This is older implementation used in remote/base.py + We are reusing it in RemoteSSH, because SSH's batch_exists proved to be + faster than current approach (relying on exists(path_info)) applied in + remote/base. + """ + if not self.no_traverse: + return list(set(checksums) & set(self.all())) + + with Tqdm(total=len(checksums)) as pbar: + def exists_with_progress(chunks): + return self.batch_exists(chunks, callback=pbar.update_desc) + + with ThreadPoolExecutor(max_workers=jobs or self.JOBS) as executor: + path_infos = [self.checksum_to_path_info(x) for x in checksums] + chunks = to_chunks(path_infos, num_chunks=self.JOBS) + results = executor.map(exists_with_progress, chunks) + in_remote = itertools.chain.from_iterable(results) + ret = list(itertools.compress(checksums, in_remote)) + return ret diff --git a/dvc/repo/metrics/show.py b/dvc/repo/metrics/show.py index 42ceab3d0..cd1ab7459 100644 --- a/dvc/repo/metrics/show.py +++ b/dvc/repo/metrics/show.py @@ -8,7 +8,7 @@ import logging from jsonpath_ng.ext import parse -from dvc.exceptions import OutputNotFoundError, BadMetricError, NoMetricsError +from dvc.exceptions import OutputNotFoundError, NoMetricsError from dvc.utils.compat import builtin_str, open, StringIO, csv_reader NO_METRICS_FILE_AT_REFERENCE_WARNING = ( @@ -159,8 +159,8 @@ def _collect_metrics(repo, path, recursive, typ, xpath, branch): path (str): Path to a metric file or a directory. recursive (bool): If path is a directory, do a recursive search for metrics on the given path. - typ (str): The type of metric to search for, could be one of the - following (raw|json|tsv|htsv|csv|hcsv). + typ (str): The type that will be used to interpret the metric file, + one of the followings - (raw|json|tsv|htsv|csv|hcsv). xpath (str): Path to search for. branch (str): Branch to look up for metrics. @@ -259,7 +259,7 @@ def _read_metrics(repo, metrics, branch): def show( repo, - path=None, + targets=None, typ=None, xpath=None, all_branches=False, @@ -267,16 +267,34 @@ def show( recursive=False, ): res = {} + found = set() + + if not targets: + # Iterate once to call `_collect_metrics` on all the stages + targets = [None] for branch in repo.brancher(all_branches=all_branches, all_tags=all_tags): - entries = _collect_metrics(repo, path, recursive, typ, xpath, branch) - metrics = _read_metrics(repo, entries, branch) + metrics = {} + + for target in targets: + entries = _collect_metrics( + repo, target, recursive, typ, xpath, branch + ) + metric = _read_metrics(repo, entries, branch) + + if metric: + found.add(target) + metrics.update(metric) + if metrics: res[branch] = metrics - if not res: - if path: - raise BadMetricError(path) + if not res and not any(targets): raise NoMetricsError() + missing = set(targets) - found + + if missing: + res[None] = missing + return res diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 8254ee11d..050f65366 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -193,8 +193,8 @@ def _chmod(func, p, excinfo): try: os.chmod(p, perm) except OSError as exc: - # broken symlink - if exc.errno != errno.ENOENT: + # broken symlink or file is not owned by us + if exc.errno not in [errno.ENOENT, errno.EPERM]: raise func(p) diff --git a/scripts/ci/before_install.sh b/scripts/ci/before_install.sh index 19645bcad..4f48cc6f6 100644 --- a/scripts/ci/before_install.sh +++ b/scripts/ci/before_install.sh @@ -31,7 +31,7 @@ if [[ "$TRAVIS_BUILD_STAGE_NAME" == "Test" ]]; then ssh 0.0.0.0 ls &> /dev/null fi - if [ "$TRAVIS_OS_NAME" == "linux" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then + if [ "$TRAVIS_OS_NAME" == "linux" ]; then bash "$scriptdir/install_azurite.sh" bash "$scriptdir/install_oss.sh" bash "$scriptdir/install_hadoop.sh" diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh index 658bf1fd1..86d10fef0 100644 --- a/scripts/ci/install.sh +++ b/scripts/ci/install.sh @@ -10,7 +10,12 @@ scriptdir="$(dirname $0)" # work or not. $scriptdir/retry.sh pip install --upgrade pip setuptools wheel $scriptdir/retry.sh pip install .[all,tests] -$scriptdir/retry.sh pip install psutil +# Installing specific packages to workaround some bugs. Please see [1] and [2] +# +# [1] https://github.com/iterative/dvc/issues/2284 +# [2] https://github.com/iterative/dvc/issues/2387 +$scriptdir/retry.sh pip uninstall -y azure-storage-blob +$scriptdir/retry.sh pip install psutil azure-storage-blob==1.5.0 git config --global user.email "dvctester@example.com" git config --global user.name "DVC Tester" diff --git a/scripts/ci/install_azurite.sh b/scripts/ci/install_azurite.sh index 094864db4..ad6d23e71 100755 --- a/scripts/ci/install_azurite.sh +++ b/scripts/ci/install_azurite.sh @@ -1,16 +1,20 @@ #!/usr/bin/env bash -set -euo pipefail +set -e +set -x # install docker -export DEBIAN_FRONTEND=noninteractive -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -sudo apt-get update -sudo apt-get install -y docker-ce + export DEBIAN_FRONTEND=noninteractive + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + sudo apt-get update + sudo apt-get install -y docker-ce # run azurite -sudo docker run --restart always -e executable=blob -p 10000:10000 --tmpfs /opt/azurite/folder -d arafato/azurite:2.6.5 +sudo docker run -d --restart always -e executable=blob -p 10000:10000 \ + --tmpfs /opt/azurite/folder \ + mcr.microsoft.com/azure-storage/azurite:3.1.2-preview \ + azurite -l /data --blobHost 0.0.0.0 # save secrets echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >> env.sh diff --git a/tests/func/test_metrics.py b/tests/func/test_metrics.py index e7892f506..9f0b2a00f 100644 --- a/tests/func/test_metrics.py +++ b/tests/func/test_metrics.py @@ -7,7 +7,7 @@ import logging from dvc.repo import Repo as DvcRepo from dvc.main import main -from dvc.exceptions import DvcException, BadMetricError, NoMetricsError +from dvc.exceptions import DvcException, NoMetricsError from dvc.repo.metrics.show import NO_METRICS_FILE_AT_REFERENCE_WARNING from dvc.stage import Stage from dvc.utils import relpath @@ -85,14 +85,14 @@ class TestMetricsBase(TestDvcGit): class TestMetrics(TestMetricsBase): def test_show(self): - ret = self.dvc.metrics.show("metric", all_branches=True) + ret = self.dvc.metrics.show(["metric"], all_branches=True) self.assertEqual(len(ret), 3) self.assertEqual(ret["foo"]["metric"], "foo") self.assertEqual(ret["bar"]["metric"], "bar") self.assertEqual(ret["baz"]["metric"], "baz") ret = self.dvc.metrics.show( - "metric_json", typ="json", xpath="branch", all_branches=True + ["metric_json"], typ="json", xpath="branch", all_branches=True ) self.assertEqual(len(ret), 3) self.assertSequenceEqual(ret["foo"]["metric_json"], ["foo"]) @@ -100,7 +100,7 @@ class TestMetrics(TestMetricsBase): self.assertSequenceEqual(ret["baz"]["metric_json"], ["baz"]) ret = self.dvc.metrics.show( - "metric_tsv", typ="tsv", xpath="0,0", all_branches=True + ["metric_tsv"], typ="tsv", xpath="0,0", all_branches=True ) self.assertEqual(len(ret), 3) self.assertSequenceEqual(ret["foo"]["metric_tsv"], ["foo"]) @@ -108,7 +108,7 @@ class TestMetrics(TestMetricsBase): self.assertSequenceEqual(ret["baz"]["metric_tsv"], ["baz"]) ret = self.dvc.metrics.show( - "metric_htsv", typ="htsv", xpath="0,branch", all_branches=True + ["metric_htsv"], typ="htsv", xpath="0,branch", all_branches=True ) self.assertEqual(len(ret), 3) self.assertSequenceEqual(ret["foo"]["metric_htsv"], ["foo"]) @@ -116,7 +116,7 @@ class TestMetrics(TestMetricsBase): self.assertSequenceEqual(ret["baz"]["metric_htsv"], ["baz"]) ret = self.dvc.metrics.show( - "metric_csv", typ="csv", xpath="0,0", all_branches=True + ["metric_csv"], typ="csv", xpath="0,0", all_branches=True ) self.assertEqual(len(ret), 3) self.assertSequenceEqual(ret["foo"]["metric_csv"], ["foo"]) @@ -124,7 +124,7 @@ class TestMetrics(TestMetricsBase): self.assertSequenceEqual(ret["baz"]["metric_csv"], ["baz"]) ret = self.dvc.metrics.show( - "metric_hcsv", typ="hcsv", xpath="0,branch", all_branches=True + ["metric_hcsv"], typ="hcsv", xpath="0,branch", all_branches=True ) self.assertEqual(len(ret), 3) self.assertSequenceEqual(ret["foo"]["metric_hcsv"], ["foo"]) @@ -132,7 +132,7 @@ class TestMetrics(TestMetricsBase): self.assertSequenceEqual(ret["baz"]["metric_hcsv"], ["baz"]) ret = self.dvc.metrics.show( - "metric_json_ext", + ["metric_json_ext"], typ="json", xpath="$.metrics[?(@.deviation_mse<0.30) & (@.value_mse>0.4)]", all_branches=True, @@ -153,7 +153,7 @@ class TestMetrics(TestMetricsBase): def test_unknown_type_ignored(self): ret = self.dvc.metrics.show( - "metric_hcsv", typ="unknown", xpath="0,branch", all_branches=True + ["metric_hcsv"], typ="unknown", xpath="0,branch", all_branches=True ) self.assertEqual(len(ret), 3) for b in ["foo", "bar", "baz"]: @@ -161,7 +161,7 @@ class TestMetrics(TestMetricsBase): def test_type_case_normalized(self): ret = self.dvc.metrics.show( - "metric_hcsv", typ=" hCSV ", xpath="0,branch", all_branches=True + ["metric_hcsv"], typ=" hCSV ", xpath="0,branch", all_branches=True ) self.assertEqual(len(ret), 3) for b in ["foo", "bar", "baz"]: @@ -169,7 +169,7 @@ class TestMetrics(TestMetricsBase): def test_xpath_is_empty(self): ret = self.dvc.metrics.show( - "metric_json", typ="json", xpath="", all_branches=True + ["metric_json"], typ="json", xpath="", all_branches=True ) self.assertEqual(len(ret), 3) for b in ["foo", "bar", "baz"]: @@ -177,7 +177,7 @@ class TestMetrics(TestMetricsBase): def test_xpath_is_none(self): ret = self.dvc.metrics.show( - "metric_json", typ="json", xpath=None, all_branches=True + ["metric_json"], typ="json", xpath=None, all_branches=True ) self.assertEqual(len(ret), 3) for b in ["foo", "bar", "baz"]: @@ -185,7 +185,7 @@ class TestMetrics(TestMetricsBase): def test_xpath_all_columns(self): ret = self.dvc.metrics.show( - "metric_hcsv", typ="hcsv ", xpath="0,", all_branches=True + ["metric_hcsv"], typ="hcsv ", xpath="0,", all_branches=True ) self.assertEqual(len(ret), 3) for b in ["foo", "bar", "baz"]: @@ -193,7 +193,7 @@ class TestMetrics(TestMetricsBase): def test_xpath_all_rows(self): ret = self.dvc.metrics.show( - "metric_csv", typ="csv", xpath=",0", all_branches=True + ["metric_csv"], typ="csv", xpath=",0", all_branches=True ) self.assertEqual(len(ret), 3) for b in ["foo", "bar", "baz"]: @@ -201,7 +201,7 @@ class TestMetrics(TestMetricsBase): def test_xpath_all(self): ret = self.dvc.metrics.show( - "metric_csv", typ="csv", xpath=",", all_branches=True + ["metric_csv"], typ="csv", xpath=",", all_branches=True ) self.assertEqual(len(ret), 3) for b in ["foo", "bar", "baz"]: @@ -209,7 +209,7 @@ class TestMetrics(TestMetricsBase): def test_xpath_all_with_header(self): ret = self.dvc.metrics.show( - "metric_hcsv", typ="hcsv", xpath=",", all_branches=True + ["metric_hcsv"], typ="hcsv", xpath=",", all_branches=True ) self.assertEqual(len(ret), 3) for b in ["foo", "bar", "baz"]: @@ -412,11 +412,13 @@ class TestMetricsRecursive(TestDvcGit): self.dvc.scm.checkout("master") def test(self): - with self.assertRaises(BadMetricError): - self.dvc.metrics.show("nested", all_branches=True, recursive=False) + ret = self.dvc.metrics.show( + ["nested"], all_branches=True, recursive=False + ) + self.assertEqual(len(ret), 1) ret = self.dvc.metrics.show( - "nested", all_branches=True, recursive=True + ["nested"], all_branches=True, recursive=True ) self.assertEqual(len(ret), 1) self.assertEqual( @@ -697,7 +699,7 @@ class TestCachedMetrics(TestDvcGit): self.dvc = DvcRepo(".") res = self.dvc.metrics.show( - "metrics.json", all_branches=True, typ="json", xpath="metrics" + ["metrics.json"], all_branches=True, typ="json", xpath="metrics" ) self.assertEqual( @@ -711,7 +713,7 @@ class TestCachedMetrics(TestDvcGit): ) res = self.dvc.metrics.show( - "", all_branches=True, typ="json", xpath="metrics" + all_branches=True, typ="json", xpath="metrics" ) self.assertEqual( @@ -777,7 +779,9 @@ class TestMetricsType(TestDvcGit): self._do_show(file_name, xpath) def _do_show(self, file_name, xpath): - ret = self.dvc.metrics.show(file_name, xpath=xpath, all_branches=True) + ret = self.dvc.metrics.show( + [file_name], xpath=xpath, all_branches=True + ) self.assertEqual(len(ret), 3) for branch in self.branches: if isinstance(ret[branch][file_name], list): @@ -844,3 +848,41 @@ def test_show_xpath_should_override_stage_xpath(dvc_repo): result = dvc_repo.metrics.show(xpath="m1") assert result == {"": {metric_file: [0.1]}} + + +def test_show_multiple_outputs(dvc_repo, caplog): + with open("1.json", "w") as fobj: + json.dump({"AUC": 1}, fobj) + + with open("2.json", "w") as fobj: + json.dump({"AUC": 2}, fobj) + + os.mkdir("metrics") + with open("metrics/3.json", "w") as fobj: + json.dump({"AUC": 3}, fobj) + + dvc_repo.run(cmd="", overwrite=True, metrics=["1.json"]) + dvc_repo.run(cmd="", overwrite=True, metrics=["2.json"]) + dvc_repo.run(cmd="", overwrite=True, metrics=["metrics/3.json"]) + + with caplog.at_level(logging.INFO, logger="dvc"): + assert 0 == main(["metrics", "show", "1.json", "2.json"]) + assert '1.json: {"AUC": 1}' in caplog.text + assert '2.json: {"AUC": 2}' in caplog.text + + caplog.clear() + + with caplog.at_level(logging.INFO, logger="dvc"): + assert 0 == main(["metrics", "show", "-R", "1.json", "metrics"]) + assert '1.json: {"AUC": 1}' in caplog.text + assert '3.json: {"AUC": 3}' in caplog.text + + caplog.clear() + + with caplog.at_level(logging.INFO, logger="dvc"): + assert 1 == main(["metrics", "show", "1.json", "not-found"]) + assert '1.json: {"AUC": 1}' in caplog.text + assert ( + "the following metrics do not exists, " + "are not metric files or are malformed: 'not-found'" + ) in caplog.text