diff --git a/dvc/api/data.py b/dvc/api/data.py index 423941b21..bf389486a 100644 --- a/dvc/api/data.py +++ b/dvc/api/data.py @@ -1,12 +1,30 @@ from contextlib import _GeneratorContextManager as GCM +from contextlib import contextmanager from typing import Optional from funcy import reraise -from dvc.exceptions import OutputNotFoundError, PathMissingError +from dvc.exceptions import FileMissingError, OutputNotFoundError, PathMissingError from dvc.repo import Repo +@contextmanager +def _wrap_exceptions(repo, url): + from dvc.config import NoRemoteError + from dvc.exceptions import NoOutputInExternalRepoError, NoRemoteInExternalRepoError + + try: + yield + except NoRemoteError as exc: + raise NoRemoteInExternalRepoError(url) from exc + except OutputNotFoundError as exc: + if exc.repo is repo: + raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url) from exc + raise + except FileMissingError as exc: + raise PathMissingError(exc.path, url) from exc + + def get_url(path, repo=None, rev=None, remote=None): """ Returns the URL to the storage location of a data file or directory tracked @@ -19,18 +37,20 @@ def get_url(path, repo=None, rev=None, remote=None): directory in the remote storage. """ with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo: - fs_path = _repo.dvcfs.from_os_path(path) - with reraise(FileNotFoundError, PathMissingError(path, repo)): - info = _repo.dvcfs.info(fs_path) + with _wrap_exceptions(_repo, path): + fs_path = _repo.dvcfs.from_os_path(path) - dvc_info = info.get("dvc_info") - if not dvc_info: - raise OutputNotFoundError(path, repo) + with reraise(FileNotFoundError, PathMissingError(path, repo)): + info = _repo.dvcfs.info(fs_path) - dvc_repo = info["repo"] # pylint: disable=unsubscriptable-object - md5 = dvc_info["md5"] + dvc_info = info.get("dvc_info") + if not dvc_info: + raise OutputNotFoundError(path, repo) - return dvc_repo.cloud.get_url_for(remote, checksum=md5) + dvc_repo = info["repo"] # pylint: disable=unsubscriptable-object + md5 = dvc_info["md5"] + + return dvc_repo.cloud.get_url_for(remote, checksum=md5) class _OpenContextManager(GCM): @@ -195,10 +215,42 @@ def open( # noqa, pylint: disable=redefined-builtin def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None): with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo: - with _repo.open_by_relpath( - path, remote=remote, mode=mode, encoding=encoding - ) as fd: - yield fd + with _wrap_exceptions(_repo, path): + import os + from typing import TYPE_CHECKING, Union + + from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError + from dvc.fs.data import DataFileSystem + from dvc.fs.dvc import DVCFileSystem + + if TYPE_CHECKING: + from dvc.fs import FileSystem + + fs: Union["FileSystem", DataFileSystem, DVCFileSystem] + if os.path.isabs(path): + fs = DataFileSystem(index=_repo.index.data["local"]) + fs_path = path + else: + fs = DVCFileSystem(repo=_repo, subrepos=True) + fs_path = fs.from_os_path(path) + + try: + if remote: + remote_odb = _repo.cloud.get_remote_odb(name=remote) + oid = fs.info(fs_path)["dvc_info"]["md5"] + fs = remote_odb.fs + fs_path = remote_odb.oid_to_path(oid) + + with fs.open( + fs_path, + mode=mode, + encoding=encoding, + ) as fobj: + yield fobj + except FileNotFoundError as exc: + raise FileMissingError(path) from exc + except IsADirectoryError as exc: + raise DvcIsADirectoryError(f"'{path}' is a directory") from exc def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None): diff --git a/dvc/commands/get.py b/dvc/commands/get.py index 882a2d17a..c8cc81795 100644 --- a/dvc/commands/get.py +++ b/dvc/commands/get.py @@ -14,12 +14,8 @@ class CmdGet(CmdBaseNoRepo): from dvc.api import get_url from dvc.ui import ui - try: - url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev) - ui.write(url, force=True) - except DvcException: - logger.exception("failed to show URL") - return 1 + url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev) + ui.write(url, force=True) return 0 diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index c9401fa8d..8e64f6ab5 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -84,14 +84,14 @@ class DataCloud: if bool(self.repo.config["remote"]): error_msg = ( - "no remote specified. Setup default remote with\n" + f"no remote specified in {self.repo}. Setup default remote with\n" " dvc remote default \n" "or use:\n" - " dvc {} -r ".format(command) + f" dvc {command} -r " ) else: error_msg = ( - "no remote specified. Create a default remote with\n" + f"no remote specified in {self.repo}. Create a default remote with\n" " dvc remote add -d " ) diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 0c9a902f2..92b368203 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -1,3 +1,4 @@ +import errno import os from collections import defaultdict from copy import copy @@ -113,7 +114,7 @@ class RepoDependency(Dependency): self, obj_only: bool = False, **kwargs ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "Meta", "HashFile"]: from dvc.config import NoRemoteError - from dvc.exceptions import NoOutputOrStageError, PathMissingError + from dvc.exceptions import NoOutputOrStageError from dvc.utils import as_posix from dvc_data.hashfile.build import build from dvc_data.hashfile.tree import Tree, TreeError @@ -150,8 +151,10 @@ class RepoDependency(Dependency): local_odb.fs.PARAM_CHECKSUM, ) except (FileNotFoundError, TreeError) as exc: - raise PathMissingError( - self.def_path, self.def_repo[self.PARAM_URL] + raise FileNotFoundError( + errno.ENOENT, + os.strerror(errno.ENOENT) + f" in {self.def_repo[self.PARAM_URL]}", + self.def_path, ) from exc object_store = copy(object_store) object_store.read_only = True diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 37dde9724..554a05eab 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -7,14 +7,7 @@ from typing import TYPE_CHECKING, Collection, Dict, Iterator, Optional, Tuple from funcy import retry, wrap_with -from dvc.exceptions import ( - FileMissingError, - NoOutputInExternalRepoError, - NoRemoteInExternalRepoError, - NotDvcRepoError, - OutputNotFoundError, - PathMissingError, -) +from dvc.exceptions import NotDvcRepoError from dvc.repo import Repo from dvc.scm import CloneError, map_scm_exception from dvc.utils import relpath @@ -37,8 +30,6 @@ def external_repo( cache_types: Optional[Collection[str]] = None, **kwargs, ) -> Iterator["Repo"]: - from dvc.config import NoRemoteError - logger.debug("Creating external repo %s@%s", url, rev) path = _cached_clone(url, rev, for_write=for_write) # Local HEAD points to the tip of whatever branch we first cloned from @@ -72,14 +63,6 @@ def external_repo( try: yield repo - except NoRemoteError as exc: - raise NoRemoteInExternalRepoError(url) from exc - except OutputNotFoundError as exc: - if exc.repo is repo: - raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url) from exc - raise - except FileMissingError as exc: - raise PathMissingError(exc.path, url) from exc finally: repo.close() if for_write: diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 27945559a..f80d19f3a 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -14,8 +14,6 @@ from typing import ( Union, ) -from dvc.exceptions import FileMissingError -from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError from dvc.exceptions import NotDvcRepoError, OutputNotFoundError from dvc.ignore import DvcIgnoreFilter from dvc.utils import env2bool @@ -586,38 +584,6 @@ class Repo: return os.path.join(repos_dir, repo_token) - @contextmanager - def open_by_relpath(self, path, remote=None, mode="r", encoding=None): - """Opens a specified resource as a file descriptor""" - from dvc.fs.data import DataFileSystem - from dvc.fs.dvc import DVCFileSystem - - fs: Union["FileSystem", DataFileSystem, DVCFileSystem] - if os.path.isabs(path): - fs = DataFileSystem(index=self.index.data["local"]) - fs_path = path - else: - fs = DVCFileSystem(repo=self, subrepos=True) - fs_path = fs.from_os_path(path) - - try: - if remote: - remote_odb = self.cloud.get_remote_odb(name=remote) - oid = fs.info(fs_path)["dvc_info"]["md5"] - fs = remote_odb.fs - fs_path = remote_odb.oid_to_path(oid) - - with fs.open( - fs_path, - mode=mode, - encoding=encoding, - ) as fobj: - yield fobj - except FileNotFoundError as exc: - raise FileMissingError(path) from exc - except IsADirectoryError as exc: - raise DvcIsADirectoryError(f"'{path}' is a directory") from exc - def close(self): self.scm.close() self.state.close() diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index 6e9748745..5cbfe6640 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -1,9 +1,9 @@ +import errno import logging import os from collections import defaultdict from typing import Dict, List, Optional -from dvc.exceptions import PathMissingError from dvc.repo import locked from dvc.ui import ui @@ -154,7 +154,7 @@ def diff( # check for overlapping missing targets between a_rev and b_rev for target in old_missing & new_missing: - raise PathMissingError(target, self) + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), target) old = indexes[a_rev] new = indexes[b_rev] diff --git a/dvc/repo/ls.py b/dvc/repo/ls.py index ea4ecc6d0..65b9b497a 100644 --- a/dvc/repo/ls.py +++ b/dvc/repo/ls.py @@ -1,8 +1,6 @@ import os from typing import TYPE_CHECKING, Optional -from dvc.exceptions import PathMissingError - if TYPE_CHECKING: from dvc.fs.dvc import DVCFileSystem @@ -61,10 +59,7 @@ def _ls( fs: "DVCFileSystem" = repo.dvcfs fs_path = fs.from_os_path(path) - try: - fs_path = fs.info(fs_path)["name"] - except FileNotFoundError: - raise PathMissingError(path, repo, dvc_only=dvc_only) # noqa: B904 + fs_path = fs.info(fs_path)["name"] infos = {} for root, dirs, files in fs.walk( diff --git a/tests/func/api/test_data.py b/tests/func/api/test_data.py index 9275af836..91cc63435 100644 --- a/tests/func/api/test_data.py +++ b/tests/func/api/test_data.py @@ -4,7 +4,7 @@ import pytest from funcy import first, get_in from dvc import api -from dvc.exceptions import FileMissingError, OutputNotFoundError, PathMissingError +from dvc.exceptions import OutputNotFoundError, PathMissingError from dvc.testing.api_tests import TestAPI # noqa, pylint: disable=unused-import from dvc.testing.tmp_dir import make_subrepo from dvc.utils.fs import remove @@ -75,7 +75,7 @@ def test_missing(tmp_dir, dvc, remote): remove("foo") - with pytest.raises(FileMissingError): + with pytest.raises(PathMissingError): api.read("foo") @@ -100,7 +100,7 @@ def test_open_not_cached(dvc): assert fd.read() == metric_content os.remove(metric_file) - with pytest.raises(FileMissingError): + with pytest.raises(PathMissingError): api.read(metric_file) diff --git a/tests/func/test_diff.py b/tests/func/test_diff.py index 55555fce0..8b0280063 100644 --- a/tests/func/test_diff.py +++ b/tests/func/test_diff.py @@ -358,11 +358,9 @@ def setup_targets_test(tmp_dir): def test_targets_missing_path(tmp_dir, scm, dvc): - from dvc.exceptions import PathMissingError - setup_targets_test(tmp_dir) - with pytest.raises(PathMissingError): + with pytest.raises(FileNotFoundError): dvc.diff(targets=["missing"]) diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index 569fb437e..1eedaf540 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -21,11 +21,11 @@ def test_external_repo(erepo_dir, mocker): clone_spy = mocker.spy(Git, "clone") with external_repo(url) as repo: - with repo.open_by_relpath("file") as fd: + with repo.dvcfs.open("file") as fd: assert fd.read() == "master" with external_repo(url, rev="branch") as repo: - with repo.open_by_relpath("file") as fd: + with repo.dvcfs.open("file") as fd: assert fd.read() == "branch" assert clone_spy.call_count == 1 @@ -122,7 +122,7 @@ def test_relative_remote(erepo_dir, tmp_dir): with external_repo(url) as repo: assert os.path.isabs(repo.config["remote"]["upstream"]["url"]) assert os.path.isdir(repo.config["remote"]["upstream"]["url"]) - with repo.open_by_relpath("file") as fd: + with repo.dvcfs.open("file") as fd: assert fd.read() == "contents" @@ -136,7 +136,7 @@ def test_shallow_clone_branch(erepo_dir, mocker): clone_spy = mocker.spy(Git, "clone") with external_repo(url, rev="branch") as repo: - with repo.open_by_relpath("file") as fd: + with repo.dvcfs.open("file") as fd: assert fd.read() == "branch" clone_spy.assert_called_with(url, ANY, shallow_branch="branch", progress=ANY) @@ -146,7 +146,7 @@ def test_shallow_clone_branch(erepo_dir, mocker): mock_fetch = mocker.patch.object(Git, "fetch") with external_repo(url) as repo: - with repo.open_by_relpath("file") as fd: + with repo.dvcfs.open("file") as fd: assert fd.read() == "master" mock_fetch.assert_called_with(unshallow=True) @@ -161,7 +161,7 @@ def test_shallow_clone_tag(erepo_dir, mocker): clone_spy = mocker.spy(Git, "clone") with external_repo(url, rev="v1") as repo: - with repo.open_by_relpath("file") as fd: + with repo.dvcfs.open("file") as fd: assert fd.read() == "foo" clone_spy.assert_called_with(url, ANY, shallow_branch="v1", progress=ANY) @@ -171,7 +171,7 @@ def test_shallow_clone_tag(erepo_dir, mocker): mock_fetch = mocker.patch.object(Git, "fetch") with external_repo(url, rev="master") as repo: - with repo.open_by_relpath("file") as fd: + with repo.dvcfs.open("file") as fd: assert fd.read() == "bar" mock_fetch.assert_called_with(unshallow=True) diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 1bb0ce31d..cce2f0e98 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -231,17 +231,15 @@ def test_get_url_not_existing(tmp_dir, erepo_dir, caplog): "--show-url", ] ) - == 1 + != 0 ) - assert "failed to show URL" in caplog.text def test_get_url_git_only_repo(tmp_dir, scm, caplog): tmp_dir.scm_gen({"foo": "foo"}, commit="initial") with caplog.at_level(logging.ERROR): - assert main(["get", os.fspath(tmp_dir), "foo", "--show-url"]) == 1 - assert "failed to show URL" in caplog.text + assert main(["get", os.fspath(tmp_dir), "foo", "--show-url"]) != 0 def test_get_pipeline_tracked_outs(tmp_dir, dvc, scm, git_dir, run_copy, local_remote): diff --git a/tests/func/test_import.py b/tests/func/test_import.py index ecaa327be..b087918f1 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -8,7 +8,7 @@ from dvc.annotations import Annotation from dvc.cachemgr import CacheManager from dvc.config import NoRemoteError from dvc.dvcfile import load_file -from dvc.exceptions import DownloadError, PathMissingError +from dvc.exceptions import DownloadError from dvc.fs import system from dvc.scm import Git from dvc.stage.exceptions import StagePathNotFoundError @@ -338,7 +338,7 @@ def test_push_wildcard_from_bare_git_repo( with dvc_repo.chdir(): dvc_repo.dvc.imp(os.fspath(tmp_dir), "dirextra") - with pytest.raises(PathMissingError): + with pytest.raises(FileNotFoundError): dvc_repo.dvc.imp(os.fspath(tmp_dir), "dir123") @@ -405,11 +405,11 @@ def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir): def test_import_non_existing(erepo_dir, tmp_dir, dvc): - with pytest.raises(PathMissingError): + with pytest.raises(FileNotFoundError): tmp_dir.dvc.imp(os.fspath(erepo_dir), "invalid_output") # https://github.com/iterative/dvc/pull/2837#discussion_r352123053 - with pytest.raises(PathMissingError): + with pytest.raises(FileNotFoundError): tmp_dir.dvc.imp(os.fspath(erepo_dir), "/root/", "root") diff --git a/tests/func/test_ls.py b/tests/func/test_ls.py index e24db4a61..602cd61dc 100644 --- a/tests/func/test_ls.py +++ b/tests/func/test_ls.py @@ -5,7 +5,6 @@ from operator import itemgetter import pytest -from dvc.exceptions import PathMissingError from dvc.repo import Repo from dvc.scm import CloneError @@ -151,7 +150,7 @@ def test_ls_repo_with_path_dir_dvc_only_empty(tmp_dir, dvc, scm): tmp_dir.scm_gen({"empty_scm_folder/": {}}, commit="add scm empty") tmp_dir.dvc_gen({"empty_dvc_folder": {}}, commit="empty dvc folder") - with pytest.raises(PathMissingError): + with pytest.raises(FileNotFoundError): Repo.ls(os.fspath(tmp_dir), path="not_exist_folder") assert Repo.ls(os.fspath(tmp_dir), path="empty_scm_folder") == [] @@ -218,23 +217,21 @@ def test_ls_repo_with_missed_path(tmp_dir, dvc, scm): tmp_dir.scm_gen(FS_STRUCTURE, commit="init") tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc") - with pytest.raises(PathMissingError) as exc_info: + with pytest.raises(FileNotFoundError): Repo.ls(os.fspath(tmp_dir), path="missed_path") - assert not exc_info.value.dvc_only def test_ls_repo_with_missed_path_dvc_only(tmp_dir, dvc, scm): tmp_dir.scm_gen(FS_STRUCTURE, commit="init") tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc") - with pytest.raises(PathMissingError) as exc_info: + with pytest.raises(FileNotFoundError): Repo.ls( os.fspath(tmp_dir), path="missed_path", recursive=True, dvc_only=True, ) - assert exc_info.value.dvc_only def test_ls_repo_with_removed_dvc_dir(tmp_dir, dvc, scm):