erepo: move legacy exceptions to api (#9228)

* erepo: get rid of NoRemoteInExternalRepoError

This exception hasn't been making any sense at all since we've introduced
get and import, as they can be chained in repos.

* erepo: get rid of NoOutputInExternalRepoError

This is dead code these days.

* erepo: get rid of FileMissingError and PathMissingError

* api: preserve legacy exc wrappers
This commit is contained in:
Ruslan Kuprieiev
2023-03-23 01:40:42 +02:00
committed by GitHub
parent 7868c1301c
commit 2d5c15bee1
14 changed files with 101 additions and 113 deletions
+66 -14
View File
@@ -1,12 +1,30 @@
from contextlib import _GeneratorContextManager as GCM
from contextlib import contextmanager
from typing import Optional
from funcy import reraise
from dvc.exceptions import OutputNotFoundError, PathMissingError
from dvc.exceptions import FileMissingError, OutputNotFoundError, PathMissingError
from dvc.repo import Repo
@contextmanager
def _wrap_exceptions(repo, url):
from dvc.config import NoRemoteError
from dvc.exceptions import NoOutputInExternalRepoError, NoRemoteInExternalRepoError
try:
yield
except NoRemoteError as exc:
raise NoRemoteInExternalRepoError(url) from exc
except OutputNotFoundError as exc:
if exc.repo is repo:
raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url) from exc
raise
except FileMissingError as exc:
raise PathMissingError(exc.path, url) from exc
def get_url(path, repo=None, rev=None, remote=None):
"""
Returns the URL to the storage location of a data file or directory tracked
@@ -19,18 +37,20 @@ def get_url(path, repo=None, rev=None, remote=None):
directory in the remote storage.
"""
with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
fs_path = _repo.dvcfs.from_os_path(path)
with reraise(FileNotFoundError, PathMissingError(path, repo)):
info = _repo.dvcfs.info(fs_path)
with _wrap_exceptions(_repo, path):
fs_path = _repo.dvcfs.from_os_path(path)
dvc_info = info.get("dvc_info")
if not dvc_info:
raise OutputNotFoundError(path, repo)
with reraise(FileNotFoundError, PathMissingError(path, repo)):
info = _repo.dvcfs.info(fs_path)
dvc_repo = info["repo"] # pylint: disable=unsubscriptable-object
md5 = dvc_info["md5"]
dvc_info = info.get("dvc_info")
if not dvc_info:
raise OutputNotFoundError(path, repo)
return dvc_repo.cloud.get_url_for(remote, checksum=md5)
dvc_repo = info["repo"] # pylint: disable=unsubscriptable-object
md5 = dvc_info["md5"]
return dvc_repo.cloud.get_url_for(remote, checksum=md5)
class _OpenContextManager(GCM):
@@ -195,10 +215,42 @@ def open( # noqa, pylint: disable=redefined-builtin
def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
with _repo.open_by_relpath(
path, remote=remote, mode=mode, encoding=encoding
) as fd:
yield fd
with _wrap_exceptions(_repo, path):
import os
from typing import TYPE_CHECKING, Union
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
from dvc.fs.data import DataFileSystem
from dvc.fs.dvc import DVCFileSystem
if TYPE_CHECKING:
from dvc.fs import FileSystem
fs: Union["FileSystem", DataFileSystem, DVCFileSystem]
if os.path.isabs(path):
fs = DataFileSystem(index=_repo.index.data["local"])
fs_path = path
else:
fs = DVCFileSystem(repo=_repo, subrepos=True)
fs_path = fs.from_os_path(path)
try:
if remote:
remote_odb = _repo.cloud.get_remote_odb(name=remote)
oid = fs.info(fs_path)["dvc_info"]["md5"]
fs = remote_odb.fs
fs_path = remote_odb.oid_to_path(oid)
with fs.open(
fs_path,
mode=mode,
encoding=encoding,
) as fobj:
yield fobj
except FileNotFoundError as exc:
raise FileMissingError(path) from exc
except IsADirectoryError as exc:
raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
+2 -6
View File
@@ -14,12 +14,8 @@ class CmdGet(CmdBaseNoRepo):
from dvc.api import get_url
from dvc.ui import ui
try:
url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev)
ui.write(url, force=True)
except DvcException:
logger.exception("failed to show URL")
return 1
url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev)
ui.write(url, force=True)
return 0
+3 -3
View File
@@ -84,14 +84,14 @@ class DataCloud:
if bool(self.repo.config["remote"]):
error_msg = (
"no remote specified. Setup default remote with\n"
f"no remote specified in {self.repo}. Setup default remote with\n"
" dvc remote default <remote name>\n"
"or use:\n"
" dvc {} -r <remote name>".format(command)
f" dvc {command} -r <remote name>"
)
else:
error_msg = (
"no remote specified. Create a default remote with\n"
f"no remote specified in {self.repo}. Create a default remote with\n"
" dvc remote add -d <remote name> <remote url>"
)
+6 -3
View File
@@ -1,3 +1,4 @@
import errno
import os
from collections import defaultdict
from copy import copy
@@ -113,7 +114,7 @@ class RepoDependency(Dependency):
self, obj_only: bool = False, **kwargs
) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "Meta", "HashFile"]:
from dvc.config import NoRemoteError
from dvc.exceptions import NoOutputOrStageError, PathMissingError
from dvc.exceptions import NoOutputOrStageError
from dvc.utils import as_posix
from dvc_data.hashfile.build import build
from dvc_data.hashfile.tree import Tree, TreeError
@@ -150,8 +151,10 @@ class RepoDependency(Dependency):
local_odb.fs.PARAM_CHECKSUM,
)
except (FileNotFoundError, TreeError) as exc:
raise PathMissingError(
self.def_path, self.def_repo[self.PARAM_URL]
raise FileNotFoundError(
errno.ENOENT,
os.strerror(errno.ENOENT) + f" in {self.def_repo[self.PARAM_URL]}",
self.def_path,
) from exc
object_store = copy(object_store)
object_store.read_only = True
+1 -18
View File
@@ -7,14 +7,7 @@ from typing import TYPE_CHECKING, Collection, Dict, Iterator, Optional, Tuple
from funcy import retry, wrap_with
from dvc.exceptions import (
FileMissingError,
NoOutputInExternalRepoError,
NoRemoteInExternalRepoError,
NotDvcRepoError,
OutputNotFoundError,
PathMissingError,
)
from dvc.exceptions import NotDvcRepoError
from dvc.repo import Repo
from dvc.scm import CloneError, map_scm_exception
from dvc.utils import relpath
@@ -37,8 +30,6 @@ def external_repo(
cache_types: Optional[Collection[str]] = None,
**kwargs,
) -> Iterator["Repo"]:
from dvc.config import NoRemoteError
logger.debug("Creating external repo %s@%s", url, rev)
path = _cached_clone(url, rev, for_write=for_write)
# Local HEAD points to the tip of whatever branch we first cloned from
@@ -72,14 +63,6 @@ def external_repo(
try:
yield repo
except NoRemoteError as exc:
raise NoRemoteInExternalRepoError(url) from exc
except OutputNotFoundError as exc:
if exc.repo is repo:
raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url) from exc
raise
except FileMissingError as exc:
raise PathMissingError(exc.path, url) from exc
finally:
repo.close()
if for_write:
-34
View File
@@ -14,8 +14,6 @@ from typing import (
Union,
)
from dvc.exceptions import FileMissingError
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
from dvc.ignore import DvcIgnoreFilter
from dvc.utils import env2bool
@@ -586,38 +584,6 @@ class Repo:
return os.path.join(repos_dir, repo_token)
@contextmanager
def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
"""Opens a specified resource as a file descriptor"""
from dvc.fs.data import DataFileSystem
from dvc.fs.dvc import DVCFileSystem
fs: Union["FileSystem", DataFileSystem, DVCFileSystem]
if os.path.isabs(path):
fs = DataFileSystem(index=self.index.data["local"])
fs_path = path
else:
fs = DVCFileSystem(repo=self, subrepos=True)
fs_path = fs.from_os_path(path)
try:
if remote:
remote_odb = self.cloud.get_remote_odb(name=remote)
oid = fs.info(fs_path)["dvc_info"]["md5"]
fs = remote_odb.fs
fs_path = remote_odb.oid_to_path(oid)
with fs.open(
fs_path,
mode=mode,
encoding=encoding,
) as fobj:
yield fobj
except FileNotFoundError as exc:
raise FileMissingError(path) from exc
except IsADirectoryError as exc:
raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
def close(self):
self.scm.close()
self.state.close()
+2 -2
View File
@@ -1,9 +1,9 @@
import errno
import logging
import os
from collections import defaultdict
from typing import Dict, List, Optional
from dvc.exceptions import PathMissingError
from dvc.repo import locked
from dvc.ui import ui
@@ -154,7 +154,7 @@ def diff(
# check for overlapping missing targets between a_rev and b_rev
for target in old_missing & new_missing:
raise PathMissingError(target, self)
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), target)
old = indexes[a_rev]
new = indexes[b_rev]
+1 -6
View File
@@ -1,8 +1,6 @@
import os
from typing import TYPE_CHECKING, Optional
from dvc.exceptions import PathMissingError
if TYPE_CHECKING:
from dvc.fs.dvc import DVCFileSystem
@@ -61,10 +59,7 @@ def _ls(
fs: "DVCFileSystem" = repo.dvcfs
fs_path = fs.from_os_path(path)
try:
fs_path = fs.info(fs_path)["name"]
except FileNotFoundError:
raise PathMissingError(path, repo, dvc_only=dvc_only) # noqa: B904
fs_path = fs.info(fs_path)["name"]
infos = {}
for root, dirs, files in fs.walk(
+3 -3
View File
@@ -4,7 +4,7 @@ import pytest
from funcy import first, get_in
from dvc import api
from dvc.exceptions import FileMissingError, OutputNotFoundError, PathMissingError
from dvc.exceptions import OutputNotFoundError, PathMissingError
from dvc.testing.api_tests import TestAPI # noqa, pylint: disable=unused-import
from dvc.testing.tmp_dir import make_subrepo
from dvc.utils.fs import remove
@@ -75,7 +75,7 @@ def test_missing(tmp_dir, dvc, remote):
remove("foo")
with pytest.raises(FileMissingError):
with pytest.raises(PathMissingError):
api.read("foo")
@@ -100,7 +100,7 @@ def test_open_not_cached(dvc):
assert fd.read() == metric_content
os.remove(metric_file)
with pytest.raises(FileMissingError):
with pytest.raises(PathMissingError):
api.read(metric_file)
+1 -3
View File
@@ -358,11 +358,9 @@ def setup_targets_test(tmp_dir):
def test_targets_missing_path(tmp_dir, scm, dvc):
from dvc.exceptions import PathMissingError
setup_targets_test(tmp_dir)
with pytest.raises(PathMissingError):
with pytest.raises(FileNotFoundError):
dvc.diff(targets=["missing"])
+7 -7
View File
@@ -21,11 +21,11 @@ def test_external_repo(erepo_dir, mocker):
clone_spy = mocker.spy(Git, "clone")
with external_repo(url) as repo:
with repo.open_by_relpath("file") as fd:
with repo.dvcfs.open("file") as fd:
assert fd.read() == "master"
with external_repo(url, rev="branch") as repo:
with repo.open_by_relpath("file") as fd:
with repo.dvcfs.open("file") as fd:
assert fd.read() == "branch"
assert clone_spy.call_count == 1
@@ -122,7 +122,7 @@ def test_relative_remote(erepo_dir, tmp_dir):
with external_repo(url) as repo:
assert os.path.isabs(repo.config["remote"]["upstream"]["url"])
assert os.path.isdir(repo.config["remote"]["upstream"]["url"])
with repo.open_by_relpath("file") as fd:
with repo.dvcfs.open("file") as fd:
assert fd.read() == "contents"
@@ -136,7 +136,7 @@ def test_shallow_clone_branch(erepo_dir, mocker):
clone_spy = mocker.spy(Git, "clone")
with external_repo(url, rev="branch") as repo:
with repo.open_by_relpath("file") as fd:
with repo.dvcfs.open("file") as fd:
assert fd.read() == "branch"
clone_spy.assert_called_with(url, ANY, shallow_branch="branch", progress=ANY)
@@ -146,7 +146,7 @@ def test_shallow_clone_branch(erepo_dir, mocker):
mock_fetch = mocker.patch.object(Git, "fetch")
with external_repo(url) as repo:
with repo.open_by_relpath("file") as fd:
with repo.dvcfs.open("file") as fd:
assert fd.read() == "master"
mock_fetch.assert_called_with(unshallow=True)
@@ -161,7 +161,7 @@ def test_shallow_clone_tag(erepo_dir, mocker):
clone_spy = mocker.spy(Git, "clone")
with external_repo(url, rev="v1") as repo:
with repo.open_by_relpath("file") as fd:
with repo.dvcfs.open("file") as fd:
assert fd.read() == "foo"
clone_spy.assert_called_with(url, ANY, shallow_branch="v1", progress=ANY)
@@ -171,7 +171,7 @@ def test_shallow_clone_tag(erepo_dir, mocker):
mock_fetch = mocker.patch.object(Git, "fetch")
with external_repo(url, rev="master") as repo:
with repo.open_by_relpath("file") as fd:
with repo.dvcfs.open("file") as fd:
assert fd.read() == "bar"
mock_fetch.assert_called_with(unshallow=True)
+2 -4
View File
@@ -231,17 +231,15 @@ def test_get_url_not_existing(tmp_dir, erepo_dir, caplog):
"--show-url",
]
)
== 1
!= 0
)
assert "failed to show URL" in caplog.text
def test_get_url_git_only_repo(tmp_dir, scm, caplog):
tmp_dir.scm_gen({"foo": "foo"}, commit="initial")
with caplog.at_level(logging.ERROR):
assert main(["get", os.fspath(tmp_dir), "foo", "--show-url"]) == 1
assert "failed to show URL" in caplog.text
assert main(["get", os.fspath(tmp_dir), "foo", "--show-url"]) != 0
def test_get_pipeline_tracked_outs(tmp_dir, dvc, scm, git_dir, run_copy, local_remote):
+4 -4
View File
@@ -8,7 +8,7 @@ from dvc.annotations import Annotation
from dvc.cachemgr import CacheManager
from dvc.config import NoRemoteError
from dvc.dvcfile import load_file
from dvc.exceptions import DownloadError, PathMissingError
from dvc.exceptions import DownloadError
from dvc.fs import system
from dvc.scm import Git
from dvc.stage.exceptions import StagePathNotFoundError
@@ -338,7 +338,7 @@ def test_push_wildcard_from_bare_git_repo(
with dvc_repo.chdir():
dvc_repo.dvc.imp(os.fspath(tmp_dir), "dirextra")
with pytest.raises(PathMissingError):
with pytest.raises(FileNotFoundError):
dvc_repo.dvc.imp(os.fspath(tmp_dir), "dir123")
@@ -405,11 +405,11 @@ def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir):
def test_import_non_existing(erepo_dir, tmp_dir, dvc):
with pytest.raises(PathMissingError):
with pytest.raises(FileNotFoundError):
tmp_dir.dvc.imp(os.fspath(erepo_dir), "invalid_output")
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
with pytest.raises(PathMissingError):
with pytest.raises(FileNotFoundError):
tmp_dir.dvc.imp(os.fspath(erepo_dir), "/root/", "root")
+3 -6
View File
@@ -5,7 +5,6 @@ from operator import itemgetter
import pytest
from dvc.exceptions import PathMissingError
from dvc.repo import Repo
from dvc.scm import CloneError
@@ -151,7 +150,7 @@ def test_ls_repo_with_path_dir_dvc_only_empty(tmp_dir, dvc, scm):
tmp_dir.scm_gen({"empty_scm_folder/": {}}, commit="add scm empty")
tmp_dir.dvc_gen({"empty_dvc_folder": {}}, commit="empty dvc folder")
with pytest.raises(PathMissingError):
with pytest.raises(FileNotFoundError):
Repo.ls(os.fspath(tmp_dir), path="not_exist_folder")
assert Repo.ls(os.fspath(tmp_dir), path="empty_scm_folder") == []
@@ -218,23 +217,21 @@ def test_ls_repo_with_missed_path(tmp_dir, dvc, scm):
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
with pytest.raises(PathMissingError) as exc_info:
with pytest.raises(FileNotFoundError):
Repo.ls(os.fspath(tmp_dir), path="missed_path")
assert not exc_info.value.dvc_only
def test_ls_repo_with_missed_path_dvc_only(tmp_dir, dvc, scm):
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
with pytest.raises(PathMissingError) as exc_info:
with pytest.raises(FileNotFoundError):
Repo.ls(
os.fspath(tmp_dir),
path="missed_path",
recursive=True,
dvc_only=True,
)
assert exc_info.value.dvc_only
def test_ls_repo_with_removed_dvc_dir(tmp_dir, dvc, scm):