mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
erepo: move legacy exceptions to api (#9228)
* erepo: get rid of NoRemoteInExternalRepoError This exception hasn't been making any sense at all since we've introduced get and import, as they can be chained in repos. * erepo: get rid of NoOutputInExternalRepoError This is dead code these days. * erepo: get rid of FileMissingError and PathMissingError * api: preserve legacy exc wrappers
This commit is contained in:
+66
-14
@@ -1,12 +1,30 @@
|
||||
from contextlib import _GeneratorContextManager as GCM
|
||||
from contextlib import contextmanager
|
||||
from typing import Optional
|
||||
|
||||
from funcy import reraise
|
||||
|
||||
from dvc.exceptions import OutputNotFoundError, PathMissingError
|
||||
from dvc.exceptions import FileMissingError, OutputNotFoundError, PathMissingError
|
||||
from dvc.repo import Repo
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _wrap_exceptions(repo, url):
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.exceptions import NoOutputInExternalRepoError, NoRemoteInExternalRepoError
|
||||
|
||||
try:
|
||||
yield
|
||||
except NoRemoteError as exc:
|
||||
raise NoRemoteInExternalRepoError(url) from exc
|
||||
except OutputNotFoundError as exc:
|
||||
if exc.repo is repo:
|
||||
raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url) from exc
|
||||
raise
|
||||
except FileMissingError as exc:
|
||||
raise PathMissingError(exc.path, url) from exc
|
||||
|
||||
|
||||
def get_url(path, repo=None, rev=None, remote=None):
|
||||
"""
|
||||
Returns the URL to the storage location of a data file or directory tracked
|
||||
@@ -19,18 +37,20 @@ def get_url(path, repo=None, rev=None, remote=None):
|
||||
directory in the remote storage.
|
||||
"""
|
||||
with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
|
||||
fs_path = _repo.dvcfs.from_os_path(path)
|
||||
with reraise(FileNotFoundError, PathMissingError(path, repo)):
|
||||
info = _repo.dvcfs.info(fs_path)
|
||||
with _wrap_exceptions(_repo, path):
|
||||
fs_path = _repo.dvcfs.from_os_path(path)
|
||||
|
||||
dvc_info = info.get("dvc_info")
|
||||
if not dvc_info:
|
||||
raise OutputNotFoundError(path, repo)
|
||||
with reraise(FileNotFoundError, PathMissingError(path, repo)):
|
||||
info = _repo.dvcfs.info(fs_path)
|
||||
|
||||
dvc_repo = info["repo"] # pylint: disable=unsubscriptable-object
|
||||
md5 = dvc_info["md5"]
|
||||
dvc_info = info.get("dvc_info")
|
||||
if not dvc_info:
|
||||
raise OutputNotFoundError(path, repo)
|
||||
|
||||
return dvc_repo.cloud.get_url_for(remote, checksum=md5)
|
||||
dvc_repo = info["repo"] # pylint: disable=unsubscriptable-object
|
||||
md5 = dvc_info["md5"]
|
||||
|
||||
return dvc_repo.cloud.get_url_for(remote, checksum=md5)
|
||||
|
||||
|
||||
class _OpenContextManager(GCM):
|
||||
@@ -195,10 +215,42 @@ def open( # noqa, pylint: disable=redefined-builtin
|
||||
|
||||
def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
|
||||
with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
|
||||
with _repo.open_by_relpath(
|
||||
path, remote=remote, mode=mode, encoding=encoding
|
||||
) as fd:
|
||||
yield fd
|
||||
with _wrap_exceptions(_repo, path):
|
||||
import os
|
||||
from typing import TYPE_CHECKING, Union
|
||||
|
||||
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
|
||||
from dvc.fs.data import DataFileSystem
|
||||
from dvc.fs.dvc import DVCFileSystem
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.fs import FileSystem
|
||||
|
||||
fs: Union["FileSystem", DataFileSystem, DVCFileSystem]
|
||||
if os.path.isabs(path):
|
||||
fs = DataFileSystem(index=_repo.index.data["local"])
|
||||
fs_path = path
|
||||
else:
|
||||
fs = DVCFileSystem(repo=_repo, subrepos=True)
|
||||
fs_path = fs.from_os_path(path)
|
||||
|
||||
try:
|
||||
if remote:
|
||||
remote_odb = _repo.cloud.get_remote_odb(name=remote)
|
||||
oid = fs.info(fs_path)["dvc_info"]["md5"]
|
||||
fs = remote_odb.fs
|
||||
fs_path = remote_odb.oid_to_path(oid)
|
||||
|
||||
with fs.open(
|
||||
fs_path,
|
||||
mode=mode,
|
||||
encoding=encoding,
|
||||
) as fobj:
|
||||
yield fobj
|
||||
except FileNotFoundError as exc:
|
||||
raise FileMissingError(path) from exc
|
||||
except IsADirectoryError as exc:
|
||||
raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
|
||||
|
||||
|
||||
def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
|
||||
|
||||
+2
-6
@@ -14,12 +14,8 @@ class CmdGet(CmdBaseNoRepo):
|
||||
from dvc.api import get_url
|
||||
from dvc.ui import ui
|
||||
|
||||
try:
|
||||
url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev)
|
||||
ui.write(url, force=True)
|
||||
except DvcException:
|
||||
logger.exception("failed to show URL")
|
||||
return 1
|
||||
url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev)
|
||||
ui.write(url, force=True)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
+3
-3
@@ -84,14 +84,14 @@ class DataCloud:
|
||||
|
||||
if bool(self.repo.config["remote"]):
|
||||
error_msg = (
|
||||
"no remote specified. Setup default remote with\n"
|
||||
f"no remote specified in {self.repo}. Setup default remote with\n"
|
||||
" dvc remote default <remote name>\n"
|
||||
"or use:\n"
|
||||
" dvc {} -r <remote name>".format(command)
|
||||
f" dvc {command} -r <remote name>"
|
||||
)
|
||||
else:
|
||||
error_msg = (
|
||||
"no remote specified. Create a default remote with\n"
|
||||
f"no remote specified in {self.repo}. Create a default remote with\n"
|
||||
" dvc remote add -d <remote name> <remote url>"
|
||||
)
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import errno
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from copy import copy
|
||||
@@ -113,7 +114,7 @@ class RepoDependency(Dependency):
|
||||
self, obj_only: bool = False, **kwargs
|
||||
) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "Meta", "HashFile"]:
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.exceptions import NoOutputOrStageError, PathMissingError
|
||||
from dvc.exceptions import NoOutputOrStageError
|
||||
from dvc.utils import as_posix
|
||||
from dvc_data.hashfile.build import build
|
||||
from dvc_data.hashfile.tree import Tree, TreeError
|
||||
@@ -150,8 +151,10 @@ class RepoDependency(Dependency):
|
||||
local_odb.fs.PARAM_CHECKSUM,
|
||||
)
|
||||
except (FileNotFoundError, TreeError) as exc:
|
||||
raise PathMissingError(
|
||||
self.def_path, self.def_repo[self.PARAM_URL]
|
||||
raise FileNotFoundError(
|
||||
errno.ENOENT,
|
||||
os.strerror(errno.ENOENT) + f" in {self.def_repo[self.PARAM_URL]}",
|
||||
self.def_path,
|
||||
) from exc
|
||||
object_store = copy(object_store)
|
||||
object_store.read_only = True
|
||||
|
||||
+1
-18
@@ -7,14 +7,7 @@ from typing import TYPE_CHECKING, Collection, Dict, Iterator, Optional, Tuple
|
||||
|
||||
from funcy import retry, wrap_with
|
||||
|
||||
from dvc.exceptions import (
|
||||
FileMissingError,
|
||||
NoOutputInExternalRepoError,
|
||||
NoRemoteInExternalRepoError,
|
||||
NotDvcRepoError,
|
||||
OutputNotFoundError,
|
||||
PathMissingError,
|
||||
)
|
||||
from dvc.exceptions import NotDvcRepoError
|
||||
from dvc.repo import Repo
|
||||
from dvc.scm import CloneError, map_scm_exception
|
||||
from dvc.utils import relpath
|
||||
@@ -37,8 +30,6 @@ def external_repo(
|
||||
cache_types: Optional[Collection[str]] = None,
|
||||
**kwargs,
|
||||
) -> Iterator["Repo"]:
|
||||
from dvc.config import NoRemoteError
|
||||
|
||||
logger.debug("Creating external repo %s@%s", url, rev)
|
||||
path = _cached_clone(url, rev, for_write=for_write)
|
||||
# Local HEAD points to the tip of whatever branch we first cloned from
|
||||
@@ -72,14 +63,6 @@ def external_repo(
|
||||
|
||||
try:
|
||||
yield repo
|
||||
except NoRemoteError as exc:
|
||||
raise NoRemoteInExternalRepoError(url) from exc
|
||||
except OutputNotFoundError as exc:
|
||||
if exc.repo is repo:
|
||||
raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url) from exc
|
||||
raise
|
||||
except FileMissingError as exc:
|
||||
raise PathMissingError(exc.path, url) from exc
|
||||
finally:
|
||||
repo.close()
|
||||
if for_write:
|
||||
|
||||
@@ -14,8 +14,6 @@ from typing import (
|
||||
Union,
|
||||
)
|
||||
|
||||
from dvc.exceptions import FileMissingError
|
||||
from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
|
||||
from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
|
||||
from dvc.ignore import DvcIgnoreFilter
|
||||
from dvc.utils import env2bool
|
||||
@@ -586,38 +584,6 @@ class Repo:
|
||||
|
||||
return os.path.join(repos_dir, repo_token)
|
||||
|
||||
@contextmanager
|
||||
def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
|
||||
"""Opens a specified resource as a file descriptor"""
|
||||
from dvc.fs.data import DataFileSystem
|
||||
from dvc.fs.dvc import DVCFileSystem
|
||||
|
||||
fs: Union["FileSystem", DataFileSystem, DVCFileSystem]
|
||||
if os.path.isabs(path):
|
||||
fs = DataFileSystem(index=self.index.data["local"])
|
||||
fs_path = path
|
||||
else:
|
||||
fs = DVCFileSystem(repo=self, subrepos=True)
|
||||
fs_path = fs.from_os_path(path)
|
||||
|
||||
try:
|
||||
if remote:
|
||||
remote_odb = self.cloud.get_remote_odb(name=remote)
|
||||
oid = fs.info(fs_path)["dvc_info"]["md5"]
|
||||
fs = remote_odb.fs
|
||||
fs_path = remote_odb.oid_to_path(oid)
|
||||
|
||||
with fs.open(
|
||||
fs_path,
|
||||
mode=mode,
|
||||
encoding=encoding,
|
||||
) as fobj:
|
||||
yield fobj
|
||||
except FileNotFoundError as exc:
|
||||
raise FileMissingError(path) from exc
|
||||
except IsADirectoryError as exc:
|
||||
raise DvcIsADirectoryError(f"'{path}' is a directory") from exc
|
||||
|
||||
def close(self):
|
||||
self.scm.close()
|
||||
self.state.close()
|
||||
|
||||
+2
-2
@@ -1,9 +1,9 @@
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from dvc.exceptions import PathMissingError
|
||||
from dvc.repo import locked
|
||||
from dvc.ui import ui
|
||||
|
||||
@@ -154,7 +154,7 @@ def diff(
|
||||
|
||||
# check for overlapping missing targets between a_rev and b_rev
|
||||
for target in old_missing & new_missing:
|
||||
raise PathMissingError(target, self)
|
||||
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), target)
|
||||
|
||||
old = indexes[a_rev]
|
||||
new = indexes[b_rev]
|
||||
|
||||
+1
-6
@@ -1,8 +1,6 @@
|
||||
import os
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from dvc.exceptions import PathMissingError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.fs.dvc import DVCFileSystem
|
||||
|
||||
@@ -61,10 +59,7 @@ def _ls(
|
||||
fs: "DVCFileSystem" = repo.dvcfs
|
||||
fs_path = fs.from_os_path(path)
|
||||
|
||||
try:
|
||||
fs_path = fs.info(fs_path)["name"]
|
||||
except FileNotFoundError:
|
||||
raise PathMissingError(path, repo, dvc_only=dvc_only) # noqa: B904
|
||||
fs_path = fs.info(fs_path)["name"]
|
||||
|
||||
infos = {}
|
||||
for root, dirs, files in fs.walk(
|
||||
|
||||
@@ -4,7 +4,7 @@ import pytest
|
||||
from funcy import first, get_in
|
||||
|
||||
from dvc import api
|
||||
from dvc.exceptions import FileMissingError, OutputNotFoundError, PathMissingError
|
||||
from dvc.exceptions import OutputNotFoundError, PathMissingError
|
||||
from dvc.testing.api_tests import TestAPI # noqa, pylint: disable=unused-import
|
||||
from dvc.testing.tmp_dir import make_subrepo
|
||||
from dvc.utils.fs import remove
|
||||
@@ -75,7 +75,7 @@ def test_missing(tmp_dir, dvc, remote):
|
||||
|
||||
remove("foo")
|
||||
|
||||
with pytest.raises(FileMissingError):
|
||||
with pytest.raises(PathMissingError):
|
||||
api.read("foo")
|
||||
|
||||
|
||||
@@ -100,7 +100,7 @@ def test_open_not_cached(dvc):
|
||||
assert fd.read() == metric_content
|
||||
|
||||
os.remove(metric_file)
|
||||
with pytest.raises(FileMissingError):
|
||||
with pytest.raises(PathMissingError):
|
||||
api.read(metric_file)
|
||||
|
||||
|
||||
|
||||
@@ -358,11 +358,9 @@ def setup_targets_test(tmp_dir):
|
||||
|
||||
|
||||
def test_targets_missing_path(tmp_dir, scm, dvc):
|
||||
from dvc.exceptions import PathMissingError
|
||||
|
||||
setup_targets_test(tmp_dir)
|
||||
|
||||
with pytest.raises(PathMissingError):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
dvc.diff(targets=["missing"])
|
||||
|
||||
|
||||
|
||||
@@ -21,11 +21,11 @@ def test_external_repo(erepo_dir, mocker):
|
||||
clone_spy = mocker.spy(Git, "clone")
|
||||
|
||||
with external_repo(url) as repo:
|
||||
with repo.open_by_relpath("file") as fd:
|
||||
with repo.dvcfs.open("file") as fd:
|
||||
assert fd.read() == "master"
|
||||
|
||||
with external_repo(url, rev="branch") as repo:
|
||||
with repo.open_by_relpath("file") as fd:
|
||||
with repo.dvcfs.open("file") as fd:
|
||||
assert fd.read() == "branch"
|
||||
|
||||
assert clone_spy.call_count == 1
|
||||
@@ -122,7 +122,7 @@ def test_relative_remote(erepo_dir, tmp_dir):
|
||||
with external_repo(url) as repo:
|
||||
assert os.path.isabs(repo.config["remote"]["upstream"]["url"])
|
||||
assert os.path.isdir(repo.config["remote"]["upstream"]["url"])
|
||||
with repo.open_by_relpath("file") as fd:
|
||||
with repo.dvcfs.open("file") as fd:
|
||||
assert fd.read() == "contents"
|
||||
|
||||
|
||||
@@ -136,7 +136,7 @@ def test_shallow_clone_branch(erepo_dir, mocker):
|
||||
clone_spy = mocker.spy(Git, "clone")
|
||||
|
||||
with external_repo(url, rev="branch") as repo:
|
||||
with repo.open_by_relpath("file") as fd:
|
||||
with repo.dvcfs.open("file") as fd:
|
||||
assert fd.read() == "branch"
|
||||
|
||||
clone_spy.assert_called_with(url, ANY, shallow_branch="branch", progress=ANY)
|
||||
@@ -146,7 +146,7 @@ def test_shallow_clone_branch(erepo_dir, mocker):
|
||||
|
||||
mock_fetch = mocker.patch.object(Git, "fetch")
|
||||
with external_repo(url) as repo:
|
||||
with repo.open_by_relpath("file") as fd:
|
||||
with repo.dvcfs.open("file") as fd:
|
||||
assert fd.read() == "master"
|
||||
mock_fetch.assert_called_with(unshallow=True)
|
||||
|
||||
@@ -161,7 +161,7 @@ def test_shallow_clone_tag(erepo_dir, mocker):
|
||||
|
||||
clone_spy = mocker.spy(Git, "clone")
|
||||
with external_repo(url, rev="v1") as repo:
|
||||
with repo.open_by_relpath("file") as fd:
|
||||
with repo.dvcfs.open("file") as fd:
|
||||
assert fd.read() == "foo"
|
||||
|
||||
clone_spy.assert_called_with(url, ANY, shallow_branch="v1", progress=ANY)
|
||||
@@ -171,7 +171,7 @@ def test_shallow_clone_tag(erepo_dir, mocker):
|
||||
|
||||
mock_fetch = mocker.patch.object(Git, "fetch")
|
||||
with external_repo(url, rev="master") as repo:
|
||||
with repo.open_by_relpath("file") as fd:
|
||||
with repo.dvcfs.open("file") as fd:
|
||||
assert fd.read() == "bar"
|
||||
mock_fetch.assert_called_with(unshallow=True)
|
||||
|
||||
|
||||
@@ -231,17 +231,15 @@ def test_get_url_not_existing(tmp_dir, erepo_dir, caplog):
|
||||
"--show-url",
|
||||
]
|
||||
)
|
||||
== 1
|
||||
!= 0
|
||||
)
|
||||
assert "failed to show URL" in caplog.text
|
||||
|
||||
|
||||
def test_get_url_git_only_repo(tmp_dir, scm, caplog):
|
||||
tmp_dir.scm_gen({"foo": "foo"}, commit="initial")
|
||||
|
||||
with caplog.at_level(logging.ERROR):
|
||||
assert main(["get", os.fspath(tmp_dir), "foo", "--show-url"]) == 1
|
||||
assert "failed to show URL" in caplog.text
|
||||
assert main(["get", os.fspath(tmp_dir), "foo", "--show-url"]) != 0
|
||||
|
||||
|
||||
def test_get_pipeline_tracked_outs(tmp_dir, dvc, scm, git_dir, run_copy, local_remote):
|
||||
|
||||
@@ -8,7 +8,7 @@ from dvc.annotations import Annotation
|
||||
from dvc.cachemgr import CacheManager
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.dvcfile import load_file
|
||||
from dvc.exceptions import DownloadError, PathMissingError
|
||||
from dvc.exceptions import DownloadError
|
||||
from dvc.fs import system
|
||||
from dvc.scm import Git
|
||||
from dvc.stage.exceptions import StagePathNotFoundError
|
||||
@@ -338,7 +338,7 @@ def test_push_wildcard_from_bare_git_repo(
|
||||
with dvc_repo.chdir():
|
||||
dvc_repo.dvc.imp(os.fspath(tmp_dir), "dirextra")
|
||||
|
||||
with pytest.raises(PathMissingError):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
dvc_repo.dvc.imp(os.fspath(tmp_dir), "dir123")
|
||||
|
||||
|
||||
@@ -405,11 +405,11 @@ def test_pull_non_workspace(tmp_dir, scm, dvc, erepo_dir):
|
||||
|
||||
|
||||
def test_import_non_existing(erepo_dir, tmp_dir, dvc):
|
||||
with pytest.raises(PathMissingError):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
tmp_dir.dvc.imp(os.fspath(erepo_dir), "invalid_output")
|
||||
|
||||
# https://github.com/iterative/dvc/pull/2837#discussion_r352123053
|
||||
with pytest.raises(PathMissingError):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
tmp_dir.dvc.imp(os.fspath(erepo_dir), "/root/", "root")
|
||||
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ from operator import itemgetter
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.exceptions import PathMissingError
|
||||
from dvc.repo import Repo
|
||||
from dvc.scm import CloneError
|
||||
|
||||
@@ -151,7 +150,7 @@ def test_ls_repo_with_path_dir_dvc_only_empty(tmp_dir, dvc, scm):
|
||||
tmp_dir.scm_gen({"empty_scm_folder/": {}}, commit="add scm empty")
|
||||
tmp_dir.dvc_gen({"empty_dvc_folder": {}}, commit="empty dvc folder")
|
||||
|
||||
with pytest.raises(PathMissingError):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
Repo.ls(os.fspath(tmp_dir), path="not_exist_folder")
|
||||
|
||||
assert Repo.ls(os.fspath(tmp_dir), path="empty_scm_folder") == []
|
||||
@@ -218,23 +217,21 @@ def test_ls_repo_with_missed_path(tmp_dir, dvc, scm):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
with pytest.raises(PathMissingError) as exc_info:
|
||||
with pytest.raises(FileNotFoundError):
|
||||
Repo.ls(os.fspath(tmp_dir), path="missed_path")
|
||||
assert not exc_info.value.dvc_only
|
||||
|
||||
|
||||
def test_ls_repo_with_missed_path_dvc_only(tmp_dir, dvc, scm):
|
||||
tmp_dir.scm_gen(FS_STRUCTURE, commit="init")
|
||||
tmp_dir.dvc_gen(DVC_STRUCTURE, commit="dvc")
|
||||
|
||||
with pytest.raises(PathMissingError) as exc_info:
|
||||
with pytest.raises(FileNotFoundError):
|
||||
Repo.ls(
|
||||
os.fspath(tmp_dir),
|
||||
path="missed_path",
|
||||
recursive=True,
|
||||
dvc_only=True,
|
||||
)
|
||||
assert exc_info.value.dvc_only
|
||||
|
||||
|
||||
def test_ls_repo_with_removed_dvc_dir(tmp_dir, dvc, scm):
|
||||
|
||||
Reference in New Issue
Block a user