diff --git a/dvc/fs/base.py b/dvc/fs/base.py index c54bfc14b..4ab79a3fd 100644 --- a/dvc/fs/base.py +++ b/dvc/fs/base.py @@ -36,7 +36,7 @@ class RemoteMissingDepsError(DvcException): pass -class BaseFileSystem: +class FileSystem: sep = "/" scheme = "base" diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index c6dddc348..6322c45fd 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -7,7 +7,7 @@ from dvc.utils import relpath from ..progress import DEFAULT_CALLBACK from ._metadata import Metadata -from .base import BaseFileSystem +from .base import FileSystem if typing.TYPE_CHECKING: from dvc.output import Output @@ -16,7 +16,7 @@ if typing.TYPE_CHECKING: logger = logging.getLogger(__name__) -class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method +class DvcFileSystem(FileSystem): # pylint:disable=abstract-method """DVC repo fs. Args: diff --git a/dvc/fs/fsspec_wrapper.py b/dvc/fs/fsspec_wrapper.py index ab6bfa28f..33b5969e7 100644 --- a/dvc/fs/fsspec_wrapper.py +++ b/dvc/fs/fsspec_wrapper.py @@ -7,7 +7,7 @@ from tqdm.utils import CallbackIOWrapper from dvc.progress import DEFAULT_CALLBACK -from .base import BaseFileSystem +from .base import FileSystem FSPath = str AnyFSPath = str @@ -22,7 +22,7 @@ Entry = Dict[str, Any] # pylint: disable=no-member -class FSSpecWrapper(BaseFileSystem): +class FSSpecWrapper(FileSystem): TRAVERSE_PREFIX_LEN = 2 def __init__(self, **kwargs): diff --git a/dvc/fs/git.py b/dvc/fs/git.py index a48e2dfde..4feefba78 100644 --- a/dvc/fs/git.py +++ b/dvc/fs/git.py @@ -6,10 +6,10 @@ from tqdm.utils import CallbackIOWrapper from dvc.utils import is_exec, relpath from ..progress import DEFAULT_CALLBACK -from .base import BaseFileSystem +from .base import FileSystem -class GitFileSystem(BaseFileSystem): # pylint:disable=abstract-method +class GitFileSystem(FileSystem): # pylint:disable=abstract-method """Proxies the repo file access methods to Git objects""" sep = os.sep diff --git a/dvc/fs/local.py b/dvc/fs/local.py index 51bdfc0b5..1ddb876e0 100644 --- a/dvc/fs/local.py +++ b/dvc/fs/local.py @@ -7,12 +7,12 @@ from dvc.utils import is_exec, tmp_fname from dvc.utils.fs import copy_fobj_to_file, copyfile, makedirs, move, remove from ..progress import DEFAULT_CALLBACK -from .base import BaseFileSystem +from .base import FileSystem logger = logging.getLogger(__name__) -class LocalFileSystem(BaseFileSystem): +class LocalFileSystem(FileSystem): sep = os.sep scheme = Schemes.LOCAL diff --git a/dvc/fs/repo.py b/dvc/fs/repo.py index 46a830290..a3781340e 100644 --- a/dvc/fs/repo.py +++ b/dvc/fs/repo.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Callable, Optional, Tuple, Type, Union from funcy import lfilter, wrap_with from ..progress import DEFAULT_CALLBACK -from .base import BaseFileSystem +from .base import FileSystem from .dvc import DvcFileSystem if TYPE_CHECKING: @@ -19,7 +19,7 @@ logger = logging.getLogger(__name__) RepoFactory = Union[Callable[[str], "Repo"], Type["Repo"]] -class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method +class RepoFileSystem(FileSystem): # pylint:disable=abstract-method """DVC + git-tracked files fs. Args: @@ -193,9 +193,7 @@ class RepoFileSystem(BaseFileSystem): # pylint:disable=abstract-method repo_path = os.path.join(dir_path, Repo.DVC_DIR) return self._main_repo.fs.isdir(repo_path) - def _get_fs_pair( - self, path - ) -> Tuple[BaseFileSystem, Optional[DvcFileSystem]]: + def _get_fs_pair(self, path) -> Tuple[FileSystem, Optional[DvcFileSystem]]: """ Returns a pair of fss based on repo the path falls in, using prefix. """ diff --git a/dvc/fs/utils.py b/dvc/fs/utils.py index d408b8380..301a648d4 100644 --- a/dvc/fs/utils.py +++ b/dvc/fs/utils.py @@ -7,7 +7,7 @@ from .base import RemoteActionNotImplemented from .local import LocalFileSystem if TYPE_CHECKING: - from .base import BaseFileSystem + from .base import FileSystem from .types import AnyPath logger = logging.getLogger(__name__) @@ -15,9 +15,9 @@ logger = logging.getLogger(__name__) def _link( link: "AnyPath", - from_fs: "BaseFileSystem", + from_fs: "FileSystem", from_path: "AnyPath", - to_fs: "BaseFileSystem", + to_fs: "FileSystem", to_path: "AnyPath", ) -> None: if not isinstance(from_fs, type(to_fs)): @@ -35,9 +35,9 @@ def _link( def _copy( - from_fs: "BaseFileSystem", + from_fs: "FileSystem", from_path: "AnyPath", - to_fs: "BaseFileSystem", + to_fs: "FileSystem", to_path: "AnyPath", ) -> None: if isinstance(from_fs, LocalFileSystem): @@ -53,9 +53,9 @@ def _copy( def _try_links( links: List["AnyPath"], - from_fs: "BaseFileSystem", + from_fs: "FileSystem", from_path: "AnyPath", - to_fs: "BaseFileSystem", + to_fs: "FileSystem", to_path: "AnyPath", ) -> None: error = None @@ -80,9 +80,9 @@ def _try_links( def transfer( - from_fs: "BaseFileSystem", + from_fs: "FileSystem", from_path: "AnyPath", - to_fs: "BaseFileSystem", + to_fs: "FileSystem", to_path: "AnyPath", hardlink: bool = False, links: Optional[List["AnyPath"]] = None, @@ -115,9 +115,9 @@ def transfer( def _test_link( link: "AnyPath", - from_fs: "BaseFileSystem", + from_fs: "FileSystem", from_file: "AnyPath", - to_fs: "BaseFileSystem", + to_fs: "FileSystem", to_file: "AnyPath", ) -> bool: try: @@ -137,9 +137,9 @@ def _test_link( def test_links( links: List["AnyPath"], - from_fs: "BaseFileSystem", + from_fs: "FileSystem", from_path: "AnyPath", - to_fs: "BaseFileSystem", + to_fs: "FileSystem", to_path: "AnyPath", ) -> List["AnyPath"]: from dvc.utils import tmp_fname diff --git a/dvc/ignore.py b/dvc/ignore.py index a52e8767d..3eadb0cf4 100644 --- a/dvc/ignore.py +++ b/dvc/ignore.py @@ -7,7 +7,7 @@ from itertools import groupby, takewhile from pathspec.patterns import GitWildMatchPattern from pathspec.util import normalize_file -from dvc.fs.base import BaseFileSystem +from dvc.fs.base import FileSystem from dvc.fs.local import localfs from dvc.pathspec_math import PatternInfo, merge_patterns from dvc.scheme import Schemes @@ -267,7 +267,7 @@ class DvcIgnoreFilter: dirs, files = ignore_pattern(abs_root, dirs, files) return dirs, files - def walk(self, fs: BaseFileSystem, path: AnyPath, **kwargs): + def walk(self, fs: FileSystem, path: AnyPath, **kwargs): ignore_subrepos = kwargs.pop("ignore_subrepos", True) if fs.scheme == Schemes.LOCAL: for root, dirs, files in fs.walk(path, **kwargs): @@ -278,7 +278,7 @@ class DvcIgnoreFilter: else: yield from fs.walk(path, **kwargs) - def find(self, fs: BaseFileSystem, path: AnyPath, **kwargs): + def find(self, fs: FileSystem, path: AnyPath, **kwargs): if fs.scheme == Schemes.LOCAL: for root, _, files in self.walk(fs, path, **kwargs): for file in files: @@ -371,7 +371,7 @@ class DvcIgnoreFilter: return _no_match(target) def is_ignored( - self, fs: BaseFileSystem, path: str, ignore_subrepos: bool = True + self, fs: FileSystem, path: str, ignore_subrepos: bool = True ) -> bool: # NOTE: can't use self.check_ignore(path).match for now, see # https://github.com/iterative/dvc/issues/4555 diff --git a/dvc/objects/db/base.py b/dvc/objects/db/base.py index 7abcb28dd..1ff68c268 100644 --- a/dvc/objects/db/base.py +++ b/dvc/objects/db/base.py @@ -10,7 +10,7 @@ from dvc.objects.file import HashFile from dvc.progress import Tqdm if TYPE_CHECKING: - from dvc.fs.base import BaseFileSystem + from dvc.fs.base import FileSystem from dvc.hash_info import HashInfo from dvc.types import AnyPath @@ -23,7 +23,7 @@ class ObjectDB: DEFAULT_CACHE_TYPES = ["copy"] CACHE_MODE: Optional[int] = None - def __init__(self, fs: "BaseFileSystem", path: str, **config): + def __init__(self, fs: "FileSystem", path: str, **config): from dvc.state import StateNoop self.fs = fs @@ -75,7 +75,7 @@ class ObjectDB: def _add_file( self, - from_fs: "BaseFileSystem", + from_fs: "FileSystem", from_info: "AnyPath", to_info: "AnyPath", _hash_info: "HashInfo", @@ -95,7 +95,7 @@ class ObjectDB: def add( self, fs_path: "AnyPath", - fs: "BaseFileSystem", + fs: "FileSystem", hash_info: "HashInfo", hardlink: bool = False, verify: Optional[bool] = None, diff --git a/dvc/objects/db/reference.py b/dvc/objects/db/reference.py index 62ff22711..517c46c76 100644 --- a/dvc/objects/db/reference.py +++ b/dvc/objects/db/reference.py @@ -10,7 +10,7 @@ from ..reference import ReferenceHashFile from .base import ObjectDB if TYPE_CHECKING: - from dvc.fs.base import BaseFileSystem + from dvc.fs.base import FileSystem from dvc.hash_info import HashInfo from dvc.types import AnyPath @@ -26,7 +26,7 @@ class ReferenceObjectDB(ObjectDB): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._fs_cache: Dict[tuple, "BaseFileSystem"] = {} + self._fs_cache: Dict[tuple, "FileSystem"] = {} self._obj_cache: Dict["HashInfo", "ReferenceHashFile"] = {} def get(self, hash_info: "HashInfo"): @@ -54,7 +54,7 @@ class ReferenceObjectDB(ObjectDB): def _add_file( self, - from_fs: "BaseFileSystem", + from_fs: "FileSystem", from_info: "AnyPath", to_info: "AnyPath", hash_info: "HashInfo", diff --git a/dvc/objects/file.py b/dvc/objects/file.py index a17699520..b4435ba37 100644 --- a/dvc/objects/file.py +++ b/dvc/objects/file.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Optional from .errors import ObjectFormatError if TYPE_CHECKING: - from dvc.fs.base import BaseFileSystem + from dvc.fs.base import FileSystem from dvc.hash_info import HashInfo from dvc.types import AnyPath @@ -19,7 +19,7 @@ class HashFile: def __init__( self, fs_path: Optional["AnyPath"], - fs: Optional["BaseFileSystem"], + fs: Optional["FileSystem"], hash_info: "HashInfo", name: Optional[str] = None, ): diff --git a/dvc/objects/reference.py b/dvc/objects/reference.py index 93a9eb0db..7fe7c414d 100644 --- a/dvc/objects/reference.py +++ b/dvc/objects/reference.py @@ -8,7 +8,7 @@ from .errors import ObjectFormatError from .file import HashFile if TYPE_CHECKING: - from dvc.fs.base import BaseFileSystem + from dvc.fs.base import FileSystem from dvc.hash_info import HashInfo from dvc.types import AnyPath @@ -26,7 +26,7 @@ class ReferenceHashFile(HashFile): def __init__( self, fs_path: "AnyPath", - fs: "BaseFileSystem", + fs: "FileSystem", hash_info: "HashInfo", checksum: Optional[str] = None, **kwargs, @@ -109,7 +109,7 @@ class ReferenceHashFile(HashFile): ) @staticmethod - def config_tuple(fs: "BaseFileSystem"): + def config_tuple(fs: "FileSystem"): return ( fs.scheme, tuple( diff --git a/dvc/objects/stage.py b/dvc/objects/stage.py index b63c87a98..d487af53d 100644 --- a/dvc/objects/stage.py +++ b/dvc/objects/stage.py @@ -16,7 +16,7 @@ from .file import HashFile from .meta import Meta if TYPE_CHECKING: - from dvc.fs.base import BaseFileSystem + from dvc.fs.base import FileSystem from dvc.types import AnyPath from .db.base import ObjectDB @@ -210,7 +210,7 @@ _url_cache: Dict[str, str] = {} def _make_staging_url( - fs: "BaseFileSystem", odb: "ObjectDB", fs_path: Optional[str] + fs: "FileSystem", odb: "ObjectDB", fs_path: Optional[str] ): from dvc.scheme import Schemes @@ -288,7 +288,7 @@ def _stage_external_tree_info(odb, tree, name): def stage( odb: "ObjectDB", fs_path: "AnyPath", - fs: "BaseFileSystem", + fs: "FileSystem", name: str, upload: bool = False, dry_run: bool = False, diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 22e459818..bbda875e5 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -15,7 +15,7 @@ from dvc.utils import env2bool from dvc.utils.fs import path_isin if TYPE_CHECKING: - from dvc.fs.base import BaseFileSystem + from dvc.fs.base import FileSystem from dvc.objects.file import HashFile from dvc.repo.scm_context import SCMContext from dvc.scm import Base @@ -307,11 +307,11 @@ class Repo: return None @property - def fs(self) -> "BaseFileSystem": + def fs(self) -> "FileSystem": return self._fs @fs.setter - def fs(self, fs: "BaseFileSystem"): + def fs(self, fs: "FileSystem"): self._fs = fs # Our graph cache is no longer valid, as it was based on the previous # fs. diff --git a/dvc/repo/index.py b/dvc/repo/index.py index 8989f78ec..a3f789c37 100644 --- a/dvc/repo/index.py +++ b/dvc/repo/index.py @@ -20,7 +20,7 @@ if TYPE_CHECKING: from pygtrie import Trie from dvc.dependency import Dependency, ParamsDependency - from dvc.fs.base import BaseFileSystem + from dvc.fs.base import FileSystem from dvc.objects import HashInfo, ObjectDB from dvc.output import Output from dvc.repo.stage import StageLoad @@ -35,7 +35,7 @@ class Index: def __init__( self, repo: "Repo", # pylint: disable=redefined-outer-name - fs: "BaseFileSystem" = None, + fs: "FileSystem" = None, stages: List["Stage"] = None, ) -> None: """Index is an immutable collection of stages. @@ -58,7 +58,7 @@ class Index: """ self.repo: "Repo" = repo - self.fs: "BaseFileSystem" = fs or repo.fs + self.fs: "FileSystem" = fs or repo.fs self.stage_collector: "StageLoad" = repo.stage if stages is not None: self.stages: List["Stage"] = stages diff --git a/dvc/utils/serialize/_common.py b/dvc/utils/serialize/_common.py index 0ef6ec864..8914dacf4 100644 --- a/dvc/utils/serialize/_common.py +++ b/dvc/utils/serialize/_common.py @@ -9,26 +9,26 @@ from typing_extensions import Protocol from dvc.exceptions import DvcException if TYPE_CHECKING: - from dvc.fs.base import BaseFileSystem + from dvc.fs.base import FileSystem from dvc.types import AnyPath class DumperFn(Protocol): def __call__( - self, path: "AnyPath", data: Any, fs: "BaseFileSystem" = None + self, path: "AnyPath", data: Any, fs: "FileSystem" = None ) -> Any: ... class ModifierFn(Protocol): def __call__( - self, path: "AnyPath", fs: "BaseFileSystem" = None + self, path: "AnyPath", fs: "FileSystem" = None ) -> ContextManager[Dict]: ... class LoaderFn(Protocol): - def __call__(self, path: "AnyPath", fs: "BaseFileSystem" = None) -> Any: + def __call__(self, path: "AnyPath", fs: "FileSystem" = None) -> Any: ... @@ -55,7 +55,7 @@ class EncodingError(ParseError): super().__init__(path, f"is not valid {encoding}") -def _load_data(path: "AnyPath", parser: ParserFn, fs: "BaseFileSystem" = None): +def _load_data(path: "AnyPath", parser: ParserFn, fs: "FileSystem" = None): open_fn = fs.open if fs else open encoding = "utf-8" with open_fn(path, encoding=encoding) as fd: # type: ignore @@ -67,7 +67,7 @@ def _dump_data( path, data: Any, dumper: DumperFn, - fs: "BaseFileSystem" = None, + fs: "FileSystem" = None, **dumper_args, ): open_fn = fs.open if fs else open @@ -80,7 +80,7 @@ def _modify_data( path: "AnyPath", parser: ParserFn, dumper: DumperFn, - fs: "BaseFileSystem" = None, + fs: "FileSystem" = None, ): exists_fn = fs.exists if fs else os.path.exists file_exists = exists_fn(path) # type: ignore diff --git a/dvc/utils/strictyaml.py b/dvc/utils/strictyaml.py index 582949d58..ee4ba95a4 100644 --- a/dvc/utils/strictyaml.py +++ b/dvc/utils/strictyaml.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from ruamel.yaml import StreamMark from voluptuous import MultipleInvalid - from dvc.fs.base import BaseFileSystem + from dvc.fs.base import FileSystem from dvc.ui import RichText @@ -266,7 +266,7 @@ def validate( def load( path: str, schema: Callable[[_T], _T] = None, - fs: "BaseFileSystem" = None, + fs: "FileSystem" = None, encoding: str = "utf-8", round_trip: bool = False, ) -> Any: diff --git a/tests/unit/fs/test_base.py b/tests/unit/fs/test_base.py index 42016b32b..8d35654bf 100644 --- a/tests/unit/fs/test_base.py +++ b/tests/unit/fs/test_base.py @@ -1,6 +1,6 @@ import pytest -from dvc.fs.base import BaseFileSystem, RemoteMissingDepsError +from dvc.fs.base import FileSystem, RemoteMissingDepsError @pytest.mark.parametrize( @@ -13,7 +13,7 @@ from dvc.fs.base import BaseFileSystem, RemoteMissingDepsError ) def test_missing_deps(pkg, msg, mocker): requires = {"missing": "missing"} - mocker.patch.object(BaseFileSystem, "REQUIRES", requires) + mocker.patch.object(FileSystem, "REQUIRES", requires) mocker.patch("dvc.utils.pkg.PKG", pkg) with pytest.raises(RemoteMissingDepsError, match=msg): - BaseFileSystem() + FileSystem() diff --git a/tests/unit/remote/test_base.py b/tests/unit/remote/test_base.py index fca93bd54..686a191e3 100644 --- a/tests/unit/remote/test_base.py +++ b/tests/unit/remote/test_base.py @@ -4,7 +4,7 @@ from unittest import mock import pytest -from dvc.fs.base import BaseFileSystem, RemoteCmdError +from dvc.fs.base import FileSystem, RemoteCmdError from dvc.objects.db.base import ObjectDB @@ -26,18 +26,18 @@ def test_cmd_error(dvc): err = "sed: expression #1, char 2: extra characters after command" with mock.patch.object( - BaseFileSystem, + FileSystem, "remove", side_effect=RemoteCmdError("base", cmd, ret, err), ): with pytest.raises(RemoteCmdError): - BaseFileSystem(**config).remove("file") + FileSystem(**config).remove("file") @mock.patch.object(ObjectDB, "list_hashes_traverse") @mock.patch.object(ObjectDB, "list_hashes_exists") def test_hashes_exist(object_exists, traverse, dvc): - odb = ObjectDB(BaseFileSystem(), None) + odb = ObjectDB(FileSystem(), None) # remote does not support traverse odb.fs.CAN_TRAVERSE = False @@ -86,7 +86,7 @@ def test_hashes_exist(object_exists, traverse, dvc): @mock.patch.object(ObjectDB, "list_hashes", return_value=[]) @mock.patch.object(ObjectDB, "_path_to_hash", side_effect=lambda x: x) def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc): - odb = ObjectDB(BaseFileSystem(), None) + odb = ObjectDB(FileSystem(), None) odb.fs_path = "foo" # parallel traverse @@ -111,7 +111,7 @@ def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc): def test_list_hashes(dvc): - odb = ObjectDB(BaseFileSystem(), None) + odb = ObjectDB(FileSystem(), None) odb.fs_path = "foo" with mock.patch.object( @@ -123,7 +123,7 @@ def test_list_hashes(dvc): def test_list_paths(dvc): path = "foo" - odb = ObjectDB(BaseFileSystem(), path) + odb = ObjectDB(FileSystem(), path) with mock.patch.object(odb.fs, "find", return_value=[]) as walk_mock: for _ in odb._list_paths(): @@ -142,4 +142,4 @@ def test_list_paths(dvc): [(None, False), ("", False), ("3456.dir", True), ("3456", False)], ) def test_is_dir_hash(hash_, result): - assert BaseFileSystem.is_dir_hash(hash_) == result + assert FileSystem.is_dir_hash(hash_) == result