From 8c9a9e9a1fc1dd76b9ddcbc0f550b4c3ea84b362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?= =?UTF-8?q?=E0=A4=A4=29?= Date: Tue, 17 May 2022 16:18:03 +0545 Subject: [PATCH] objects.fs: move schemes to objects.fs --- dvc/data/checkout.py | 2 +- dvc/data/db/__init__.py | 4 +-- dvc/data/db/reference.py | 4 +-- dvc/data/reference.py | 8 +++--- dvc/data/stage.py | 6 ++-- dvc/data/status.py | 4 +-- dvc/fs/__init__.py | 1 + dvc/fs/data.py | 2 +- dvc/fs/dvc.py | 2 +- dvc/fs/git.py | 2 +- dvc/ignore.py | 9 +++--- dvc/info.py | 2 +- dvc/objects/db.py | 2 +- dvc/objects/file.py | 2 +- dvc/objects/fs/__init__.py | 3 +- dvc/objects/fs/base.py | 15 +++++----- dvc/objects/fs/implementations/azure.py | 3 +- dvc/objects/fs/implementations/gdrive.py | 3 +- dvc/objects/fs/implementations/gs.py | 4 +-- dvc/objects/fs/implementations/hdfs.py | 3 +- dvc/objects/fs/implementations/http.py | 4 +-- dvc/objects/fs/implementations/https.py | 4 +-- dvc/objects/fs/implementations/local.py | 4 +-- dvc/objects/fs/implementations/memory.py | 4 +-- dvc/objects/fs/implementations/oss.py | 4 +-- dvc/objects/fs/implementations/s3.py | 4 +-- dvc/objects/fs/implementations/ssh.py | 4 +-- dvc/objects/fs/implementations/webdav.py | 6 ++-- dvc/objects/fs/implementations/webhdfs.py | 4 +-- dvc/{ => objects/fs}/scheme.py | 0 dvc/objects/hash.py | 2 +- dvc/output.py | 34 +++++++++++------------ dvc/repo/__init__.py | 2 +- dvc/repo/checkout.py | 2 +- dvc/repo/fetch.py | 4 +-- dvc/repo/index.py | 2 +- dvc/repo/move.py | 2 +- dvc/repo/stage.py | 2 +- dvc/stage/cache.py | 4 +-- tests/unit/remote/test_remote.py | 2 +- 40 files changed, 76 insertions(+), 99 deletions(-) rename dvc/{ => objects/fs}/scheme.py (100%) diff --git a/dvc/data/checkout.py b/dvc/data/checkout.py index 81d0760a0..756f2832d 100644 --- a/dvc/data/checkout.py +++ b/dvc/data/checkout.py @@ -223,7 +223,7 @@ def checkout( dvcignore: Optional[DvcIgnoreFilter] = None, state=None, ): - # if scheme(fs_path) not in ["local", cache.fs.scheme]: + # if protocol(fs_path) not in ["local", cache.fs.protocol]: # raise NotImplementedError diff = _diff( diff --git a/dvc/data/db/__init__.py b/dvc/data/db/__init__.py index d39d62bea..049037e85 100644 --- a/dvc/data/db/__init__.py +++ b/dvc/data/db/__init__.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING -from dvc.scheme import Schemes +from dvc.objects.fs import Schemes if TYPE_CHECKING: from .index import ObjectDBIndexBase @@ -11,7 +11,7 @@ def get_odb(fs, fs_path, **config): from .local import LocalObjectDB - if fs.scheme == Schemes.LOCAL: + if fs.protocol == Schemes.LOCAL: return LocalObjectDB(fs, fs_path, **config) return ObjectDB(fs, fs_path, **config) diff --git a/dvc/data/db/reference.py b/dvc/data/db/reference.py index 27b6f7f2d..a44f7bf24 100644 --- a/dvc/data/db/reference.py +++ b/dvc/data/db/reference.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Dict from dvc.objects.db import ObjectDB from dvc.objects.errors import ObjectFormatError -from dvc.scheme import Schemes +from dvc.objects.fs import Schemes from ..reference import ReferenceHashFile @@ -84,5 +84,5 @@ class ReferenceObjectDB(ObjectDB): logger.debug("'%s' file already exists, skipping", to_info) else: raise - if from_fs.scheme != Schemes.LOCAL: + if from_fs.protocol != Schemes.LOCAL: self._fs_cache[ReferenceHashFile.config_tuple(from_fs)] = from_fs diff --git a/dvc/data/reference.py b/dvc/data/reference.py index 70d8ba6da..16449fa50 100644 --- a/dvc/data/reference.py +++ b/dvc/data/reference.py @@ -84,14 +84,14 @@ class ReferenceHashFile(HashFile): except KeyError as exc: raise ObjectFormatError("ReferenceHashFile is corrupted") from exc - scheme, config_pairs = dict_.get(cls.PARAM_FS_CONFIG) - fs = fs_cache.get((scheme, config_pairs)) if fs_cache else None + protocol, config_pairs = dict_.get(cls.PARAM_FS_CONFIG) + fs = fs_cache.get((protocol, config_pairs)) if fs_cache else None if not fs: config = dict(config_pairs) if _DvcFileSystem.PARAM_REPO_URL in config: fs_cls = DvcFileSystem else: - fs_cls = get_fs_cls(config, scheme=scheme) + fs_cls = get_fs_cls(config, scheme=protocol) fs = fs_cls(**config) return ReferenceHashFile( fs_path, @@ -103,7 +103,7 @@ class ReferenceHashFile(HashFile): @staticmethod def config_tuple(fs: "FileSystem"): return ( - fs.scheme, + fs.protocol, tuple( (key, value) for key, value in sorted( diff --git a/dvc/data/stage.py b/dvc/data/stage.py index c03111c45..84cb35daa 100644 --- a/dvc/data/stage.py +++ b/dvc/data/stage.py @@ -170,12 +170,12 @@ _url_cache: Dict[str, str] = {} def _make_staging_url( fs: "FileSystem", odb: "ObjectDB", fs_path: Optional[str] ): - from dvc.scheme import Schemes + from dvc.objects.fs import Schemes url = f"{Schemes.MEMORY}://{_STAGING_MEMFS_PATH}" if fs_path is not None: - if odb.fs.scheme == Schemes.LOCAL: + if odb.fs.protocol == Schemes.LOCAL: fs_path = os.path.abspath(fs_path) if fs_path not in _url_cache: @@ -330,7 +330,7 @@ def stage( than the original source path. """ assert fs_path - # assert scheme(fs_path) == fs.scheme + # assert protocol(fs_path) == fs.protocol details = fs.info(fs_path) staging = _get_staging(odb) diff --git a/dvc/data/status.py b/dvc/data/status.py index 9f871f78b..3d942f17f 100644 --- a/dvc/data/status.py +++ b/dvc/data/status.py @@ -1,8 +1,8 @@ import logging from typing import TYPE_CHECKING, Dict, Iterable, NamedTuple, Optional, Set +from dvc.objects.fs import Schemes from dvc.objects.hash_info import HashInfo -from dvc.scheme import Schemes from .tree import Tree @@ -123,7 +123,7 @@ def status( dir_objs[hash_info.value] = tree hash_infos[hash_info.value] = hash_info - if odb.fs.scheme == Schemes.MEMORY: + if odb.fs.protocol == Schemes.MEMORY: # assume memfs staged objects already exist return StatusResult(set(hash_infos.values()), set()) diff --git a/dvc/fs/__init__.py b/dvc/fs/__init__.py index 98e66fbd6..59b3f023b 100644 --- a/dvc/fs/__init__.py +++ b/dvc/fs/__init__.py @@ -12,6 +12,7 @@ from dvc.objects.fs import ( # noqa: F401 MemoryFileSystem, OSSFileSystem, S3FileSystem, + Schemes, SSHFileSystem, WebDAVFileSystem, WebDAVSFileSystem, diff --git a/dvc/fs/data.py b/dvc/fs/data.py index 6c6aed3ce..cba392eb2 100644 --- a/dvc/fs/data.py +++ b/dvc/fs/data.py @@ -179,7 +179,7 @@ class _DataFileSystem(AbstractFileSystem): # pylint:disable=abstract-method class DataFileSystem(FileSystem): - scheme = "local" + protocol = "local" PARAM_CHECKSUM = "md5" diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index 31beb1be9..b824a1a8b 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -444,7 +444,7 @@ class _DvcFileSystem(AbstractFileSystem): # pylint:disable=abstract-method class DvcFileSystem(FileSystem): - scheme = "local" + protocol = "local" PARAM_CHECKSUM = "md5" def _prepare_credentials(self, **config): diff --git a/dvc/fs/git.py b/dvc/fs/git.py index 281aa2f66..a80e3ced1 100644 --- a/dvc/fs/git.py +++ b/dvc/fs/git.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: class GitFileSystem(FileSystem): # pylint:disable=abstract-method """Proxies the repo file access methods to Git objects""" - scheme = "local" + protocol = "local" def __init__( self, diff --git a/dvc/ignore.py b/dvc/ignore.py index 6f54cf596..b26d62ff1 100644 --- a/dvc/ignore.py +++ b/dvc/ignore.py @@ -8,9 +8,8 @@ from pathspec.patterns import GitWildMatchPattern from pathspec.util import normalize_file from pygtrie import Trie -from dvc.fs import AnyFSPath, FileSystem, localfs +from dvc.fs import AnyFSPath, FileSystem, Schemes, localfs from dvc.pathspec_math import PatternInfo, merge_patterns -from dvc.scheme import Schemes from dvc.types import List, Optional logger = logging.getLogger(__name__) @@ -302,7 +301,7 @@ class DvcIgnoreFilter: def walk(self, fs: FileSystem, path: AnyFSPath, **kwargs): ignore_subrepos = kwargs.pop("ignore_subrepos", True) - if fs.scheme == Schemes.LOCAL: + if fs.protocol == Schemes.LOCAL: for root, dirs, files in fs.walk(path, **kwargs): dirs[:], files[:] = self( root, dirs, files, ignore_subrepos=ignore_subrepos @@ -312,7 +311,7 @@ class DvcIgnoreFilter: yield from fs.walk(path, **kwargs) def find(self, fs: FileSystem, path: AnyFSPath, **kwargs): - if fs.scheme == Schemes.LOCAL: + if fs.protocol == Schemes.LOCAL: for root, _, files in self.walk(fs, path, **kwargs): for file in files: # NOTE: os.path.join is ~5.5 times slower @@ -405,7 +404,7 @@ class DvcIgnoreFilter: ) -> bool: # NOTE: can't use self.check_ignore(path).match for now, see # https://github.com/iterative/dvc/issues/4555 - if fs.scheme != Schemes.LOCAL: + if fs.protocol != Schemes.LOCAL: return False if fs.isfile(path): return self.is_ignored_file(path, ignore_subrepos) diff --git a/dvc/info.py b/dvc/info.py index 12d2f86a3..1a9fce9dc 100644 --- a/dvc/info.py +++ b/dvc/info.py @@ -72,7 +72,7 @@ def _get_caches(cache): def _get_remotes(config): schemes = ( - get_fs_cls(get_fs_config(None, config, name=remote)).scheme + get_fs_cls(get_fs_config(None, config, name=remote)).protocol for remote in config["remote"] ) diff --git a/dvc/objects/db.py b/dvc/objects/db.py index 0dec70f2b..7d35a092f 100644 --- a/dvc/objects/db.py +++ b/dvc/objects/db.py @@ -66,7 +66,7 @@ class ObjectDB: ) def __hash__(self): - return hash((self.fs.scheme, self.fs_path)) + return hash((self.fs.protocol, self.fs_path)) def exists(self, hash_info: "HashInfo"): return self.fs.exists(self.hash_to_path(hash_info.value)) diff --git a/dvc/objects/file.py b/dvc/objects/file.py index 579ae2d19..03a07a24b 100644 --- a/dvc/objects/file.py +++ b/dvc/objects/file.py @@ -47,7 +47,7 @@ class HashFile: ( self.hash_info, self.fs_path, - self.fs.scheme if self.fs else None, + self.fs.protocol if self.fs else None, ) ) diff --git a/dvc/objects/fs/__init__.py b/dvc/objects/fs/__init__.py index bc0dfd56e..0a6e9e9eb 100644 --- a/dvc/objects/fs/__init__.py +++ b/dvc/objects/fs/__init__.py @@ -1,7 +1,5 @@ from urllib.parse import urlparse -from dvc.scheme import Schemes - from . import generic # noqa: F401 from .implementations.azure import AzureFileSystem from .implementations.gdrive import GDriveFileSystem @@ -16,6 +14,7 @@ from .implementations.s3 import S3FileSystem from .implementations.ssh import SSHFileSystem from .implementations.webdav import WebDAVFileSystem, WebDAVSFileSystem from .implementations.webhdfs import WebHDFSFileSystem +from .scheme import Schemes FS_MAP = { Schemes.AZURE: AzureFileSystem, diff --git a/dvc/objects/fs/base.py b/dvc/objects/fs/base.py index 22efd61fa..dac86e4a7 100644 --- a/dvc/objects/fs/base.py +++ b/dvc/objects/fs/base.py @@ -55,7 +55,7 @@ class RemoteMissingDepsError(DvcException): class FileSystem: sep = "/" - scheme = "base" + protocol = "base" REQUIRES: ClassVar[Dict[str, str]] = {} _JOBS = 4 * cpu_count() @@ -131,17 +131,18 @@ class FileSystem: return missing def _check_requires(self, **kwargs): - from dvc.scheme import Schemes from dvc.utils import format_link from dvc.utils.pkg import PKG + from .scheme import Schemes + missing = self.get_missing_deps() if not missing: return - url = kwargs.get("url", f"{self.scheme}://") + url = kwargs.get("url", f"{self.protocol}://") - scheme = self.scheme + scheme = self.protocol if scheme == Schemes.WEBDAVS: scheme = Schemes.WEBDAV @@ -294,19 +295,19 @@ class FileSystem: try: return self.fs.symlink(from_info, to_info) except AttributeError: - raise RemoteActionNotImplemented("symlink", self.scheme) + raise RemoteActionNotImplemented("symlink", self.protocol) def hardlink(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None: try: return self.fs.hardlink(from_info, to_info) except AttributeError: - raise RemoteActionNotImplemented("hardlink", self.scheme) + raise RemoteActionNotImplemented("hardlink", self.protocol) def reflink(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None: try: return self.fs.reflink(from_info, to_info) except AttributeError: - raise RemoteActionNotImplemented("reflink", self.scheme) + raise RemoteActionNotImplemented("reflink", self.protocol) def is_symlink(self, path: AnyFSPath) -> bool: try: diff --git a/dvc/objects/fs/implementations/azure.py b/dvc/objects/fs/implementations/azure.py index 4c9434ea7..f9cc4c1d6 100644 --- a/dvc/objects/fs/implementations/azure.py +++ b/dvc/objects/fs/implementations/azure.py @@ -7,7 +7,6 @@ from fsspec.utils import infer_storage_options from funcy import cached_property, memoize, wrap_prop from dvc.exceptions import DvcException -from dvc.scheme import Schemes from dvc.utils import format_link from ..base import ObjectFileSystem @@ -39,7 +38,7 @@ def _az_config(): # pylint:disable=abstract-method class AzureFileSystem(ObjectFileSystem): - scheme = Schemes.AZURE + protocol = "azure" PARAM_CHECKSUM = "etag" REQUIRES = { "adlfs": "adlfs", diff --git a/dvc/objects/fs/implementations/gdrive.py b/dvc/objects/fs/implementations/gdrive.py index e766cfef7..2c841535e 100644 --- a/dvc/objects/fs/implementations/gdrive.py +++ b/dvc/objects/fs/implementations/gdrive.py @@ -5,7 +5,6 @@ import threading from funcy import cached_property, wrap_prop from dvc.exceptions import DvcException -from dvc.scheme import Schemes from dvc.utils import format_link from ..base import FileSystem @@ -32,7 +31,7 @@ class GDriveAuthError(DvcException): class GDriveFileSystem(FileSystem): # pylint:disable=abstract-method - scheme = Schemes.GDRIVE + protocol = "gdrive" PARAM_CHECKSUM = "checksum" REQUIRES = {"pydrive2": "pydrive2"} # Always prefer traverse for GDrive since API usage quotas are a concern. diff --git a/dvc/objects/fs/implementations/gs.py b/dvc/objects/fs/implementations/gs.py index cbfcc3676..334a2bb19 100644 --- a/dvc/objects/fs/implementations/gs.py +++ b/dvc/objects/fs/implementations/gs.py @@ -2,14 +2,12 @@ import threading from funcy import cached_property, wrap_prop -from dvc.scheme import Schemes - # pylint:disable=abstract-method from ..base import ObjectFileSystem class GSFileSystem(ObjectFileSystem): - scheme = Schemes.GS + protocol = "gs" REQUIRES = {"gcsfs": "gcsfs"} PARAM_CHECKSUM = "etag" diff --git a/dvc/objects/fs/implementations/hdfs.py b/dvc/objects/fs/implementations/hdfs.py index 6cb68a84c..399cc4966 100644 --- a/dvc/objects/fs/implementations/hdfs.py +++ b/dvc/objects/fs/implementations/hdfs.py @@ -5,7 +5,6 @@ import threading from funcy import cached_property, wrap_prop -from dvc.scheme import Schemes from dvc.utils import fix_env from ..base import FileSystem @@ -15,7 +14,7 @@ CHECKSUM_REGEX = re.compile(r".*\t.*\t(?P.*)") # pylint: disable=abstract-method class HDFSFileSystem(FileSystem): - scheme = Schemes.HDFS + protocol = "hdfs" REQUIRES = {"fsspec": "fsspec", "pyarrow": "pyarrow"} PARAM_CHECKSUM = "checksum" diff --git a/dvc/objects/fs/implementations/http.py b/dvc/objects/fs/implementations/http.py index 102291f74..bdc456743 100644 --- a/dvc/objects/fs/implementations/http.py +++ b/dvc/objects/fs/implementations/http.py @@ -4,8 +4,6 @@ from typing import BinaryIO, Union from funcy import cached_property, memoize, wrap_with -from dvc.scheme import Schemes - from .._callback import DEFAULT_CALLBACK, FsspecCallback from ..base import AnyFSPath, FileSystem @@ -31,7 +29,7 @@ def make_context(ssl_verify): # pylint: disable=abstract-method class HTTPFileSystem(FileSystem): - scheme = Schemes.HTTP + protocol = "http" PARAM_CHECKSUM = "checksum" REQUIRES = {"aiohttp": "aiohttp", "aiohttp-retry": "aiohttp_retry"} CAN_TRAVERSE = False diff --git a/dvc/objects/fs/implementations/https.py b/dvc/objects/fs/implementations/https.py index 25a93c192..7d3b8c777 100644 --- a/dvc/objects/fs/implementations/https.py +++ b/dvc/objects/fs/implementations/https.py @@ -1,7 +1,5 @@ -from dvc.scheme import Schemes - from .http import HTTPFileSystem class HTTPSFileSystem(HTTPFileSystem): # pylint:disable=abstract-method - scheme = Schemes.HTTPS + protocol = "https" diff --git a/dvc/objects/fs/implementations/local.py b/dvc/objects/fs/implementations/local.py index 0791386ce..7f888ec8f 100644 --- a/dvc/objects/fs/implementations/local.py +++ b/dvc/objects/fs/implementations/local.py @@ -6,8 +6,6 @@ import threading from fsspec import AbstractFileSystem from funcy import cached_property, wrap_prop -from dvc.scheme import Schemes - from .. import system from ..base import FileSystem from ..utils import copyfile, makedirs, move, remove, tmp_fname @@ -168,7 +166,7 @@ class FsspecLocalFileSystem(AbstractFileSystem): class LocalFileSystem(FileSystem): sep = os.sep - scheme = Schemes.LOCAL + protocol = "local" PARAM_CHECKSUM = "md5" PARAM_PATH = "path" TRAVERSE_PREFIX_LEN = 2 diff --git a/dvc/objects/fs/implementations/memory.py b/dvc/objects/fs/implementations/memory.py index a8e1b81d3..a305506bb 100644 --- a/dvc/objects/fs/implementations/memory.py +++ b/dvc/objects/fs/implementations/memory.py @@ -2,13 +2,11 @@ import threading from funcy import cached_property, wrap_prop -from dvc.scheme import Schemes - from ..base import FileSystem class MemoryFileSystem(FileSystem): # pylint:disable=abstract-method - scheme = Schemes.MEMORY + protocol = "memory" PARAM_CHECKSUM = "md5" def __eq__(self, other): diff --git a/dvc/objects/fs/implementations/oss.py b/dvc/objects/fs/implementations/oss.py index 2bd2d5c37..1a22a3890 100644 --- a/dvc/objects/fs/implementations/oss.py +++ b/dvc/objects/fs/implementations/oss.py @@ -4,8 +4,6 @@ import threading from funcy import cached_property, wrap_prop -from dvc.scheme import Schemes - from ..base import ObjectFileSystem logger = logging.getLogger(__name__) @@ -13,7 +11,7 @@ logger = logging.getLogger(__name__) # pylint:disable=abstract-method class OSSFileSystem(ObjectFileSystem): - scheme = Schemes.OSS + protocol = "oss" REQUIRES = {"ossfs": "ossfs"} PARAM_CHECKSUM = "etag" LIST_OBJECT_PAGE_SIZE = 100 diff --git a/dvc/objects/fs/implementations/s3.py b/dvc/objects/fs/implementations/s3.py index 38d97f3d8..2aa53f98d 100644 --- a/dvc/objects/fs/implementations/s3.py +++ b/dvc/objects/fs/implementations/s3.py @@ -4,8 +4,6 @@ from collections import defaultdict from funcy import cached_property, wrap_prop -from dvc.scheme import Schemes - from ..base import ObjectFileSystem _AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config") @@ -13,7 +11,7 @@ _AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config") # pylint:disable=abstract-method class S3FileSystem(ObjectFileSystem): - scheme = Schemes.S3 + protocol = "s3" REQUIRES = {"s3fs": "s3fs", "boto3": "boto3"} PARAM_CHECKSUM = "etag" diff --git a/dvc/objects/fs/implementations/ssh.py b/dvc/objects/fs/implementations/ssh.py index 1374c1ed3..c186d67fd 100644 --- a/dvc/objects/fs/implementations/ssh.py +++ b/dvc/objects/fs/implementations/ssh.py @@ -4,8 +4,6 @@ import threading from funcy import cached_property, memoize, silent, wrap_prop, wrap_with -from dvc.scheme import Schemes - from .._callback import DEFAULT_CALLBACK from ..base import FileSystem from ..utils import as_atomic @@ -24,7 +22,7 @@ def ask_password(host, user, port): # pylint:disable=abstract-method class SSHFileSystem(FileSystem): - scheme = Schemes.SSH + protocol = "ssh" REQUIRES = {"sshfs": "sshfs"} PARAM_CHECKSUM = "md5" diff --git a/dvc/objects/fs/implementations/webdav.py b/dvc/objects/fs/implementations/webdav.py index 174bbaa74..509899dff 100644 --- a/dvc/objects/fs/implementations/webdav.py +++ b/dvc/objects/fs/implementations/webdav.py @@ -3,8 +3,6 @@ import threading from funcy import cached_property, wrap_prop -from dvc.scheme import Schemes - from ..base import FileSystem from .http import ask_password @@ -12,7 +10,7 @@ logger = logging.getLogger(__name__) class WebDAVFileSystem(FileSystem): # pylint:disable=abstract-method - scheme = Schemes.WEBDAV + protocol = "webdav" root_marker = "" REQUIRES = {"webdav4": "webdav4"} PARAM_CHECKSUM = "etag" @@ -89,4 +87,4 @@ class WebDAVFileSystem(FileSystem): # pylint:disable=abstract-method class WebDAVSFileSystem(WebDAVFileSystem): # pylint:disable=abstract-method - scheme = Schemes.WEBDAVS + protocol = "webdavs" diff --git a/dvc/objects/fs/implementations/webhdfs.py b/dvc/objects/fs/implementations/webhdfs.py index 97a7d24c5..76e0920b5 100644 --- a/dvc/objects/fs/implementations/webhdfs.py +++ b/dvc/objects/fs/implementations/webhdfs.py @@ -2,14 +2,12 @@ import threading from funcy import cached_property, wrap_prop -from dvc.scheme import Schemes - # pylint:disable=abstract-method from ..base import FileSystem class WebHDFSFileSystem(FileSystem): - scheme = Schemes.WEBHDFS + protocol = "webhdfs" REQUIRES = {"fsspec": "fsspec"} PARAM_CHECKSUM = "checksum" diff --git a/dvc/scheme.py b/dvc/objects/fs/scheme.py similarity index 100% rename from dvc/scheme.py rename to dvc/objects/fs/scheme.py diff --git a/dvc/objects/hash.py b/dvc/objects/hash.py index 3ff648208..b3cb02daf 100644 --- a/dvc/objects/hash.py +++ b/dvc/objects/hash.py @@ -115,7 +115,7 @@ def _hash_file( name: str, callback: "FsspecCallback" = DEFAULT_CALLBACK, ) -> Tuple["str", Dict[str, Any]]: - info = _adapt_info(fs.info(fs_path), fs.scheme) + info = _adapt_info(fs.info(fs_path), fs.protocol) if name in info: assert not info[name].endswith(".dir") diff --git a/dvc/output.py b/dvc/output.py index 54a1d20d8..1b2e4d76b 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -30,10 +30,10 @@ from .fs import ( LocalFileSystem, RemoteMissingDepsError, S3FileSystem, + Schemes, get_cloud_fs, ) from .objects.errors import ObjectFormatError -from .scheme import Schemes from .utils import relpath from .utils.fs import path_isin @@ -295,7 +295,7 @@ class Output: self.fs = fs_cls(**fs_config) if ( - self.fs.scheme == "local" + self.fs.protocol == "local" and stage and isinstance(stage.repo.fs, LocalFileSystem) and path_isin(path, stage.repo.root_dir) @@ -307,7 +307,7 @@ class Output: if ( self.repo - and self.fs.scheme == "local" + and self.fs.protocol == "local" and not self.fs.path.isabs(self.def_path) ): self.fs = self.repo.fs @@ -339,7 +339,7 @@ class Output: self.remote = remote def _parse_path(self, fs, fs_path): - if fs.scheme != "local": + if fs.protocol != "local": return fs_path parsed = urlparse(self.def_path) @@ -362,7 +362,7 @@ class Output: ) def __str__(self): - if self.fs.scheme != "local": + if self.fs.protocol != "local": return self.def_path if ( @@ -379,12 +379,12 @@ class Output: return self.fs.path.relpath(self.fs_path, self.repo.root_dir) @property - def scheme(self): - return self.fs.scheme + def protocol(self): + return self.fs.protocol @property def is_in_repo(self): - if self.fs.scheme != "local": + if self.fs.protocol != "local": return False if urlparse(self.def_path).scheme == "remote": @@ -407,9 +407,9 @@ class Output: @property def odb(self): - odb = getattr(self.repo.odb, self.scheme) + odb = getattr(self.repo.odb, self.protocol) if self.use_cache and odb is None: - raise RemoteCacheRequiredError(self.fs.scheme, self.fs_path) + raise RemoteCacheRequiredError(self.fs.protocol, self.fs_path) return odb @property @@ -494,7 +494,7 @@ class Output: @property def dvcignore(self): - if self.fs.scheme == "local": + if self.fs.protocol == "local": return self.repo.dvcignore return None @@ -688,9 +688,9 @@ class Output: return ret def verify_metric(self): - if self.fs.scheme != "local": + if self.fs.protocol != "local": raise DvcException( - f"verify metric is not supported for {self.scheme}" + f"verify metric is not supported for {self.protocol}" ) if not self.metric or self.plot: @@ -786,7 +786,7 @@ class Output: def remove(self, ignore_remove=False): self.fs.remove(self.fs_path) - if self.scheme != Schemes.LOCAL: + if self.protocol != Schemes.LOCAL: return if ignore_remove: @@ -794,7 +794,7 @@ class Output: def move(self, out): # pylint: disable=no-member - if self.scheme == "local" and self.use_scm_ignore: + if self.protocol == "local" and self.use_scm_ignore: self.repo.scm_context.ignore_remove(self.fspath) self.fs.move(self.fs_path, out.fs_path) @@ -803,7 +803,7 @@ class Output: self.save() self.commit() - if self.scheme == "local" and self.use_scm_ignore: + if self.protocol == "local" and self.use_scm_ignore: self.repo.scm_context.ignore(self.fspath) def transfer( @@ -1000,7 +1000,7 @@ class Output: raise self.IsIgnoredError(check) def _check_can_merge(self, out): - if self.scheme != out.scheme: + if self.protocol != out.protocol: raise MergeError("unable to auto-merge outputs of different types") my = self.dumpd() diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index bd4f5c3d5..2bdcdbbff 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -464,7 +464,7 @@ class Repo: match = eq if strict else out.fs.path.isin_or_eq - if out.scheme == "local" and match(fs_path, out.fs_path): + if out.protocol == "local" and match(fs_path, out.fs_path): return True if recursive and out.fs.path.isin(out.fs_path, fs_path): diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index 041812610..079f33482 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -28,7 +28,7 @@ def _fspath_dir(path): def _remove_unused_links(repo): - used = [out.fspath for out in repo.index.outs if out.scheme == "local"] + used = [out.fspath for out in repo.index.outs if out.protocol == "local"] unused = repo.state.get_unused_links(used, repo.fs) ret = [_fspath_dir(u) for u in unused] repo.state.remove_links(unused, repo.fs) diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index 80acdf2d7..ea42c030d 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -2,7 +2,7 @@ import logging from typing import TYPE_CHECKING, Optional from dvc.exceptions import DownloadError, FileTransferError -from dvc.scheme import Schemes +from dvc.fs import Schemes from . import locked @@ -81,7 +81,7 @@ def fetch( for src_odb, obj_ids in sorted( used.items(), key=lambda item: item[0] is not None - and item[0].fs.scheme == Schemes.MEMORY, + and item[0].fs.protocol == Schemes.MEMORY, ): d, f = _fetch( self, diff --git a/dvc/repo/index.py b/dvc/repo/index.py index 8c0001d19..ebabd1ebe 100644 --- a/dvc/repo/index.py +++ b/dvc/repo/index.py @@ -184,7 +184,7 @@ class Index: out.fs_path, self.repo.root_dir ) else: - fs_key = out.fs.scheme + fs_key = out.fs.protocol key = out.fs.path.parts(out.fs_path) out.meta.odb = out.odb diff --git a/dvc/repo/move.py b/dvc/repo/move.py index a4f1a7c14..12f2f9816 100644 --- a/dvc/repo/move.py +++ b/dvc/repo/move.py @@ -38,7 +38,7 @@ def move(self, from_path, to_path): from ..dvcfile import DVC_FILE_SUFFIX, Dvcfile from_out = Output.loads_from(Stage(self), [from_path])[0] - assert from_out.scheme == "local" + assert from_out.protocol == "local" to_path = _expand_target_path(from_path, to_path) diff --git a/dvc/repo/stage.py b/dvc/repo/stage.py index 01c79f87a..26dfb773d 100644 --- a/dvc/repo/stage.py +++ b/dvc/repo/stage.py @@ -508,7 +508,7 @@ class StageLoad: out.fspath for stage in new_stages for out in stage.outs - if out.scheme == "local" + if out.protocol == "local" ) dirs[:] = [d for d in dirs if not is_out_or_ignored(root, d)] diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index 6d2131aa7..37bddc32b 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -38,11 +38,11 @@ def _can_hash(stage): return False for dep in stage.deps: - if not (dep.scheme == "local" and dep.def_path and dep.get_hash()): + if not (dep.protocol == "local" and dep.def_path and dep.get_hash()): return False for out in stage.outs: - if out.scheme != "local" or not out.def_path or out.persist: + if out.protocol != "local" or not out.def_path or out.persist: return False return True diff --git a/tests/unit/remote/test_remote.py b/tests/unit/remote/test_remote.py index a2cf3dd07..e9ff919f1 100644 --- a/tests/unit/remote/test_remote.py +++ b/tests/unit/remote/test_remote.py @@ -49,7 +49,7 @@ def test_remote_without_hash_jobs_default(dvc): @pytest.mark.parametrize("fs_cls", [GSFileSystem, S3FileSystem]) def test_makedirs_not_create_for_top_level_path(fs_cls, dvc, mocker): - url = f"{fs_cls.scheme}://bucket/" + url = f"{fs_cls.protocol}://bucket/" fs = fs_cls(url=url) mocked_client = mocker.PropertyMock() mocker.patch.object(fs_cls, "fs", mocked_client)