From bb241315306451b86c86d2ed8544303ec0acd98a Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 13 Feb 2021 22:59:20 +0200 Subject: [PATCH] fs/outs/deps: remove supported() logic (#5462) --- dvc/dependency/__init__.py | 20 ++------------------ dvc/fs/__init__.py | 35 +++++++++++++++++------------------ dvc/fs/base.py | 12 ------------ dvc/info.py | 8 ++++---- dvc/output/__init__.py | 28 ++-------------------------- dvc/output/base.py | 4 ---- 6 files changed, 25 insertions(+), 82 deletions(-) diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index cef642821..76c957355 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -20,20 +20,6 @@ from dvc.scheme import Schemes from ..fs import get_cloud_fs from .repo import RepoDependency -DEPS = [ - AzureDependency, - GSDependency, - HDFSDependency, - HTTPDependency, - HTTPSDependency, - S3Dependency, - SSHDependency, - WebDAVDependency, - WebDAVSDependency, - WebHDFSDependency, - # NOTE: LocalDependency is the default choice -] - DEP_MAP = { Schemes.LOCAL: LocalDependency, Schemes.SSH: SSHDependency, @@ -75,10 +61,8 @@ def _get(stage, p, info): params = info.pop(ParamsDependency.PARAM_PARAMS) return ParamsDependency(stage, p, params) - for d in DEPS: - if d.supported(p): - return d(stage, p, info) - return LocalDependency(stage, p, info) + dep_cls = DEP_MAP.get(parsed.scheme, LocalDependency) + return dep_cls(stage, p, info) def loadd_from(stage, d_list): diff --git a/dvc/fs/__init__.py b/dvc/fs/__init__.py index a501695bf..bc543fee4 100644 --- a/dvc/fs/__init__.py +++ b/dvc/fs/__init__.py @@ -1,6 +1,7 @@ import posixpath from urllib.parse import urlparse +from ..scheme import Schemes from .azure import AzureFileSystem from .gdrive import GDriveFileSystem from .gs import GSFileSystem @@ -15,28 +16,26 @@ from .webdav import WebDAVFileSystem from .webdavs import WebDAVSFileSystem from .webhdfs import WebHDFSFileSystem -ALL_FS = [ - AzureFileSystem, - GDriveFileSystem, - GSFileSystem, - HDFSFileSystem, - HTTPFileSystem, - HTTPSFileSystem, - S3FileSystem, - SSHFileSystem, - OSSFileSystem, - WebDAVFileSystem, - WebDAVSFileSystem, - WebHDFSFileSystem +FS_MAP = { + Schemes.AZURE: AzureFileSystem, + Schemes.GDRIVE: GDriveFileSystem, + Schemes.GS: GSFileSystem, + Schemes.HDFS: HDFSFileSystem, + Schemes.WEBHDFS: WebHDFSFileSystem, + Schemes.HTTP: HTTPFileSystem, + Schemes.HTTPS: HTTPSFileSystem, + Schemes.S3: S3FileSystem, + Schemes.SSH: SSHFileSystem, + Schemes.OSS: OSSFileSystem, + Schemes.WEBDAV: WebDAVFileSystem, + Schemes.WEBDAVS: WebDAVSFileSystem, # NOTE: LocalFileSystem is the default -] +} def get_fs_cls(remote_conf): - for fs_cls in ALL_FS: - if fs_cls.supported(remote_conf): - return fs_cls - return LocalFileSystem + scheme = urlparse(remote_conf["url"]).scheme + return FS_MAP.get(scheme, LocalFileSystem) def get_fs_config(config, **kwargs): diff --git a/dvc/fs/base.py b/dvc/fs/base.py index 4c7e579d2..ebeb56c21 100644 --- a/dvc/fs/base.py +++ b/dvc/fs/base.py @@ -3,7 +3,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from functools import partial from multiprocessing import cpu_count from typing import Any, ClassVar, Dict, FrozenSet, Optional -from urllib.parse import urlparse from funcy import cached_property, decorator @@ -141,17 +140,6 @@ class BaseFileSystem: f"dependencies: {missing}. {hint}" ) - @classmethod - def supported(cls, config): - if isinstance(config, (str, bytes)): - url = config - else: - url = config["url"] - - # NOTE: silently skipping remote, calling code should handle that - parsed = urlparse(url) - return parsed.scheme == cls.scheme - @property def cache(self): return getattr(self.repo.cache, self.scheme) diff --git a/dvc/info.py b/dvc/info.py index e7e186c43..f0e52ecdc 100644 --- a/dvc/info.py +++ b/dvc/info.py @@ -7,7 +7,7 @@ import uuid import psutil from dvc.exceptions import DvcException, NotDvcRepoError -from dvc.fs import ALL_FS, get_fs_cls, get_fs_config +from dvc.fs import FS_MAP, get_fs_cls, get_fs_config from dvc.repo import Repo from dvc.scm.base import SCMError from dvc.system import System @@ -117,11 +117,11 @@ def _get_linktype_support_info(repo): def _get_supported_remotes(): supported_remotes = [] - for fs_cls in ALL_FS: + for scheme, fs_cls in FS_MAP.items(): if not fs_cls.get_missing_deps(): - supported_remotes.append(fs_cls.scheme) + supported_remotes.append(scheme) - if len(supported_remotes) == len(ALL_FS): + if len(supported_remotes) == len(FS_MAP): return "All remotes" if len(supported_remotes) == 1: diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index ea93c847c..3154299e3 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -20,15 +20,6 @@ from ..fs.local import LocalFileSystem from ..fs.s3 import S3FileSystem from ..fs.webhdfs import WebHDFSFileSystem -OUTS = [ - HDFSOutput, - S3Output, - GSOutput, - SSHOutput, - WebHDFSOutput, - # NOTE: LocalOutput is the default choice -] - OUTS_MAP = { Schemes.HDFS: HDFSOutput, Schemes.S3: S3Output, @@ -104,23 +95,8 @@ def _get( isexec=isexec, ) - for o in OUTS: - if o.supported(p): - return o( - stage, - p, - info, - cache=cache, - fs=None, - metric=metric, - plot=plot, - persist=persist, - checkpoint=checkpoint, - live=live, - desc=desc, - isexec=isexec, - ) - return LocalOutput( + out_cls = OUTS_MAP.get(parsed.scheme, LocalOutput) + return out_cls( stage, p, info, diff --git a/dvc/output/base.py b/dvc/output/base.py index 2779d3f9c..a9ea734e1 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -186,10 +186,6 @@ class BaseOutput: def dir_cache(self): return self.hash_info.dir_info - @classmethod - def supported(cls, url): - return cls.FS_CLS.supported(url) - @property def cache_path(self): return self.cache.hash_to_path_info(self.hash_info.value).url