mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-21 23:08:59 +01:00
objects.fs: move schemes to objects.fs
This commit is contained in:
committed by
skshetry
parent
120eb09c4d
commit
8c9a9e9a1f
@@ -223,7 +223,7 @@ def checkout(
|
||||
dvcignore: Optional[DvcIgnoreFilter] = None,
|
||||
state=None,
|
||||
):
|
||||
# if scheme(fs_path) not in ["local", cache.fs.scheme]:
|
||||
# if protocol(fs_path) not in ["local", cache.fs.protocol]:
|
||||
# raise NotImplementedError
|
||||
|
||||
diff = _diff(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.objects.fs import Schemes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .index import ObjectDBIndexBase
|
||||
@@ -11,7 +11,7 @@ def get_odb(fs, fs_path, **config):
|
||||
|
||||
from .local import LocalObjectDB
|
||||
|
||||
if fs.scheme == Schemes.LOCAL:
|
||||
if fs.protocol == Schemes.LOCAL:
|
||||
return LocalObjectDB(fs, fs_path, **config)
|
||||
|
||||
return ObjectDB(fs, fs_path, **config)
|
||||
|
||||
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Dict
|
||||
|
||||
from dvc.objects.db import ObjectDB
|
||||
from dvc.objects.errors import ObjectFormatError
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.objects.fs import Schemes
|
||||
|
||||
from ..reference import ReferenceHashFile
|
||||
|
||||
@@ -84,5 +84,5 @@ class ReferenceObjectDB(ObjectDB):
|
||||
logger.debug("'%s' file already exists, skipping", to_info)
|
||||
else:
|
||||
raise
|
||||
if from_fs.scheme != Schemes.LOCAL:
|
||||
if from_fs.protocol != Schemes.LOCAL:
|
||||
self._fs_cache[ReferenceHashFile.config_tuple(from_fs)] = from_fs
|
||||
|
||||
@@ -84,14 +84,14 @@ class ReferenceHashFile(HashFile):
|
||||
except KeyError as exc:
|
||||
raise ObjectFormatError("ReferenceHashFile is corrupted") from exc
|
||||
|
||||
scheme, config_pairs = dict_.get(cls.PARAM_FS_CONFIG)
|
||||
fs = fs_cache.get((scheme, config_pairs)) if fs_cache else None
|
||||
protocol, config_pairs = dict_.get(cls.PARAM_FS_CONFIG)
|
||||
fs = fs_cache.get((protocol, config_pairs)) if fs_cache else None
|
||||
if not fs:
|
||||
config = dict(config_pairs)
|
||||
if _DvcFileSystem.PARAM_REPO_URL in config:
|
||||
fs_cls = DvcFileSystem
|
||||
else:
|
||||
fs_cls = get_fs_cls(config, scheme=scheme)
|
||||
fs_cls = get_fs_cls(config, scheme=protocol)
|
||||
fs = fs_cls(**config)
|
||||
return ReferenceHashFile(
|
||||
fs_path,
|
||||
@@ -103,7 +103,7 @@ class ReferenceHashFile(HashFile):
|
||||
@staticmethod
|
||||
def config_tuple(fs: "FileSystem"):
|
||||
return (
|
||||
fs.scheme,
|
||||
fs.protocol,
|
||||
tuple(
|
||||
(key, value)
|
||||
for key, value in sorted(
|
||||
|
||||
+3
-3
@@ -170,12 +170,12 @@ _url_cache: Dict[str, str] = {}
|
||||
def _make_staging_url(
|
||||
fs: "FileSystem", odb: "ObjectDB", fs_path: Optional[str]
|
||||
):
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.objects.fs import Schemes
|
||||
|
||||
url = f"{Schemes.MEMORY}://{_STAGING_MEMFS_PATH}"
|
||||
|
||||
if fs_path is not None:
|
||||
if odb.fs.scheme == Schemes.LOCAL:
|
||||
if odb.fs.protocol == Schemes.LOCAL:
|
||||
fs_path = os.path.abspath(fs_path)
|
||||
|
||||
if fs_path not in _url_cache:
|
||||
@@ -330,7 +330,7 @@ def stage(
|
||||
than the original source path.
|
||||
"""
|
||||
assert fs_path
|
||||
# assert scheme(fs_path) == fs.scheme
|
||||
# assert protocol(fs_path) == fs.protocol
|
||||
|
||||
details = fs.info(fs_path)
|
||||
staging = _get_staging(odb)
|
||||
|
||||
+2
-2
@@ -1,8 +1,8 @@
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Dict, Iterable, NamedTuple, Optional, Set
|
||||
|
||||
from dvc.objects.fs import Schemes
|
||||
from dvc.objects.hash_info import HashInfo
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .tree import Tree
|
||||
|
||||
@@ -123,7 +123,7 @@ def status(
|
||||
dir_objs[hash_info.value] = tree
|
||||
hash_infos[hash_info.value] = hash_info
|
||||
|
||||
if odb.fs.scheme == Schemes.MEMORY:
|
||||
if odb.fs.protocol == Schemes.MEMORY:
|
||||
# assume memfs staged objects already exist
|
||||
return StatusResult(set(hash_infos.values()), set())
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ from dvc.objects.fs import ( # noqa: F401
|
||||
MemoryFileSystem,
|
||||
OSSFileSystem,
|
||||
S3FileSystem,
|
||||
Schemes,
|
||||
SSHFileSystem,
|
||||
WebDAVFileSystem,
|
||||
WebDAVSFileSystem,
|
||||
|
||||
+1
-1
@@ -179,7 +179,7 @@ class _DataFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
|
||||
|
||||
class DataFileSystem(FileSystem):
|
||||
scheme = "local"
|
||||
protocol = "local"
|
||||
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
|
||||
+1
-1
@@ -444,7 +444,7 @@ class _DvcFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
|
||||
|
||||
|
||||
class DvcFileSystem(FileSystem):
|
||||
scheme = "local"
|
||||
protocol = "local"
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
def _prepare_credentials(self, **config):
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ if TYPE_CHECKING:
|
||||
class GitFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
"""Proxies the repo file access methods to Git objects"""
|
||||
|
||||
scheme = "local"
|
||||
protocol = "local"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
+4
-5
@@ -8,9 +8,8 @@ from pathspec.patterns import GitWildMatchPattern
|
||||
from pathspec.util import normalize_file
|
||||
from pygtrie import Trie
|
||||
|
||||
from dvc.fs import AnyFSPath, FileSystem, localfs
|
||||
from dvc.fs import AnyFSPath, FileSystem, Schemes, localfs
|
||||
from dvc.pathspec_math import PatternInfo, merge_patterns
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.types import List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -302,7 +301,7 @@ class DvcIgnoreFilter:
|
||||
|
||||
def walk(self, fs: FileSystem, path: AnyFSPath, **kwargs):
|
||||
ignore_subrepos = kwargs.pop("ignore_subrepos", True)
|
||||
if fs.scheme == Schemes.LOCAL:
|
||||
if fs.protocol == Schemes.LOCAL:
|
||||
for root, dirs, files in fs.walk(path, **kwargs):
|
||||
dirs[:], files[:] = self(
|
||||
root, dirs, files, ignore_subrepos=ignore_subrepos
|
||||
@@ -312,7 +311,7 @@ class DvcIgnoreFilter:
|
||||
yield from fs.walk(path, **kwargs)
|
||||
|
||||
def find(self, fs: FileSystem, path: AnyFSPath, **kwargs):
|
||||
if fs.scheme == Schemes.LOCAL:
|
||||
if fs.protocol == Schemes.LOCAL:
|
||||
for root, _, files in self.walk(fs, path, **kwargs):
|
||||
for file in files:
|
||||
# NOTE: os.path.join is ~5.5 times slower
|
||||
@@ -405,7 +404,7 @@ class DvcIgnoreFilter:
|
||||
) -> bool:
|
||||
# NOTE: can't use self.check_ignore(path).match for now, see
|
||||
# https://github.com/iterative/dvc/issues/4555
|
||||
if fs.scheme != Schemes.LOCAL:
|
||||
if fs.protocol != Schemes.LOCAL:
|
||||
return False
|
||||
if fs.isfile(path):
|
||||
return self.is_ignored_file(path, ignore_subrepos)
|
||||
|
||||
+1
-1
@@ -72,7 +72,7 @@ def _get_caches(cache):
|
||||
|
||||
def _get_remotes(config):
|
||||
schemes = (
|
||||
get_fs_cls(get_fs_config(None, config, name=remote)).scheme
|
||||
get_fs_cls(get_fs_config(None, config, name=remote)).protocol
|
||||
for remote in config["remote"]
|
||||
)
|
||||
|
||||
|
||||
+1
-1
@@ -66,7 +66,7 @@ class ObjectDB:
|
||||
)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.fs.scheme, self.fs_path))
|
||||
return hash((self.fs.protocol, self.fs_path))
|
||||
|
||||
def exists(self, hash_info: "HashInfo"):
|
||||
return self.fs.exists(self.hash_to_path(hash_info.value))
|
||||
|
||||
+1
-1
@@ -47,7 +47,7 @@ class HashFile:
|
||||
(
|
||||
self.hash_info,
|
||||
self.fs_path,
|
||||
self.fs.scheme if self.fs else None,
|
||||
self.fs.protocol if self.fs else None,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from . import generic # noqa: F401
|
||||
from .implementations.azure import AzureFileSystem
|
||||
from .implementations.gdrive import GDriveFileSystem
|
||||
@@ -16,6 +14,7 @@ from .implementations.s3 import S3FileSystem
|
||||
from .implementations.ssh import SSHFileSystem
|
||||
from .implementations.webdav import WebDAVFileSystem, WebDAVSFileSystem
|
||||
from .implementations.webhdfs import WebHDFSFileSystem
|
||||
from .scheme import Schemes
|
||||
|
||||
FS_MAP = {
|
||||
Schemes.AZURE: AzureFileSystem,
|
||||
|
||||
@@ -55,7 +55,7 @@ class RemoteMissingDepsError(DvcException):
|
||||
class FileSystem:
|
||||
sep = "/"
|
||||
|
||||
scheme = "base"
|
||||
protocol = "base"
|
||||
REQUIRES: ClassVar[Dict[str, str]] = {}
|
||||
_JOBS = 4 * cpu_count()
|
||||
|
||||
@@ -131,17 +131,18 @@ class FileSystem:
|
||||
return missing
|
||||
|
||||
def _check_requires(self, **kwargs):
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import format_link
|
||||
from dvc.utils.pkg import PKG
|
||||
|
||||
from .scheme import Schemes
|
||||
|
||||
missing = self.get_missing_deps()
|
||||
if not missing:
|
||||
return
|
||||
|
||||
url = kwargs.get("url", f"{self.scheme}://")
|
||||
url = kwargs.get("url", f"{self.protocol}://")
|
||||
|
||||
scheme = self.scheme
|
||||
scheme = self.protocol
|
||||
if scheme == Schemes.WEBDAVS:
|
||||
scheme = Schemes.WEBDAV
|
||||
|
||||
@@ -294,19 +295,19 @@ class FileSystem:
|
||||
try:
|
||||
return self.fs.symlink(from_info, to_info)
|
||||
except AttributeError:
|
||||
raise RemoteActionNotImplemented("symlink", self.scheme)
|
||||
raise RemoteActionNotImplemented("symlink", self.protocol)
|
||||
|
||||
def hardlink(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None:
|
||||
try:
|
||||
return self.fs.hardlink(from_info, to_info)
|
||||
except AttributeError:
|
||||
raise RemoteActionNotImplemented("hardlink", self.scheme)
|
||||
raise RemoteActionNotImplemented("hardlink", self.protocol)
|
||||
|
||||
def reflink(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None:
|
||||
try:
|
||||
return self.fs.reflink(from_info, to_info)
|
||||
except AttributeError:
|
||||
raise RemoteActionNotImplemented("reflink", self.scheme)
|
||||
raise RemoteActionNotImplemented("reflink", self.protocol)
|
||||
|
||||
def is_symlink(self, path: AnyFSPath) -> bool:
|
||||
try:
|
||||
|
||||
@@ -7,7 +7,6 @@ from fsspec.utils import infer_storage_options
|
||||
from funcy import cached_property, memoize, wrap_prop
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import format_link
|
||||
|
||||
from ..base import ObjectFileSystem
|
||||
@@ -39,7 +38,7 @@ def _az_config():
|
||||
|
||||
# pylint:disable=abstract-method
|
||||
class AzureFileSystem(ObjectFileSystem):
|
||||
scheme = Schemes.AZURE
|
||||
protocol = "azure"
|
||||
PARAM_CHECKSUM = "etag"
|
||||
REQUIRES = {
|
||||
"adlfs": "adlfs",
|
||||
|
||||
@@ -5,7 +5,6 @@ import threading
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import format_link
|
||||
|
||||
from ..base import FileSystem
|
||||
@@ -32,7 +31,7 @@ class GDriveAuthError(DvcException):
|
||||
|
||||
|
||||
class GDriveFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
scheme = Schemes.GDRIVE
|
||||
protocol = "gdrive"
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
REQUIRES = {"pydrive2": "pydrive2"}
|
||||
# Always prefer traverse for GDrive since API usage quotas are a concern.
|
||||
|
||||
@@ -2,14 +2,12 @@ import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
# pylint:disable=abstract-method
|
||||
from ..base import ObjectFileSystem
|
||||
|
||||
|
||||
class GSFileSystem(ObjectFileSystem):
|
||||
scheme = Schemes.GS
|
||||
protocol = "gs"
|
||||
REQUIRES = {"gcsfs": "gcsfs"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.utils import fix_env
|
||||
|
||||
from ..base import FileSystem
|
||||
@@ -15,7 +14,7 @@ CHECKSUM_REGEX = re.compile(r".*\t.*\t(?P<checksum>.*)")
|
||||
|
||||
# pylint: disable=abstract-method
|
||||
class HDFSFileSystem(FileSystem):
|
||||
scheme = Schemes.HDFS
|
||||
protocol = "hdfs"
|
||||
REQUIRES = {"fsspec": "fsspec", "pyarrow": "pyarrow"}
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
|
||||
|
||||
@@ -4,8 +4,6 @@ from typing import BinaryIO, Union
|
||||
|
||||
from funcy import cached_property, memoize, wrap_with
|
||||
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .._callback import DEFAULT_CALLBACK, FsspecCallback
|
||||
from ..base import AnyFSPath, FileSystem
|
||||
|
||||
@@ -31,7 +29,7 @@ def make_context(ssl_verify):
|
||||
|
||||
# pylint: disable=abstract-method
|
||||
class HTTPFileSystem(FileSystem):
|
||||
scheme = Schemes.HTTP
|
||||
protocol = "http"
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
REQUIRES = {"aiohttp": "aiohttp", "aiohttp-retry": "aiohttp_retry"}
|
||||
CAN_TRAVERSE = False
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .http import HTTPFileSystem
|
||||
|
||||
|
||||
class HTTPSFileSystem(HTTPFileSystem): # pylint:disable=abstract-method
|
||||
scheme = Schemes.HTTPS
|
||||
protocol = "https"
|
||||
|
||||
@@ -6,8 +6,6 @@ import threading
|
||||
from fsspec import AbstractFileSystem
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .. import system
|
||||
from ..base import FileSystem
|
||||
from ..utils import copyfile, makedirs, move, remove, tmp_fname
|
||||
@@ -168,7 +166,7 @@ class FsspecLocalFileSystem(AbstractFileSystem):
|
||||
class LocalFileSystem(FileSystem):
|
||||
sep = os.sep
|
||||
|
||||
scheme = Schemes.LOCAL
|
||||
protocol = "local"
|
||||
PARAM_CHECKSUM = "md5"
|
||||
PARAM_PATH = "path"
|
||||
TRAVERSE_PREFIX_LEN = 2
|
||||
|
||||
@@ -2,13 +2,11 @@ import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from ..base import FileSystem
|
||||
|
||||
|
||||
class MemoryFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
scheme = Schemes.MEMORY
|
||||
protocol = "memory"
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
def __eq__(self, other):
|
||||
|
||||
@@ -4,8 +4,6 @@ import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from ..base import ObjectFileSystem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -13,7 +11,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
# pylint:disable=abstract-method
|
||||
class OSSFileSystem(ObjectFileSystem):
|
||||
scheme = Schemes.OSS
|
||||
protocol = "oss"
|
||||
REQUIRES = {"ossfs": "ossfs"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
LIST_OBJECT_PAGE_SIZE = 100
|
||||
|
||||
@@ -4,8 +4,6 @@ from collections import defaultdict
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from ..base import ObjectFileSystem
|
||||
|
||||
_AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config")
|
||||
@@ -13,7 +11,7 @@ _AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config")
|
||||
|
||||
# pylint:disable=abstract-method
|
||||
class S3FileSystem(ObjectFileSystem):
|
||||
scheme = Schemes.S3
|
||||
protocol = "s3"
|
||||
REQUIRES = {"s3fs": "s3fs", "boto3": "boto3"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
|
||||
|
||||
@@ -4,8 +4,6 @@ import threading
|
||||
|
||||
from funcy import cached_property, memoize, silent, wrap_prop, wrap_with
|
||||
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from .._callback import DEFAULT_CALLBACK
|
||||
from ..base import FileSystem
|
||||
from ..utils import as_atomic
|
||||
@@ -24,7 +22,7 @@ def ask_password(host, user, port):
|
||||
|
||||
# pylint:disable=abstract-method
|
||||
class SSHFileSystem(FileSystem):
|
||||
scheme = Schemes.SSH
|
||||
protocol = "ssh"
|
||||
REQUIRES = {"sshfs": "sshfs"}
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
|
||||
@@ -3,8 +3,6 @@ import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
from ..base import FileSystem
|
||||
from .http import ask_password
|
||||
|
||||
@@ -12,7 +10,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WebDAVFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
scheme = Schemes.WEBDAV
|
||||
protocol = "webdav"
|
||||
root_marker = ""
|
||||
REQUIRES = {"webdav4": "webdav4"}
|
||||
PARAM_CHECKSUM = "etag"
|
||||
@@ -89,4 +87,4 @@ class WebDAVFileSystem(FileSystem): # pylint:disable=abstract-method
|
||||
|
||||
|
||||
class WebDAVSFileSystem(WebDAVFileSystem): # pylint:disable=abstract-method
|
||||
scheme = Schemes.WEBDAVS
|
||||
protocol = "webdavs"
|
||||
|
||||
@@ -2,14 +2,12 @@ import threading
|
||||
|
||||
from funcy import cached_property, wrap_prop
|
||||
|
||||
from dvc.scheme import Schemes
|
||||
|
||||
# pylint:disable=abstract-method
|
||||
from ..base import FileSystem
|
||||
|
||||
|
||||
class WebHDFSFileSystem(FileSystem):
|
||||
scheme = Schemes.WEBHDFS
|
||||
protocol = "webhdfs"
|
||||
REQUIRES = {"fsspec": "fsspec"}
|
||||
PARAM_CHECKSUM = "checksum"
|
||||
|
||||
|
||||
+1
-1
@@ -115,7 +115,7 @@ def _hash_file(
|
||||
name: str,
|
||||
callback: "FsspecCallback" = DEFAULT_CALLBACK,
|
||||
) -> Tuple["str", Dict[str, Any]]:
|
||||
info = _adapt_info(fs.info(fs_path), fs.scheme)
|
||||
info = _adapt_info(fs.info(fs_path), fs.protocol)
|
||||
|
||||
if name in info:
|
||||
assert not info[name].endswith(".dir")
|
||||
|
||||
+17
-17
@@ -30,10 +30,10 @@ from .fs import (
|
||||
LocalFileSystem,
|
||||
RemoteMissingDepsError,
|
||||
S3FileSystem,
|
||||
Schemes,
|
||||
get_cloud_fs,
|
||||
)
|
||||
from .objects.errors import ObjectFormatError
|
||||
from .scheme import Schemes
|
||||
from .utils import relpath
|
||||
from .utils.fs import path_isin
|
||||
|
||||
@@ -295,7 +295,7 @@ class Output:
|
||||
self.fs = fs_cls(**fs_config)
|
||||
|
||||
if (
|
||||
self.fs.scheme == "local"
|
||||
self.fs.protocol == "local"
|
||||
and stage
|
||||
and isinstance(stage.repo.fs, LocalFileSystem)
|
||||
and path_isin(path, stage.repo.root_dir)
|
||||
@@ -307,7 +307,7 @@ class Output:
|
||||
|
||||
if (
|
||||
self.repo
|
||||
and self.fs.scheme == "local"
|
||||
and self.fs.protocol == "local"
|
||||
and not self.fs.path.isabs(self.def_path)
|
||||
):
|
||||
self.fs = self.repo.fs
|
||||
@@ -339,7 +339,7 @@ class Output:
|
||||
self.remote = remote
|
||||
|
||||
def _parse_path(self, fs, fs_path):
|
||||
if fs.scheme != "local":
|
||||
if fs.protocol != "local":
|
||||
return fs_path
|
||||
|
||||
parsed = urlparse(self.def_path)
|
||||
@@ -362,7 +362,7 @@ class Output:
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
if self.fs.scheme != "local":
|
||||
if self.fs.protocol != "local":
|
||||
return self.def_path
|
||||
|
||||
if (
|
||||
@@ -379,12 +379,12 @@ class Output:
|
||||
return self.fs.path.relpath(self.fs_path, self.repo.root_dir)
|
||||
|
||||
@property
|
||||
def scheme(self):
|
||||
return self.fs.scheme
|
||||
def protocol(self):
|
||||
return self.fs.protocol
|
||||
|
||||
@property
|
||||
def is_in_repo(self):
|
||||
if self.fs.scheme != "local":
|
||||
if self.fs.protocol != "local":
|
||||
return False
|
||||
|
||||
if urlparse(self.def_path).scheme == "remote":
|
||||
@@ -407,9 +407,9 @@ class Output:
|
||||
|
||||
@property
|
||||
def odb(self):
|
||||
odb = getattr(self.repo.odb, self.scheme)
|
||||
odb = getattr(self.repo.odb, self.protocol)
|
||||
if self.use_cache and odb is None:
|
||||
raise RemoteCacheRequiredError(self.fs.scheme, self.fs_path)
|
||||
raise RemoteCacheRequiredError(self.fs.protocol, self.fs_path)
|
||||
return odb
|
||||
|
||||
@property
|
||||
@@ -494,7 +494,7 @@ class Output:
|
||||
|
||||
@property
|
||||
def dvcignore(self):
|
||||
if self.fs.scheme == "local":
|
||||
if self.fs.protocol == "local":
|
||||
return self.repo.dvcignore
|
||||
return None
|
||||
|
||||
@@ -688,9 +688,9 @@ class Output:
|
||||
return ret
|
||||
|
||||
def verify_metric(self):
|
||||
if self.fs.scheme != "local":
|
||||
if self.fs.protocol != "local":
|
||||
raise DvcException(
|
||||
f"verify metric is not supported for {self.scheme}"
|
||||
f"verify metric is not supported for {self.protocol}"
|
||||
)
|
||||
|
||||
if not self.metric or self.plot:
|
||||
@@ -786,7 +786,7 @@ class Output:
|
||||
|
||||
def remove(self, ignore_remove=False):
|
||||
self.fs.remove(self.fs_path)
|
||||
if self.scheme != Schemes.LOCAL:
|
||||
if self.protocol != Schemes.LOCAL:
|
||||
return
|
||||
|
||||
if ignore_remove:
|
||||
@@ -794,7 +794,7 @@ class Output:
|
||||
|
||||
def move(self, out):
|
||||
# pylint: disable=no-member
|
||||
if self.scheme == "local" and self.use_scm_ignore:
|
||||
if self.protocol == "local" and self.use_scm_ignore:
|
||||
self.repo.scm_context.ignore_remove(self.fspath)
|
||||
|
||||
self.fs.move(self.fs_path, out.fs_path)
|
||||
@@ -803,7 +803,7 @@ class Output:
|
||||
self.save()
|
||||
self.commit()
|
||||
|
||||
if self.scheme == "local" and self.use_scm_ignore:
|
||||
if self.protocol == "local" and self.use_scm_ignore:
|
||||
self.repo.scm_context.ignore(self.fspath)
|
||||
|
||||
def transfer(
|
||||
@@ -1000,7 +1000,7 @@ class Output:
|
||||
raise self.IsIgnoredError(check)
|
||||
|
||||
def _check_can_merge(self, out):
|
||||
if self.scheme != out.scheme:
|
||||
if self.protocol != out.protocol:
|
||||
raise MergeError("unable to auto-merge outputs of different types")
|
||||
|
||||
my = self.dumpd()
|
||||
|
||||
@@ -464,7 +464,7 @@ class Repo:
|
||||
|
||||
match = eq if strict else out.fs.path.isin_or_eq
|
||||
|
||||
if out.scheme == "local" and match(fs_path, out.fs_path):
|
||||
if out.protocol == "local" and match(fs_path, out.fs_path):
|
||||
return True
|
||||
|
||||
if recursive and out.fs.path.isin(out.fs_path, fs_path):
|
||||
|
||||
@@ -28,7 +28,7 @@ def _fspath_dir(path):
|
||||
|
||||
|
||||
def _remove_unused_links(repo):
|
||||
used = [out.fspath for out in repo.index.outs if out.scheme == "local"]
|
||||
used = [out.fspath for out in repo.index.outs if out.protocol == "local"]
|
||||
unused = repo.state.get_unused_links(used, repo.fs)
|
||||
ret = [_fspath_dir(u) for u in unused]
|
||||
repo.state.remove_links(unused, repo.fs)
|
||||
|
||||
+2
-2
@@ -2,7 +2,7 @@ import logging
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from dvc.exceptions import DownloadError, FileTransferError
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.fs import Schemes
|
||||
|
||||
from . import locked
|
||||
|
||||
@@ -81,7 +81,7 @@ def fetch(
|
||||
for src_odb, obj_ids in sorted(
|
||||
used.items(),
|
||||
key=lambda item: item[0] is not None
|
||||
and item[0].fs.scheme == Schemes.MEMORY,
|
||||
and item[0].fs.protocol == Schemes.MEMORY,
|
||||
):
|
||||
d, f = _fetch(
|
||||
self,
|
||||
|
||||
+1
-1
@@ -184,7 +184,7 @@ class Index:
|
||||
out.fs_path, self.repo.root_dir
|
||||
)
|
||||
else:
|
||||
fs_key = out.fs.scheme
|
||||
fs_key = out.fs.protocol
|
||||
key = out.fs.path.parts(out.fs_path)
|
||||
|
||||
out.meta.odb = out.odb
|
||||
|
||||
+1
-1
@@ -38,7 +38,7 @@ def move(self, from_path, to_path):
|
||||
from ..dvcfile import DVC_FILE_SUFFIX, Dvcfile
|
||||
|
||||
from_out = Output.loads_from(Stage(self), [from_path])[0]
|
||||
assert from_out.scheme == "local"
|
||||
assert from_out.protocol == "local"
|
||||
|
||||
to_path = _expand_target_path(from_path, to_path)
|
||||
|
||||
|
||||
+1
-1
@@ -508,7 +508,7 @@ class StageLoad:
|
||||
out.fspath
|
||||
for stage in new_stages
|
||||
for out in stage.outs
|
||||
if out.scheme == "local"
|
||||
if out.protocol == "local"
|
||||
)
|
||||
dirs[:] = [d for d in dirs if not is_out_or_ignored(root, d)]
|
||||
|
||||
|
||||
+2
-2
@@ -38,11 +38,11 @@ def _can_hash(stage):
|
||||
return False
|
||||
|
||||
for dep in stage.deps:
|
||||
if not (dep.scheme == "local" and dep.def_path and dep.get_hash()):
|
||||
if not (dep.protocol == "local" and dep.def_path and dep.get_hash()):
|
||||
return False
|
||||
|
||||
for out in stage.outs:
|
||||
if out.scheme != "local" or not out.def_path or out.persist:
|
||||
if out.protocol != "local" or not out.def_path or out.persist:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@@ -49,7 +49,7 @@ def test_remote_without_hash_jobs_default(dvc):
|
||||
|
||||
@pytest.mark.parametrize("fs_cls", [GSFileSystem, S3FileSystem])
|
||||
def test_makedirs_not_create_for_top_level_path(fs_cls, dvc, mocker):
|
||||
url = f"{fs_cls.scheme}://bucket/"
|
||||
url = f"{fs_cls.protocol}://bucket/"
|
||||
fs = fs_cls(url=url)
|
||||
mocked_client = mocker.PropertyMock()
|
||||
mocker.patch.object(fs_cls, "fs", mocked_client)
|
||||
|
||||
Reference in New Issue
Block a user