diff --git a/dvc/config.py b/dvc/config.py index 61264539e..8468d0b88 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -86,7 +86,7 @@ class Config(dict): def __init__( self, dvc_dir=None, validate=True, fs=None, config=None ): # pylint: disable=super-init-not-called - from dvc.fs.local import LocalFileSystem + from dvc.fs import LocalFileSystem self.dvc_dir = dvc_dir self.wfs = LocalFileSystem() @@ -227,7 +227,7 @@ class Config(dict): @staticmethod def _to_relpath(conf_dir, path): - from dvc.fs.local import localfs + from dvc.fs import localfs from dvc.utils import relpath from .config_schema import RelPath diff --git a/dvc/data/checkout.py b/dvc/data/checkout.py index cf0c9e676..81d0760a0 100644 --- a/dvc/data/checkout.py +++ b/dvc/data/checkout.py @@ -3,9 +3,9 @@ from itertools import chain from dvc import prompt from dvc.exceptions import CacheLinkError, CheckoutError, ConfirmRemoveError -from dvc.fs._callback import FsspecCallback -from dvc.fs.generic import test_links, transfer from dvc.ignore import DvcIgnoreFilter +from dvc.objects.fs._callback import FsspecCallback +from dvc.objects.fs.generic import test_links, transfer from dvc.types import Optional from .diff import ROOT diff --git a/dvc/data/db/__init__.py b/dvc/data/db/__init__.py index 1210cdf10..d39d62bea 100644 --- a/dvc/data/db/__init__.py +++ b/dvc/data/db/__init__.py @@ -18,7 +18,7 @@ def get_odb(fs, fs_path, **config): def _get_odb(repo, settings): - from dvc.fs import get_cloud_fs + from dvc.objects.fs import get_cloud_fs if not settings: return None diff --git a/dvc/data/db/index.py b/dvc/data/db/index.py index 77c9d7c12..b2faf114f 100644 --- a/dvc/data/db/index.py +++ b/dvc/data/db/index.py @@ -88,9 +88,9 @@ class ObjectDBIndex(ObjectDBIndexBase): tmp_dir: "StrPath", name: str, ): # pylint: disable=super-init-not-called - from dvc.fs.local import LocalFileSystem - from dvc.fs.utils import makedirs from dvc.objects.cache import Cache, Index + from dvc.objects.fs.local import LocalFileSystem + from dvc.objects.fs.utils import makedirs self.index_dir = os.path.join(tmp_dir, self.INDEX_DIR, name) makedirs(self.index_dir, exist_ok=True) diff --git a/dvc/data/db/local.py b/dvc/data/db/local.py index fbc3623a3..d62e48bca 100644 --- a/dvc/data/db/local.py +++ b/dvc/data/db/local.py @@ -6,10 +6,10 @@ from functools import partial from funcy import cached_property from shortuuid import uuid -from dvc.fs.system import umask -from dvc.fs.utils import copyfile, relpath, remove, walk_files from dvc.objects.db import ObjectDB, noop, wrap_iter from dvc.objects.errors import ObjectFormatError +from dvc.objects.fs.system import umask +from dvc.objects.fs.utils import copyfile, relpath, remove, walk_files from dvc.objects.hash_info import HashInfo logger = logging.getLogger(__name__) @@ -49,7 +49,7 @@ class LocalObjectDB(ObjectDB): os.chmod(to_info, self._file_mode) def makedirs(self, fs_path): - from dvc.fs.utils import makedirs + from dvc.objects.fs.utils import makedirs makedirs(fs_path, exist_ok=True, mode=self._dir_mode) diff --git a/dvc/data/db/reference.py b/dvc/data/db/reference.py index 0477ac232..27b6f7f2d 100644 --- a/dvc/data/db/reference.py +++ b/dvc/data/db/reference.py @@ -9,8 +9,8 @@ from dvc.scheme import Schemes from ..reference import ReferenceHashFile if TYPE_CHECKING: - from dvc.fs._callback import FsspecCallback - from dvc.fs.base import AnyFSPath, FileSystem + from dvc.objects.fs._callback import FsspecCallback + from dvc.objects.fs.base import AnyFSPath, FileSystem from dvc.objects.hash_info import HashInfo logger = logging.getLogger(__name__) diff --git a/dvc/data/reference.py b/dvc/data/reference.py index 7cdbdc018..70d8ba6da 100644 --- a/dvc/data/reference.py +++ b/dvc/data/reference.py @@ -8,8 +8,8 @@ from dvc.objects.errors import ObjectFormatError from dvc.objects.file import HashFile if TYPE_CHECKING: - from dvc.fs.base import AnyFSPath, FileSystem from dvc.objects.db import ObjectDB + from dvc.objects.fs.base import AnyFSPath, FileSystem from dvc.objects.hash_info import HashInfo logger = logging.getLogger(__name__) @@ -70,8 +70,8 @@ class ReferenceHashFile(HashFile): @classmethod def from_bytes(cls, data: bytes, fs_cache: Optional[dict] = None): - from dvc.fs import get_fs_cls from dvc.fs.dvc import DvcFileSystem, _DvcFileSystem + from dvc.objects.fs import get_fs_cls try: dict_ = pickle.loads(data) diff --git a/dvc/data/stage.py b/dvc/data/stage.py index 9d15440b6..67b48fa87 100644 --- a/dvc/data/stage.py +++ b/dvc/data/stage.py @@ -17,8 +17,8 @@ from dvc.progress import Tqdm from .db.reference import ReferenceObjectDB if TYPE_CHECKING: - from dvc.fs.base import AnyFSPath, FileSystem from dvc.objects.db import ObjectDB + from dvc.objects.fs.base import AnyFSPath, FileSystem from .tree import Tree @@ -30,8 +30,8 @@ _STAGING_MEMFS_PATH = "dvc-staging" def _upload_file(from_fs_path, fs, odb, upload_odb, callback=None): - from dvc.fs._callback import FsspecCallback - from dvc.fs.utils import tmp_fname + from dvc.objects.fs._callback import FsspecCallback + from dvc.objects.fs.utils import tmp_fname from .stream import HashedStreamReader @@ -194,7 +194,7 @@ def _get_staging(odb: "ObjectDB") -> "ReferenceObjectDB": Staging will be a reference ODB stored in the the global memfs. """ - from dvc.fs.memory import MemoryFileSystem + from dvc.objects.fs.memory import MemoryFileSystem fs = MemoryFileSystem() fs_path = _make_staging_url(fs, odb, odb.fs_path) diff --git a/dvc/data/tree.py b/dvc/data/tree.py index af0f4cdb5..f7c90cc56 100644 --- a/dvc/data/tree.py +++ b/dvc/data/tree.py @@ -55,8 +55,8 @@ class Tree(HashFile): self._dict[key] = (meta, oid) def digest(self, hash_info: Optional["HashInfo"] = None): - from dvc.fs.memory import MemoryFileSystem - from dvc.fs.utils import tmp_fname + from dvc.objects.fs.memory import MemoryFileSystem + from dvc.objects.fs.utils import tmp_fname memfs = MemoryFileSystem() fs_path = "memory://{}".format(tmp_fname("")) diff --git a/dvc/dvcfile.py b/dvc/dvcfile.py index e542ba986..cb0b9bf48 100644 --- a/dvc/dvcfile.py +++ b/dvc/dvcfile.py @@ -59,7 +59,7 @@ def is_lock_file(path): def is_git_ignored(repo, path): - from dvc.fs.local import LocalFileSystem + from dvc.fs import LocalFileSystem from dvc.scm import NoSCMError try: diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 16a59a300..54cf43ce6 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -33,7 +33,7 @@ def external_repo( from scmrepo.git import Git from dvc.config import NoRemoteError - from dvc.fs.git import GitFileSystem + from dvc.fs import GitFileSystem logger.debug("Creating external repo %s@%s", url, rev) path = _cached_clone(url, rev, for_write=for_write) @@ -95,7 +95,7 @@ def external_repo( def erepo_factory(url, root_dir, cache_config): - from dvc.fs.local import localfs + from dvc.fs import localfs def make_repo(path, fs=None, **_kwargs): _config = cache_config.copy() diff --git a/dvc/fs/__init__.py b/dvc/fs/__init__.py index a8b56bac1..df0426aed 100644 --- a/dvc/fs/__init__.py +++ b/dvc/fs/__init__.py @@ -1,126 +1,38 @@ -from urllib.parse import urlparse +# pylint: disable=unused-import +from dvc.objects.fs import utils # noqa: F401 +from dvc.objects.fs import ( # noqa: F401 + FS_MAP, + AzureFileSystem, + GDriveFileSystem, + GSFileSystem, + HDFSFileSystem, + HTTPFileSystem, + HTTPSFileSystem, + LocalFileSystem, + OSSFileSystem, + S3FileSystem, + SSHFileSystem, + WebDAVFileSystem, + WebDAVSFileSystem, + WebHDFSFileSystem, + _resolve_remote_refs, + generic, + get_cloud_fs, + get_fs_cls, + get_fs_config, + system, +) +from dvc.objects.fs.azure import AzureAuthError # noqa: F401 +from dvc.objects.fs.base import ( # noqa: F401 + AnyFSPath, + FileSystem, + RemoteMissingDepsError, +) +from dvc.objects.fs.gdrive import GDriveAuthError # noqa: F401 +from dvc.objects.fs.git import GitFileSystem # noqa: F401 +from dvc.objects.fs.local import localfs # noqa: F401 +from dvc.objects.fs.memory import MemoryFileSystem # noqa: F401 +from dvc.objects.fs.path import Path # noqa: F401 +from dvc.objects.fs.ssh import DEFAULT_PORT as DEFAULT_SSH_PORT # noqa: F401 -from ..scheme import Schemes -from . import generic # noqa: F401 -from .azure import AzureFileSystem -from .gdrive import GDriveFileSystem -from .gs import GSFileSystem -from .hdfs import HDFSFileSystem -from .http import HTTPFileSystem -from .https import HTTPSFileSystem -from .local import LocalFileSystem -from .oss import OSSFileSystem -from .s3 import S3FileSystem -from .ssh import SSHFileSystem -from .webdav import WebDAVFileSystem, WebDAVSFileSystem -from .webhdfs import WebHDFSFileSystem - -FS_MAP = { - Schemes.AZURE: AzureFileSystem, - Schemes.GDRIVE: GDriveFileSystem, - Schemes.GS: GSFileSystem, - Schemes.HDFS: HDFSFileSystem, - Schemes.WEBHDFS: WebHDFSFileSystem, - Schemes.HTTP: HTTPFileSystem, - Schemes.HTTPS: HTTPSFileSystem, - Schemes.S3: S3FileSystem, - Schemes.SSH: SSHFileSystem, - Schemes.OSS: OSSFileSystem, - Schemes.WEBDAV: WebDAVFileSystem, - Schemes.WEBDAVS: WebDAVSFileSystem, - # NOTE: LocalFileSystem is the default -} - - -def get_fs_cls(remote_conf, scheme=None): - if not scheme: - scheme = urlparse(remote_conf["url"]).scheme - return FS_MAP.get(scheme, LocalFileSystem) - - -def get_fs_config(repo, config, **kwargs): - name = kwargs.get("name") - if name: - try: - remote_conf = config["remote"][name.lower()] - except KeyError: - from dvc.config import RemoteNotFoundError - - raise RemoteNotFoundError(f"remote '{name}' doesn't exist") - else: - remote_conf = kwargs - return _resolve_remote_refs(repo, config, remote_conf) - - -def _resolve_remote_refs(repo, config, remote_conf): - # Support for cross referenced remotes. - # This will merge the settings, shadowing base ref with remote_conf. - # For example, having: - # - # dvc remote add server ssh://localhost - # dvc remote modify server user root - # dvc remote modify server ask_password true - # - # dvc remote add images remote://server/tmp/pictures - # dvc remote modify images user alice - # dvc remote modify images ask_password false - # dvc remote modify images password asdf1234 - # - # Results on a config dictionary like: - # - # { - # "url": "ssh://localhost/tmp/pictures", - # "user": "alice", - # "password": "asdf1234", - # "ask_password": False, - # } - parsed = urlparse(remote_conf["url"]) - if parsed.scheme != "remote": - return remote_conf - - base = get_fs_config(repo, config, name=parsed.netloc) - cls, _, _ = get_cloud_fs(repo, **base) - relpath = parsed.path.lstrip("/").replace("/", cls.sep) - url = cls.sep.join((base["url"], relpath)) - return {**base, **remote_conf, "url": url} - - -def get_cloud_fs(repo, **kwargs): - from dvc.config import ConfigError - from dvc.config_schema import SCHEMA, Invalid - - repo_config = repo.config if repo else {} - core_config = repo_config.get("core", {}) - - remote_conf = get_fs_config(repo, repo_config, **kwargs) - try: - remote_conf = SCHEMA["remote"][str](remote_conf) - except Invalid as exc: - raise ConfigError(str(exc)) from None - - if "jobs" not in remote_conf: - jobs = core_config.get("jobs") - if jobs: - remote_conf["jobs"] = jobs - - if "checksum_jobs" not in remote_conf: - checksum_jobs = core_config.get("checksum_jobs") - if checksum_jobs: - remote_conf["checksum_jobs"] = checksum_jobs - - cls = get_fs_cls(remote_conf) - - if cls == GDriveFileSystem and repo: - remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir - - url = remote_conf.pop("url") - if issubclass(cls, WebDAVFileSystem): - # For WebDAVFileSystem, provided url is the base path itself, so it - # should be treated as being a root path. - fs_path = cls.root_marker - else: - fs_path = cls._strip_protocol(url) # pylint:disable=protected-access - - extras = cls._get_kwargs_from_urls(url) # pylint:disable=protected-access - conf = {**extras, **remote_conf} # remote config takes priority - return cls, conf, fs_path +# pylint: enable=unused-import diff --git a/dvc/fs/_callback.py b/dvc/fs/_callback.py index 2e19aff66..a1189ca47 100644 --- a/dvc/fs/_callback.py +++ b/dvc/fs/_callback.py @@ -1,235 +1,7 @@ -from contextlib import ExitStack -from functools import wraps -from typing import TYPE_CHECKING, Any, Dict, Optional, TypeVar, overload - -import fsspec -from funcy import cached_property - -if TYPE_CHECKING: - from typing import BinaryIO, Callable, TextIO, Union - - from typing_extensions import ParamSpec - - from dvc.progress import Tqdm - from dvc.ui._rich_progress import RichTransferProgress - - _P = ParamSpec("_P") - _R = TypeVar("_R") - - -class FsspecCallback(fsspec.Callback): - """FsspecCallback usable as a context manager, and a few helper methods.""" - - @overload - def wrap_attr(self, fobj: "BinaryIO", method: str = "read") -> "BinaryIO": - ... - - @overload - def wrap_attr(self, fobj: "TextIO", method: str = "read") -> "TextIO": - ... - - def wrap_attr( - self, fobj: "Union[TextIO, BinaryIO]", method: str = "read" - ) -> "Union[TextIO, BinaryIO]": - from tqdm.utils import CallbackIOWrapper - - wrapped = CallbackIOWrapper(self.relative_update, fobj, method) - return wrapped - - def wrap_fn(self, fn: "Callable[_P, _R]") -> "Callable[_P, _R]": - @wraps(fn) - def wrapped(*args: "_P.args", **kwargs: "_P.kwargs") -> "_R": - res = fn(*args, **kwargs) - self.relative_update() - return res - - return wrapped - - def wrap_and_branch(self, fn: "Callable") -> "Callable": - """ - Wraps a function, and pass a new child callback to it. - When the function completes, we increment the parent callback by 1. - """ - wrapped = self.wrap_fn(fn) - - @wraps(fn) - def func(path1: str, path2: str): - kw: Dict[str, Any] = {} - with self.branch(path1, path2, kw): - return wrapped(path1, path2, **kw) - - return func - - def __enter__(self): - return self - - def __exit__(self, *exc_args): - self.close() - - def close(self): - """Handle here on exit.""" - - def relative_update(self, inc: int = 1) -> None: - inc = inc if inc is not None else 0 - return super().relative_update(inc) - - def absolute_update(self, value: int) -> None: - value = value if value is not None else self.value - return super().absolute_update(value) - - @classmethod - def as_callback( - cls, maybe_callback: Optional["FsspecCallback"] = None - ) -> "FsspecCallback": - if maybe_callback is None: - return DEFAULT_CALLBACK - return maybe_callback - - @classmethod - def as_tqdm_callback( - cls, - callback: Optional["FsspecCallback"] = None, - **tqdm_kwargs: Any, - ) -> "FsspecCallback": - return callback or TqdmCallback(**tqdm_kwargs) - - @classmethod - def as_rich_callback( - cls, callback: Optional["FsspecCallback"] = None, **rich_kwargs - ): - return callback or RichCallback(**rich_kwargs) - - def branch( - self, - path_1: str, - path_2: str, - kwargs: Dict[str, Any], - child: "FsspecCallback" = None, - ) -> "FsspecCallback": - child = kwargs["callback"] = child or DEFAULT_CALLBACK - return child - - -class NoOpCallback(FsspecCallback, fsspec.callbacks.NoOpCallback): - pass - - -class TqdmCallback(FsspecCallback): - def __init__( - self, - size: Optional[int] = None, - value: int = 0, - progress_bar: "Tqdm" = None, - **tqdm_kwargs, - ): - tqdm_kwargs["total"] = size or -1 - self._tqdm_kwargs = tqdm_kwargs - self._progress_bar = progress_bar - self._stack = ExitStack() - super().__init__(size=size, value=value) - - @cached_property - def progress_bar(self): - from dvc.progress import Tqdm - - progress_bar = ( - self._progress_bar - if self._progress_bar is not None - else Tqdm(**self._tqdm_kwargs) - ) - return self._stack.enter_context(progress_bar) - - def __enter__(self): - return self - - def close(self): - self._stack.close() - - def set_size(self, size): - # Tqdm tries to be smart when to refresh, - # so we try to force it to re-render. - super().set_size(size) - self.progress_bar.refresh() - - def call(self, hook_name=None, **kwargs): - self.progress_bar.update_to(self.value, total=self.size) - - def branch( - self, - path_1: str, - path_2: str, - kwargs, - child: Optional[FsspecCallback] = None, - ): - child = child or TqdmCallback(bytes=True, desc=path_1) - return super().branch(path_1, path_2, kwargs, child=child) - - -class RichCallback(FsspecCallback): - def __init__( - self, - size: Optional[int] = None, - value: int = 0, - progress: "RichTransferProgress" = None, - desc: str = None, - bytes: bool = False, # pylint: disable=redefined-builtin - unit: str = None, - disable: bool = False, - ) -> None: - self._progress = progress - self.disable = disable - self._task_kwargs = { - "description": desc or "", - "bytes": bytes, - "unit": unit, - "total": size or 0, - "visible": False, - "progress_type": None if bytes else "summary", - } - self._stack = ExitStack() - super().__init__(size=size, value=value) - - @cached_property - def progress(self): - from dvc.ui import ui - from dvc.ui._rich_progress import RichTransferProgress - - if self._progress is not None: - return self._progress - - progress = RichTransferProgress( - transient=True, - disable=self.disable, - console=ui.error_console, - ) - return self._stack.enter_context(progress) - - @cached_property - def task(self): - return self.progress.add_task(**self._task_kwargs) - - def __enter__(self): - return self - - def close(self): - self.progress.clear_task(self.task) - self._stack.close() - - def call(self, hook_name=None, **kwargs): - self.progress.update( - self.task, - completed=self.value, - total=self.size, - visible=not self.disable, - ) - - def branch( - self, path_1, path_2, kwargs, child: Optional[FsspecCallback] = None - ): - child = child or RichCallback( - progress=self.progress, desc=path_1, bytes=True - ) - return super().branch(path_1, path_2, kwargs, child=child) - - -DEFAULT_CALLBACK = NoOpCallback() +# pylint: disable=unused-import +from dvc.objects.fs._callback import ( # noqa: F401 + DEFAULT_CALLBACK, + FsspecCallback, + RichCallback, + TqdmCallback, +) diff --git a/dvc/fs/data.py b/dvc/fs/data.py index 115b5e399..6c6aed3ce 100644 --- a/dvc/fs/data.py +++ b/dvc/fs/data.py @@ -5,8 +5,8 @@ import typing from fsspec import AbstractFileSystem from funcy import cached_property, wrap_prop -from ._callback import DEFAULT_CALLBACK -from .base import FileSystem +from dvc.objects.fs._callback import DEFAULT_CALLBACK +from dvc.objects.fs.base import FileSystem if typing.TYPE_CHECKING: from dvc.types import AnyPath @@ -30,7 +30,7 @@ class _DataFileSystem(AbstractFileSystem): # pylint:disable=abstract-method @cached_property def path(self): - from .path import Path + from . import Path def _getcwd(): return self.root_marker diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index da4d5520b..31beb1be9 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -1,5 +1,7 @@ import logging +import ntpath import os +import posixpath import threading from contextlib import suppress from itertools import takewhile @@ -8,10 +10,11 @@ from typing import TYPE_CHECKING, Callable, Optional, Tuple, Type, Union from fsspec.spec import AbstractFileSystem from funcy import cached_property, wrap_prop, wrap_with -from ._callback import DEFAULT_CALLBACK -from .base import FileSystem +from dvc.objects.fs._callback import DEFAULT_CALLBACK +from dvc.objects.fs.base import FileSystem +from dvc.objects.fs.path import Path + from .data import DataFileSystem -from .path import Path if TYPE_CHECKING: from dvc.repo import Repo @@ -21,6 +24,10 @@ logger = logging.getLogger(__name__) RepoFactory = Union[Callable[[str], "Repo"], Type["Repo"]] +def as_posix(path: str) -> str: + return path.replace(ntpath.sep, posixpath.sep) + + def _wrap_walk(dvc_fs, *args, **kwargs): for root, dnames, fnames in dvc_fs.walk(*args, **kwargs): yield dvc_fs.path.join(dvc_fs.repo.root_dir, root), dnames, fnames @@ -41,7 +48,7 @@ def _ls(fs, path): def _merge_info(repo, fs_info, dvc_info): - from dvc.fs.utils import is_exec + from . import utils ret = {"repo": repo} @@ -57,7 +64,7 @@ def _merge_info(repo, fs_info, dvc_info): ret["size"] = fs_info["size"] isexec = False if fs_info["type"] == "file": - isexec = is_exec(fs_info["mode"]) + isexec = utils.is_exec(fs_info["mode"]) ret["isexec"] = isexec return ret @@ -468,8 +475,6 @@ class DvcFileSystem(FileSystem): return self.fs.config def from_os_path(self, path): - from .utils import as_posix - if os.path.isabs(path): path = os.path.relpath(path, self.repo.root_dir) diff --git a/dvc/ignore.py b/dvc/ignore.py index e035f5eb7..6f54cf596 100644 --- a/dvc/ignore.py +++ b/dvc/ignore.py @@ -8,8 +8,7 @@ from pathspec.patterns import GitWildMatchPattern from pathspec.util import normalize_file from pygtrie import Trie -from dvc.fs.base import AnyFSPath, FileSystem -from dvc.fs.local import localfs +from dvc.fs import AnyFSPath, FileSystem, localfs from dvc.pathspec_math import PatternInfo, merge_patterns from dvc.scheme import Schemes from dvc.types import List, Optional diff --git a/dvc/info.py b/dvc/info.py index 37e9a99f8..12d2f86a3 100644 --- a/dvc/info.py +++ b/dvc/info.py @@ -7,8 +7,7 @@ import psutil from dvc import __version__ from dvc.exceptions import NotDvcRepoError -from dvc.fs import FS_MAP, get_fs_cls, get_fs_config -from dvc.fs.generic import test_links +from dvc.fs import FS_MAP, generic, get_fs_cls, get_fs_config from dvc.repo import Repo from dvc.scm import SCMError from dvc.utils import error_link @@ -83,7 +82,7 @@ def _get_remotes(config): def _get_linktype_support_info(repo): odb = repo.odb.local - links = test_links( + links = generic.test_links( ["reflink", "hardlink", "symlink"], odb.fs, odb.fs_path, diff --git a/dvc/machine/backend/base.py b/dvc/machine/backend/base.py index f780acff9..67c1a1f6d 100644 --- a/dvc/machine/backend/base.py +++ b/dvc/machine/backend/base.py @@ -3,7 +3,7 @@ from contextlib import contextmanager from typing import TYPE_CHECKING, Iterator, Optional if TYPE_CHECKING: - from dvc.fs.ssh import SSHFileSystem + from dvc.fs import SSHFileSystem from dvc.types import StrPath diff --git a/dvc/machine/backend/terraform.py b/dvc/machine/backend/terraform.py index ef5c3b861..0ad190505 100644 --- a/dvc/machine/backend/terraform.py +++ b/dvc/machine/backend/terraform.py @@ -4,8 +4,7 @@ from functools import partial, partialmethod from typing import TYPE_CHECKING, Iterator, Optional from dvc.exceptions import DvcException -from dvc.fs.ssh import DEFAULT_PORT as DEFAULT_SSH_PORT -from dvc.fs.ssh import SSHFileSystem +from dvc.fs import DEFAULT_SSH_PORT, SSHFileSystem from dvc.utils.fs import makedirs from .base import BaseMachineBackend diff --git a/dvc/objects/db.py b/dvc/objects/db.py index 83fb9dcec..0dec70f2b 100644 --- a/dvc/objects/db.py +++ b/dvc/objects/db.py @@ -12,9 +12,8 @@ from .file import HashFile if TYPE_CHECKING: from typing import Tuple - from dvc.fs._callback import FsspecCallback - from dvc.fs.base import AnyFSPath, FileSystem - + from .fs._callback import FsspecCallback + from .fs.base import AnyFSPath, FileSystem from .hash_info import HashInfo logger = logging.getLogger(__name__) @@ -95,11 +94,11 @@ class ObjectDB: hardlink: bool = False, callback: "FsspecCallback" = None, ): - from dvc import fs - from dvc.fs._callback import FsspecCallback + from .fs import generic + from .fs._callback import FsspecCallback self.makedirs(self.fs.path.parent(to_info)) - return fs.generic.transfer( + return generic.transfer( from_fs, from_info, self.fs, @@ -117,7 +116,7 @@ class ObjectDB: verify: Optional[bool] = None, callback: "FsspecCallback" = None, ): - from dvc.fs._callback import FsspecCallback + from .fs._callback import FsspecCallback if self.read_only: raise ObjectDBPermissionError("Cannot add to read-only ODB") diff --git a/dvc/objects/file.py b/dvc/objects/file.py index a1575f98b..579ae2d19 100644 --- a/dvc/objects/file.py +++ b/dvc/objects/file.py @@ -4,9 +4,8 @@ import os from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: - from dvc.fs.base import AnyFSPath, FileSystem - from .db import ObjectDB + from .fs.base import AnyFSPath, FileSystem from .hash_info import HashInfo logger = logging.getLogger(__name__) diff --git a/dvc/objects/fs/__init__.py b/dvc/objects/fs/__init__.py new file mode 100644 index 000000000..e183953d8 --- /dev/null +++ b/dvc/objects/fs/__init__.py @@ -0,0 +1,127 @@ +from urllib.parse import urlparse + +from dvc.scheme import Schemes + +from . import generic # noqa: F401 +from .azure import AzureFileSystem +from .gdrive import GDriveFileSystem +from .gs import GSFileSystem +from .hdfs import HDFSFileSystem +from .http import HTTPFileSystem +from .https import HTTPSFileSystem +from .local import LocalFileSystem +from .oss import OSSFileSystem +from .s3 import S3FileSystem +from .ssh import SSHFileSystem +from .webdav import WebDAVFileSystem, WebDAVSFileSystem +from .webhdfs import WebHDFSFileSystem + +FS_MAP = { + Schemes.AZURE: AzureFileSystem, + Schemes.GDRIVE: GDriveFileSystem, + Schemes.GS: GSFileSystem, + Schemes.HDFS: HDFSFileSystem, + Schemes.WEBHDFS: WebHDFSFileSystem, + Schemes.HTTP: HTTPFileSystem, + Schemes.HTTPS: HTTPSFileSystem, + Schemes.S3: S3FileSystem, + Schemes.SSH: SSHFileSystem, + Schemes.OSS: OSSFileSystem, + Schemes.WEBDAV: WebDAVFileSystem, + Schemes.WEBDAVS: WebDAVSFileSystem, + # NOTE: LocalFileSystem is the default +} + + +def get_fs_cls(remote_conf, scheme=None): + if not scheme: + scheme = urlparse(remote_conf["url"]).scheme + return FS_MAP.get(scheme, LocalFileSystem) + + +def get_fs_config(repo, config, **kwargs): + name = kwargs.get("name") + if name: + try: + remote_conf = config["remote"][name.lower()] + except KeyError: + from dvc.config import RemoteNotFoundError + + raise RemoteNotFoundError(f"remote '{name}' doesn't exist") + else: + remote_conf = kwargs + return _resolve_remote_refs(repo, config, remote_conf) + + +def _resolve_remote_refs(repo, config, remote_conf): + # Support for cross referenced remotes. + # This will merge the settings, shadowing base ref with remote_conf. + # For example, having: + # + # dvc remote add server ssh://localhost + # dvc remote modify server user root + # dvc remote modify server ask_password true + # + # dvc remote add images remote://server/tmp/pictures + # dvc remote modify images user alice + # dvc remote modify images ask_password false + # dvc remote modify images password asdf1234 + # + # Results on a config dictionary like: + # + # { + # "url": "ssh://localhost/tmp/pictures", + # "user": "alice", + # "password": "asdf1234", + # "ask_password": False, + # } + parsed = urlparse(remote_conf["url"]) + if parsed.scheme != "remote": + return remote_conf + + base = get_fs_config(repo, config, name=parsed.netloc) + cls, _, _ = get_cloud_fs(repo, **base) + relpath = parsed.path.lstrip("/").replace("/", cls.sep) + url = cls.sep.join((base["url"], relpath)) + return {**base, **remote_conf, "url": url} + + +def get_cloud_fs(repo, **kwargs): + from dvc.config import ConfigError + from dvc.config_schema import SCHEMA, Invalid + + repo_config = repo.config if repo else {} + core_config = repo_config.get("core", {}) + + remote_conf = get_fs_config(repo, repo_config, **kwargs) + try: + remote_conf = SCHEMA["remote"][str](remote_conf) + except Invalid as exc: + raise ConfigError(str(exc)) from None + + if "jobs" not in remote_conf: + jobs = core_config.get("jobs") + if jobs: + remote_conf["jobs"] = jobs + + if "checksum_jobs" not in remote_conf: + checksum_jobs = core_config.get("checksum_jobs") + if checksum_jobs: + remote_conf["checksum_jobs"] = checksum_jobs + + cls = get_fs_cls(remote_conf) + + if cls == GDriveFileSystem and repo: + remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir + + url = remote_conf.pop("url") + if issubclass(cls, WebDAVFileSystem): + # For WebDAVFileSystem, provided url is the base path itself, so it + # should be treated as being a root path. + fs_path = cls.root_marker + else: + fs_path = cls._strip_protocol(url) # pylint:disable=protected-access + + extras = cls._get_kwargs_from_urls(url) # pylint:disable=protected-access + conf = {**extras, **remote_conf} # remote config takes priority + return cls, conf, fs_path diff --git a/dvc/objects/fs/_callback.py b/dvc/objects/fs/_callback.py new file mode 100644 index 000000000..2e19aff66 --- /dev/null +++ b/dvc/objects/fs/_callback.py @@ -0,0 +1,235 @@ +from contextlib import ExitStack +from functools import wraps +from typing import TYPE_CHECKING, Any, Dict, Optional, TypeVar, overload + +import fsspec +from funcy import cached_property + +if TYPE_CHECKING: + from typing import BinaryIO, Callable, TextIO, Union + + from typing_extensions import ParamSpec + + from dvc.progress import Tqdm + from dvc.ui._rich_progress import RichTransferProgress + + _P = ParamSpec("_P") + _R = TypeVar("_R") + + +class FsspecCallback(fsspec.Callback): + """FsspecCallback usable as a context manager, and a few helper methods.""" + + @overload + def wrap_attr(self, fobj: "BinaryIO", method: str = "read") -> "BinaryIO": + ... + + @overload + def wrap_attr(self, fobj: "TextIO", method: str = "read") -> "TextIO": + ... + + def wrap_attr( + self, fobj: "Union[TextIO, BinaryIO]", method: str = "read" + ) -> "Union[TextIO, BinaryIO]": + from tqdm.utils import CallbackIOWrapper + + wrapped = CallbackIOWrapper(self.relative_update, fobj, method) + return wrapped + + def wrap_fn(self, fn: "Callable[_P, _R]") -> "Callable[_P, _R]": + @wraps(fn) + def wrapped(*args: "_P.args", **kwargs: "_P.kwargs") -> "_R": + res = fn(*args, **kwargs) + self.relative_update() + return res + + return wrapped + + def wrap_and_branch(self, fn: "Callable") -> "Callable": + """ + Wraps a function, and pass a new child callback to it. + When the function completes, we increment the parent callback by 1. + """ + wrapped = self.wrap_fn(fn) + + @wraps(fn) + def func(path1: str, path2: str): + kw: Dict[str, Any] = {} + with self.branch(path1, path2, kw): + return wrapped(path1, path2, **kw) + + return func + + def __enter__(self): + return self + + def __exit__(self, *exc_args): + self.close() + + def close(self): + """Handle here on exit.""" + + def relative_update(self, inc: int = 1) -> None: + inc = inc if inc is not None else 0 + return super().relative_update(inc) + + def absolute_update(self, value: int) -> None: + value = value if value is not None else self.value + return super().absolute_update(value) + + @classmethod + def as_callback( + cls, maybe_callback: Optional["FsspecCallback"] = None + ) -> "FsspecCallback": + if maybe_callback is None: + return DEFAULT_CALLBACK + return maybe_callback + + @classmethod + def as_tqdm_callback( + cls, + callback: Optional["FsspecCallback"] = None, + **tqdm_kwargs: Any, + ) -> "FsspecCallback": + return callback or TqdmCallback(**tqdm_kwargs) + + @classmethod + def as_rich_callback( + cls, callback: Optional["FsspecCallback"] = None, **rich_kwargs + ): + return callback or RichCallback(**rich_kwargs) + + def branch( + self, + path_1: str, + path_2: str, + kwargs: Dict[str, Any], + child: "FsspecCallback" = None, + ) -> "FsspecCallback": + child = kwargs["callback"] = child or DEFAULT_CALLBACK + return child + + +class NoOpCallback(FsspecCallback, fsspec.callbacks.NoOpCallback): + pass + + +class TqdmCallback(FsspecCallback): + def __init__( + self, + size: Optional[int] = None, + value: int = 0, + progress_bar: "Tqdm" = None, + **tqdm_kwargs, + ): + tqdm_kwargs["total"] = size or -1 + self._tqdm_kwargs = tqdm_kwargs + self._progress_bar = progress_bar + self._stack = ExitStack() + super().__init__(size=size, value=value) + + @cached_property + def progress_bar(self): + from dvc.progress import Tqdm + + progress_bar = ( + self._progress_bar + if self._progress_bar is not None + else Tqdm(**self._tqdm_kwargs) + ) + return self._stack.enter_context(progress_bar) + + def __enter__(self): + return self + + def close(self): + self._stack.close() + + def set_size(self, size): + # Tqdm tries to be smart when to refresh, + # so we try to force it to re-render. + super().set_size(size) + self.progress_bar.refresh() + + def call(self, hook_name=None, **kwargs): + self.progress_bar.update_to(self.value, total=self.size) + + def branch( + self, + path_1: str, + path_2: str, + kwargs, + child: Optional[FsspecCallback] = None, + ): + child = child or TqdmCallback(bytes=True, desc=path_1) + return super().branch(path_1, path_2, kwargs, child=child) + + +class RichCallback(FsspecCallback): + def __init__( + self, + size: Optional[int] = None, + value: int = 0, + progress: "RichTransferProgress" = None, + desc: str = None, + bytes: bool = False, # pylint: disable=redefined-builtin + unit: str = None, + disable: bool = False, + ) -> None: + self._progress = progress + self.disable = disable + self._task_kwargs = { + "description": desc or "", + "bytes": bytes, + "unit": unit, + "total": size or 0, + "visible": False, + "progress_type": None if bytes else "summary", + } + self._stack = ExitStack() + super().__init__(size=size, value=value) + + @cached_property + def progress(self): + from dvc.ui import ui + from dvc.ui._rich_progress import RichTransferProgress + + if self._progress is not None: + return self._progress + + progress = RichTransferProgress( + transient=True, + disable=self.disable, + console=ui.error_console, + ) + return self._stack.enter_context(progress) + + @cached_property + def task(self): + return self.progress.add_task(**self._task_kwargs) + + def __enter__(self): + return self + + def close(self): + self.progress.clear_task(self.task) + self._stack.close() + + def call(self, hook_name=None, **kwargs): + self.progress.update( + self.task, + completed=self.value, + total=self.size, + visible=not self.disable, + ) + + def branch( + self, path_1, path_2, kwargs, child: Optional[FsspecCallback] = None + ): + child = child or RichCallback( + progress=self.progress, desc=path_1, bytes=True + ) + return super().branch(path_1, path_2, kwargs, child=child) + + +DEFAULT_CALLBACK = NoOpCallback() diff --git a/dvc/fs/azure.py b/dvc/objects/fs/azure.py similarity index 100% rename from dvc/fs/azure.py rename to dvc/objects/fs/azure.py diff --git a/dvc/fs/base.py b/dvc/objects/fs/base.py similarity index 99% rename from dvc/fs/base.py rename to dvc/objects/fs/base.py index a9d5231ad..a38e9c343 100644 --- a/dvc/fs/base.py +++ b/dvc/objects/fs/base.py @@ -131,9 +131,9 @@ class FileSystem: return missing def _check_requires(self, **kwargs): - from ..scheme import Schemes - from ..utils import format_link - from ..utils.pkg import PKG + from dvc.scheme import Schemes + from dvc.utils import format_link + from dvc.utils.pkg import PKG missing = self.get_missing_deps() if not missing: diff --git a/dvc/fs/gdrive.py b/dvc/objects/fs/gdrive.py similarity index 100% rename from dvc/fs/gdrive.py rename to dvc/objects/fs/gdrive.py diff --git a/dvc/fs/generic.py b/dvc/objects/fs/generic.py similarity index 100% rename from dvc/fs/generic.py rename to dvc/objects/fs/generic.py diff --git a/dvc/fs/git.py b/dvc/objects/fs/git.py similarity index 100% rename from dvc/fs/git.py rename to dvc/objects/fs/git.py diff --git a/dvc/fs/gs.py b/dvc/objects/fs/gs.py similarity index 100% rename from dvc/fs/gs.py rename to dvc/objects/fs/gs.py diff --git a/dvc/fs/hdfs.py b/dvc/objects/fs/hdfs.py similarity index 100% rename from dvc/fs/hdfs.py rename to dvc/objects/fs/hdfs.py diff --git a/dvc/fs/http.py b/dvc/objects/fs/http.py similarity index 100% rename from dvc/fs/http.py rename to dvc/objects/fs/http.py diff --git a/dvc/fs/https.py b/dvc/objects/fs/https.py similarity index 100% rename from dvc/fs/https.py rename to dvc/objects/fs/https.py diff --git a/dvc/fs/local.py b/dvc/objects/fs/local.py similarity index 99% rename from dvc/fs/local.py rename to dvc/objects/fs/local.py index 5160091c2..3bdf922ac 100644 --- a/dvc/fs/local.py +++ b/dvc/objects/fs/local.py @@ -6,9 +6,9 @@ import threading from fsspec import AbstractFileSystem from funcy import cached_property, wrap_prop -from dvc.fs import system from dvc.scheme import Schemes +from . import system from .base import FileSystem from .utils import copyfile, makedirs, move, remove, tmp_fname diff --git a/dvc/fs/memory.py b/dvc/objects/fs/memory.py similarity index 100% rename from dvc/fs/memory.py rename to dvc/objects/fs/memory.py diff --git a/dvc/fs/oss.py b/dvc/objects/fs/oss.py similarity index 100% rename from dvc/fs/oss.py rename to dvc/objects/fs/oss.py diff --git a/dvc/fs/path.py b/dvc/objects/fs/path.py similarity index 100% rename from dvc/fs/path.py rename to dvc/objects/fs/path.py diff --git a/dvc/fs/s3.py b/dvc/objects/fs/s3.py similarity index 100% rename from dvc/fs/s3.py rename to dvc/objects/fs/s3.py diff --git a/dvc/fs/ssh.py b/dvc/objects/fs/ssh.py similarity index 100% rename from dvc/fs/ssh.py rename to dvc/objects/fs/ssh.py diff --git a/dvc/fs/system.py b/dvc/objects/fs/system.py similarity index 100% rename from dvc/fs/system.py rename to dvc/objects/fs/system.py diff --git a/dvc/fs/utils.py b/dvc/objects/fs/utils.py similarity index 97% rename from dvc/fs/utils.py rename to dvc/objects/fs/utils.py index 0378ae031..e387537fb 100644 --- a/dvc/fs/utils.py +++ b/dvc/objects/fs/utils.py @@ -157,7 +157,7 @@ def copyfile( try: system.reflink(src, dest) except OSError: - from dvc.fs._callback import FsspecCallback + from ._callback import FsspecCallback with open(src, "rb") as fsrc, open(dest, "wb+") as fdest: with FsspecCallback.as_tqdm_callback( @@ -191,13 +191,6 @@ def tmp_fname(fname: "AnyFSPath" = "") -> "AnyFSPath": return os.fspath(fname) + "." + uuid() + ".tmp" -def as_posix(path: str) -> str: - import ntpath - import posixpath - - return path.replace(ntpath.sep, posixpath.sep) - - @contextmanager def as_atomic( fs: "FileSystem", to_info: "AnyFSPath", create_parents: bool = False diff --git a/dvc/fs/webdav.py b/dvc/objects/fs/webdav.py similarity index 100% rename from dvc/fs/webdav.py rename to dvc/objects/fs/webdav.py diff --git a/dvc/fs/webhdfs.py b/dvc/objects/fs/webhdfs.py similarity index 100% rename from dvc/fs/webhdfs.py rename to dvc/objects/fs/webhdfs.py diff --git a/dvc/objects/hash.py b/dvc/objects/hash.py index be8effc60..f2d4523b1 100644 --- a/dvc/objects/hash.py +++ b/dvc/objects/hash.py @@ -2,9 +2,8 @@ import hashlib import logging from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Tuple -from dvc.fs._callback import DEFAULT_CALLBACK, FsspecCallback, TqdmCallback -from dvc.fs.utils import is_exec - +from .fs._callback import DEFAULT_CALLBACK, FsspecCallback, TqdmCallback +from .fs.utils import is_exec from .hash_info import HashInfo from .istextfile import istextfile from .meta import Meta @@ -13,8 +12,7 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: - from dvc.fs.base import AnyFSPath, FileSystem - + from .fs.base import AnyFSPath, FileSystem from .state import StateBase diff --git a/dvc/objects/istextfile.py b/dvc/objects/istextfile.py index 6501bb694..402156329 100644 --- a/dvc/objects/istextfile.py +++ b/dvc/objects/istextfile.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from dvc.fs.base import AnyFSPath, FileSystem + from .fs.base import AnyFSPath, FileSystem TEXT_CHARS = bytes(range(32, 127)) + b"\n\r\t\f\b" DEFAULT_CHUNK_SIZE = 512 diff --git a/dvc/objects/state.py b/dvc/objects/state.py index de4a55d61..3761a7322 100644 --- a/dvc/objects/state.py +++ b/dvc/objects/state.py @@ -4,10 +4,9 @@ import logging import os from abc import ABC, abstractmethod -from dvc.fs.local import LocalFileSystem -from dvc.fs.system import inode as get_inode -from dvc.fs.utils import relpath - +from .fs import LocalFileSystem +from .fs.system import inode as get_inode +from .fs.utils import relpath from .hash_info import HashInfo from .utils import get_mtime_and_size diff --git a/dvc/objects/utils.py b/dvc/objects/utils.py index 765e240ae..989f84bf5 100644 --- a/dvc/objects/utils.py +++ b/dvc/objects/utils.py @@ -4,7 +4,7 @@ import json from typing import TYPE_CHECKING, Tuple if TYPE_CHECKING: - from dvc.fs.base import AnyFSPath, FileSystem + from .fs.base import AnyFSPath, FileSystem def get_mtime_and_size( diff --git a/dvc/output.py b/dvc/output.py index dfe1dff82..54a1d20d8 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -25,11 +25,13 @@ from .data.checkout import checkout from .data.stage import stage as ostage from .data.transfer import transfer as otransfer from .data.tree import Tree -from .fs import get_cloud_fs -from .fs.base import RemoteMissingDepsError -from .fs.hdfs import HDFSFileSystem -from .fs.local import LocalFileSystem -from .fs.s3 import S3FileSystem +from .fs import ( + HDFSFileSystem, + LocalFileSystem, + RemoteMissingDepsError, + S3FileSystem, + get_cloud_fs, +) from .objects.errors import ObjectFormatError from .scheme import Schemes from .utils import relpath diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index ba81ca948..bd4f5c3d5 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -15,7 +15,7 @@ from dvc.utils import env2bool from dvc.utils.fs import path_isin if TYPE_CHECKING: - from dvc.fs.base import FileSystem + from dvc.fs import FileSystem from dvc.objects.file import HashFile from dvc.repo.scm_context import SCMContext from dvc.scm import Base @@ -88,7 +88,7 @@ class Repo: uninitialized: bool = False, scm: "Base" = None, ): - from dvc.fs.local import localfs + from dvc.fs import localfs from dvc.scm import SCM, SCMError dvc_dir = None @@ -157,8 +157,7 @@ class Repo: from dvc.config import Config from dvc.data.db import ODBManager from dvc.data_cloud import DataCloud - from dvc.fs.git import GitFileSystem - from dvc.fs.local import localfs + from dvc.fs import GitFileSystem, localfs from dvc.lock import LockNoop, make_lock from dvc.objects.state import State, StateNoop from dvc.repo.metrics import Metrics @@ -280,7 +279,7 @@ class Repo: return DvcIgnoreFilter(self.fs, self.root_dir) def get_rev(self): - from dvc.fs.local import LocalFileSystem + from dvc.fs import LocalFileSystem assert self.scm if isinstance(self.fs, LocalFileSystem): @@ -323,7 +322,7 @@ class Repo: @classmethod def find_root(cls, root=None, fs=None) -> str: - from dvc.fs.local import LocalFileSystem, localfs + from dvc.fs import LocalFileSystem, localfs fs = fs or localfs root = root or os.curdir diff --git a/dvc/repo/brancher.py b/dvc/repo/brancher.py index f678855a4..f3df589c5 100644 --- a/dvc/repo/brancher.py +++ b/dvc/repo/brancher.py @@ -29,7 +29,7 @@ def brancher( # noqa: E302 yield "" return - from dvc.fs.local import LocalFileSystem + from dvc.fs import LocalFileSystem repo_root_parts = () if self.fs.path.isin(self.root_dir, self.scm.root_dir): @@ -63,7 +63,7 @@ def brancher( # noqa: E302 ) try: - from dvc.fs.git import GitFileSystem + from dvc.fs import GitFileSystem for sha, names in found_revs.items(): self.fs = GitFileSystem(scm=scm, rev=sha) diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index feeffe14e..f257f3c30 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -112,7 +112,7 @@ def _paths_checksums(repo, targets): def _output_paths(repo, targets): from dvc.data.stage import stage as ostage - from dvc.fs.local import LocalFileSystem + from dvc.fs import LocalFileSystem on_working_fs = isinstance(repo.fs, LocalFileSystem) diff --git a/dvc/repo/experiments/executor/ssh.py b/dvc/repo/experiments/executor/ssh.py index adc9cc7f1..080e9d8e3 100644 --- a/dvc/repo/experiments/executor/ssh.py +++ b/dvc/repo/experiments/executor/ssh.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Callable, Iterable, Optional from funcy import first -from dvc.fs.ssh import SSHFileSystem +from dvc.fs import SSHFileSystem from dvc.repo.experiments.base import ( EXEC_BRANCH, EXEC_CHECKPOINT, diff --git a/dvc/repo/experiments/init.py b/dvc/repo/experiments/init.py index a1961f4d8..92908fb22 100644 --- a/dvc/repo/experiments/init.py +++ b/dvc/repo/experiments/init.py @@ -162,7 +162,7 @@ def init_deps(stage: PipelineStage) -> List["Dependency"]: from funcy import rpartial from dvc.dependency import ParamsDependency - from dvc.fs.local import localfs + from dvc.fs import localfs new_deps = [dep for dep in stage.deps if not dep.exists] params, deps = lsplit(rpartial(isinstance, ParamsDependency), new_deps) diff --git a/dvc/repo/graph.py b/dvc/repo/graph.py index 91d6b8403..9539d7ce5 100644 --- a/dvc/repo/graph.py +++ b/dvc/repo/graph.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING, Iterator, List, Set -from dvc.fs.local import localfs +from dvc.fs import localfs from dvc.utils.fs import path_isin if TYPE_CHECKING: diff --git a/dvc/repo/index.py b/dvc/repo/index.py index d7c467d44..8c0001d19 100644 --- a/dvc/repo/index.py +++ b/dvc/repo/index.py @@ -21,7 +21,7 @@ if TYPE_CHECKING: from dvc.data.tree import Tree from dvc.dependency import Dependency, ParamsDependency - from dvc.fs.base import FileSystem + from dvc.fs import FileSystem from dvc.objects.db import ObjectDB from dvc.objects.hash_info import HashInfo from dvc.output import Output @@ -74,7 +74,7 @@ class Index: return self.stage_collector.collect_repo(onerror=onerror) def __repr__(self) -> str: - from dvc.fs.local import LocalFileSystem + from dvc.fs import LocalFileSystem rev = "workspace" if not isinstance(self.fs, LocalFileSystem): diff --git a/dvc/repo/stage.py b/dvc/repo/stage.py index ffd8d9c7e..01c79f87a 100644 --- a/dvc/repo/stage.py +++ b/dvc/repo/stage.py @@ -465,7 +465,7 @@ class StageLoad: the collection. """ from dvc.dvcfile import is_valid_filename - from dvc.fs.local import LocalFileSystem + from dvc.fs import LocalFileSystem scm = self.repo.scm sep = self.fs.sep diff --git a/dvc/rwlock.py b/dvc/rwlock.py index 226dc4540..35a0685e3 100644 --- a/dvc/rwlock.py +++ b/dvc/rwlock.py @@ -6,7 +6,7 @@ from contextlib import contextmanager from voluptuous import Invalid, Optional, Required, Schema from .exceptions import DvcException -from .fs.local import localfs +from .fs import localfs from .utils import relpath INFO_SCHEMA = {Required("pid"): int, Required("cmd"): str} diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index a410e4b64..a170c5b9e 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -317,7 +317,7 @@ def resolve_output(inp, out): def resolve_paths(repo, out, always_local=False): from urllib.parse import urlparse - from dvc.fs.local import localfs + from dvc.fs import localfs from ..dvcfile import DVC_FILE_SUFFIX from ..exceptions import DvcException diff --git a/dvc/utils/serialize/_common.py b/dvc/utils/serialize/_common.py index 8914dacf4..9a4198056 100644 --- a/dvc/utils/serialize/_common.py +++ b/dvc/utils/serialize/_common.py @@ -9,7 +9,7 @@ from typing_extensions import Protocol from dvc.exceptions import DvcException if TYPE_CHECKING: - from dvc.fs.base import FileSystem + from dvc.fs import FileSystem from dvc.types import AnyPath diff --git a/dvc/utils/strictyaml.py b/dvc/utils/strictyaml.py index 0b8cfda00..3a9d988f4 100644 --- a/dvc/utils/strictyaml.py +++ b/dvc/utils/strictyaml.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from ruamel.yaml import StreamMark from voluptuous import MultipleInvalid - from dvc.fs.base import FileSystem + from dvc.fs import FileSystem from dvc.ui import RichText diff --git a/tests/func/experiments/executor/test_ssh.py b/tests/func/experiments/executor/test_ssh.py index 104e49066..70835aad9 100644 --- a/tests/func/experiments/executor/test_ssh.py +++ b/tests/func/experiments/executor/test_ssh.py @@ -6,7 +6,7 @@ from urllib.parse import urlparse import pytest from dvc_ssh.tests.cloud import TEST_SSH_KEY_PATH, TEST_SSH_USER -from dvc.fs.ssh import SSHFileSystem +from dvc.fs import SSHFileSystem from dvc.repo.experiments.base import EXEC_HEAD, EXEC_MERGE from dvc.repo.experiments.executor.base import ExecutorInfo, ExecutorResult from dvc.repo.experiments.executor.ssh import SSHExecutor diff --git a/tests/func/objects/db/test_index.py b/tests/func/objects/db/test_index.py index af463a763..87dc8835b 100644 --- a/tests/func/objects/db/test_index.py +++ b/tests/func/objects/db/test_index.py @@ -2,7 +2,7 @@ import pytest from dvc.data.db import get_index from dvc.exceptions import DownloadError, UploadError -from dvc.fs.local import LocalFileSystem +from dvc.fs import LocalFileSystem from dvc.utils.fs import remove from tests.utils import clean_staging @@ -66,7 +66,7 @@ def test_clear_on_download_err(tmp_dir, dvc, index, mocker): assert list(index.hashes()) - mocker.patch("dvc.fs.generic.transfer", side_effect=Exception) + mocker.patch("dvc.objects.fs.generic.transfer", side_effect=Exception) with pytest.raises(DownloadError): dvc.pull() assert not list(index.hashes()) @@ -83,7 +83,7 @@ def test_partial_upload(tmp_dir, dvc, index, mocker): raise Exception("stop baz") return original(self, from_file, to_info, name, **kwargs) - mocker.patch("dvc.fs.generic.transfer", unreliable_upload) + mocker.patch("dvc.objects.fs.generic.transfer", unreliable_upload) with pytest.raises(UploadError): dvc.push() assert not list(index.hashes()) diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 9b3ee552e..de4cd4a80 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -21,8 +21,7 @@ from dvc.exceptions import ( OverlappingOutputPathsError, RecursiveAddingWhileUsingFilename, ) -from dvc.fs import system -from dvc.fs.local import LocalFileSystem +from dvc.fs import LocalFileSystem, system from dvc.objects.hash import file_md5 from dvc.objects.hash_info import HashInfo from dvc.output import ( diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index be3bb3063..e9e7994ba 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -18,8 +18,7 @@ from dvc.exceptions import ( DvcException, NoOutputOrStageError, ) -from dvc.fs import system -from dvc.fs.local import LocalFileSystem +from dvc.fs import LocalFileSystem, system from dvc.stage import Stage from dvc.stage.exceptions import StageFileDoesNotExistError from dvc.utils import relpath diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index 6aa3cad59..074c28baa 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -47,14 +47,14 @@ def test_source_change(erepo_dir): def test_cache_reused(erepo_dir, mocker, local_cloud): - from dvc import fs + from dvc.objects.fs import generic erepo_dir.add_remote(config=local_cloud.config) with erepo_dir.chdir(): erepo_dir.dvc_gen("file", "text", commit="add file") erepo_dir.dvc.push() - download_spy = mocker.spy(fs.generic, "transfer") + download_spy = mocker.spy(generic, "transfer") # Use URL to prevent any fishy optimizations url = f"file://{erepo_dir}" diff --git a/tests/func/test_fs.py b/tests/func/test_fs.py index 0fcf5fd33..6df69466e 100644 --- a/tests/func/test_fs.py +++ b/tests/func/test_fs.py @@ -5,9 +5,8 @@ from os.path import join import pytest -from dvc.fs import get_cloud_fs +from dvc.fs import LocalFileSystem, get_cloud_fs from dvc.fs._callback import DEFAULT_CALLBACK, FsspecCallback -from dvc.fs.local import LocalFileSystem from dvc.repo import Repo diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index 096ac9b23..a614ca9b4 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -9,7 +9,7 @@ from git import Repo from dvc.cli import main from dvc.data.db.local import LocalObjectDB from dvc.exceptions import CollectCacheError -from dvc.fs.local import LocalFileSystem +from dvc.fs import LocalFileSystem from dvc.repo import Repo as DvcRepo from dvc.utils.fs import remove from tests.basic_env import TestDir, TestDvcGit diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index c07b1ffc7..073c881f5 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -125,7 +125,7 @@ def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname): def test_ignore_on_branch(tmp_dir, scm, dvc): - from dvc.fs.git import GitFileSystem + from dvc.fs import GitFileSystem tmp_dir.scm_gen({"foo": "foo", "bar": "bar"}, commit="add files") diff --git a/tests/func/test_import.py b/tests/func/test_import.py index d563eeb5f..279dd4e89 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -308,7 +308,7 @@ def test_download_error_pulling_imported_stage(tmp_dir, dvc, erepo_dir): remove(dst_cache) with patch( - "dvc.fs.generic.transfer", side_effect=Exception + "dvc.objects.fs.generic.transfer", side_effect=Exception ), pytest.raises(DownloadError): dvc.pull(["foo_imported.dvc"]) diff --git a/tests/func/test_odb.py b/tests/func/test_odb.py index bda61a848..4a9b0deb5 100644 --- a/tests/func/test_odb.py +++ b/tests/func/test_odb.py @@ -180,7 +180,7 @@ def test_default_cache_type(dvc): @pytest.mark.skipif(os.name == "nt", reason="Not supported for Windows.") @pytest.mark.parametrize("group", [False, True]) def test_shared_cache(tmp_dir, dvc, group): - from dvc.fs.system import umask + from dvc.fs import system if group: with dvc.config.edit() as conf: @@ -201,7 +201,7 @@ def test_shared_cache(tmp_dir, dvc, group): actual[path] = oct(stat.S_IMODE(os.stat(path).st_mode)) file_mode = oct(0o444) - dir_mode = oct(0o2775 if group else (0o777 & ~umask)) + dir_mode = oct(0o2775 if group else (0o777 & ~system.umask)) expected = { os.path.join(cache_dir, "17"): dir_mode, diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index 63d92ffa7..ceebb6d1a 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -169,14 +169,14 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote): - from dvc import fs + from dvc.objects.fs import generic foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0] bar = tmp_dir.dvc_gen({"bar": "bar content"})[0].outs[0] baz = tmp_dir.dvc_gen({"baz": {"foo": "foo content"}})[0].outs[0] # Faulty upload version, failing on foo - original = fs.generic.transfer + original = generic.transfer odb = dvc.cloud.get_remote_odb("upstream") def unreliable_upload(from_fs, from_info, to_fs, to_info, **kwargs): @@ -186,7 +186,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote): raise Exception("stop foo") return original(from_fs, from_info, to_fs, to_info, **kwargs) - with patch.object(fs.generic, "transfer", unreliable_upload): + with patch.object(generic, "transfer", unreliable_upload): with pytest.raises(UploadError) as upload_error_info: dvc.push() assert upload_error_info.value.amount == 2 @@ -200,7 +200,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote): remove(dvc.odb.local.cache_dir) baz._collect_used_dir_cache() - with patch.object(fs.generic, "transfer", side_effect=Exception): + with patch.object(generic, "transfer", side_effect=Exception): with pytest.raises(DownloadError) as download_error_info: dvc.pull() # error count should be len(.dir + standalone file checksums) @@ -214,7 +214,7 @@ def test_raise_on_too_many_open_files( tmp_dir.dvc_gen({"file": "file content"}) mocker.patch( - "dvc.fs.generic.transfer", + "dvc.objects.fs.generic.transfer", side_effect=OSError(errno.EMFILE, "Too many open files"), ) @@ -263,12 +263,12 @@ def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory): def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote): - from dvc import fs + from dvc.objects.fs import generic foo = tmp_dir.dvc_gen({"foo": {"bar": "bar content"}})[0].outs[0] tmp_dir.dvc_gen({"baz": "baz content"}) - mocked_upload = mocker.spy(fs.generic, "transfer") + mocked_upload = mocker.spy(generic, "transfer") dvc.push() # foo .dir file should be uploaded after bar diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index 7abed51d7..c451a3fa5 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -14,8 +14,7 @@ from dvc.exceptions import ( ReproductionError, StagePathAsOutputError, ) -from dvc.fs import system -from dvc.fs.local import LocalFileSystem +from dvc.fs import LocalFileSystem, system from dvc.objects.hash import file_md5 from dvc.output import Output from dvc.stage import Stage diff --git a/tests/func/test_stage.py b/tests/func/test_stage.py index 512fd96e9..b661ea870 100644 --- a/tests/func/test_stage.py +++ b/tests/func/test_stage.py @@ -5,7 +5,7 @@ import pytest from dvc.cli import main from dvc.dvcfile import SingleStageFile -from dvc.fs.local import LocalFileSystem +from dvc.fs import LocalFileSystem from dvc.output import Output from dvc.repo import Repo, lock_repo from dvc.stage import PipelineStage, Stage diff --git a/tests/unit/fs/test_azure.py b/tests/unit/fs/test_azure.py index 7994b00f1..3825f18f5 100644 --- a/tests/unit/fs/test_azure.py +++ b/tests/unit/fs/test_azure.py @@ -1,6 +1,6 @@ import pytest -from dvc.fs.azure import AzureAuthError, AzureFileSystem +from dvc.fs import AzureAuthError, AzureFileSystem container_name = "container-name" connection_string = ( diff --git a/tests/unit/fs/test_base.py b/tests/unit/fs/test_base.py index 8d35654bf..d475114ba 100644 --- a/tests/unit/fs/test_base.py +++ b/tests/unit/fs/test_base.py @@ -1,6 +1,6 @@ import pytest -from dvc.fs.base import FileSystem, RemoteMissingDepsError +from dvc.fs import FileSystem, RemoteMissingDepsError @pytest.mark.parametrize( diff --git a/tests/unit/fs/test_fs.py b/tests/unit/fs/test_fs.py index b41419468..09014f7c1 100644 --- a/tests/unit/fs/test_fs.py +++ b/tests/unit/fs/test_fs.py @@ -1,13 +1,16 @@ import pytest from dvc.config import RemoteNotFoundError -from dvc.fs import get_fs_cls, get_fs_config -from dvc.fs.hdfs import HDFSFileSystem -from dvc.fs.http import HTTPFileSystem -from dvc.fs.https import HTTPSFileSystem -from dvc.fs.local import LocalFileSystem -from dvc.fs.s3 import S3FileSystem -from dvc.fs.ssh import SSHFileSystem +from dvc.fs import ( + HDFSFileSystem, + HTTPFileSystem, + HTTPSFileSystem, + LocalFileSystem, + S3FileSystem, + SSHFileSystem, + get_fs_cls, + get_fs_config, +) @pytest.mark.parametrize( diff --git a/tests/unit/fs/test_path.py b/tests/unit/fs/test_path.py index caf41342c..0ecaa4ec5 100644 --- a/tests/unit/fs/test_path.py +++ b/tests/unit/fs/test_path.py @@ -1,6 +1,6 @@ import pytest -from dvc.fs.path import Path +from dvc.fs import Path @pytest.mark.parametrize("prefix", ["", "/"]) diff --git a/tests/unit/fs/test_s3.py b/tests/unit/fs/test_s3.py index d2a8bda78..193aa6cc8 100644 --- a/tests/unit/fs/test_s3.py +++ b/tests/unit/fs/test_s3.py @@ -3,7 +3,7 @@ import os import pytest from dvc.config import ConfigError -from dvc.fs.s3 import S3FileSystem +from dvc.fs import S3FileSystem bucket_name = "bucket-name" prefix = "some/prefix" diff --git a/tests/unit/fs/test_ssh.py b/tests/unit/fs/test_ssh.py index b05613747..e31f3f580 100644 --- a/tests/unit/fs/test_ssh.py +++ b/tests/unit/fs/test_ssh.py @@ -6,7 +6,7 @@ import os import pytest from mock import mock_open, patch -from dvc.fs.ssh import DEFAULT_PORT, SSHFileSystem +from dvc.fs import DEFAULT_SSH_PORT, SSHFileSystem def test_get_kwargs_from_urls(): @@ -44,7 +44,7 @@ def test_init(): def test_ssh_ask_password(mocker): - mocker.patch("dvc.fs.ssh.ask_password", return_value="fish") + mocker.patch("dvc.objects.fs.ssh.ask_password", return_value="fish") fs = SSHFileSystem(user="test", host="2.2.2.2", ask_password=True) assert fs.fs_args["password"] == fs.fs_args["passphrase"] == "fish" @@ -99,7 +99,7 @@ def test_ssh_user(mock_file, config, expected_user): [ ({"host": "example.com"}, 1234), ({"host": "example.com", "port": 4321}, 4321), - ({"host": "not_in_ssh_config.com"}, DEFAULT_PORT), + ({"host": "not_in_ssh_config.com"}, DEFAULT_SSH_PORT), ({"host": "not_in_ssh_config.com", "port": 2222}, 2222), ], ) diff --git a/tests/unit/objects/db/test_local.py b/tests/unit/objects/db/test_local.py index bdcd219cf..db58be3bc 100644 --- a/tests/unit/objects/db/test_local.py +++ b/tests/unit/objects/db/test_local.py @@ -4,7 +4,7 @@ import os import pytest from dvc.data.db.local import LocalObjectDB -from dvc.fs.local import LocalFileSystem +from dvc.fs import LocalFileSystem from dvc.objects.hash_info import HashInfo diff --git a/tests/unit/output/test_load.py b/tests/unit/output/test_load.py index 37a0370a4..00cbd5e9d 100644 --- a/tests/unit/output/test_load.py +++ b/tests/unit/output/test_load.py @@ -3,8 +3,7 @@ import pytest from dvc import output -from dvc.fs.local import LocalFileSystem -from dvc.fs.s3 import S3FileSystem +from dvc.fs import LocalFileSystem, S3FileSystem from dvc.output import Output from dvc.stage import Stage diff --git a/tests/unit/output/test_output.py b/tests/unit/output/test_output.py index c539ee310..01b243aeb 100644 --- a/tests/unit/output/test_output.py +++ b/tests/unit/output/test_output.py @@ -5,7 +5,7 @@ import pytest from funcy import first from voluptuous import MultipleInvalid, Schema -from dvc.fs.base import RemoteMissingDepsError +from dvc.fs import RemoteMissingDepsError from dvc.ignore import _no_match from dvc.output import CHECKSUM_SCHEMA, Output from dvc.stage import Stage diff --git a/tests/unit/remote/test_base.py b/tests/unit/remote/test_base.py index b7e2b0ed6..7b599b999 100644 --- a/tests/unit/remote/test_base.py +++ b/tests/unit/remote/test_base.py @@ -2,7 +2,7 @@ import math import posixpath from unittest import mock -from dvc.fs.base import FileSystem +from dvc.fs import FileSystem from dvc.objects.db import ObjectDB diff --git a/tests/unit/remote/test_gdrive.py b/tests/unit/remote/test_gdrive.py index e03ff5642..90b2936ec 100644 --- a/tests/unit/remote/test_gdrive.py +++ b/tests/unit/remote/test_gdrive.py @@ -1,6 +1,6 @@ import pytest -from dvc.fs.gdrive import GDriveAuthError, GDriveFileSystem +from dvc.fs import GDriveAuthError, GDriveFileSystem USER_CREDS_TOKEN_REFRESH_ERROR = '{"access_token": "", "client_id": "", "client_secret": "", "refresh_token": "", "token_expiry": "", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "", "expires_in": 3600, "scope": "https://www.googleapis.com/auth/drive.appdata https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive", "https://www.googleapis.com/auth/drive.appdata"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": true, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}' # noqa: E501 diff --git a/tests/unit/remote/test_http.py b/tests/unit/remote/test_http.py index effa53377..72b164ff6 100644 --- a/tests/unit/remote/test_http.py +++ b/tests/unit/remote/test_http.py @@ -2,7 +2,7 @@ import ssl from mock import patch -from dvc.fs.http import HTTPFileSystem +from dvc.fs import HTTPFileSystem def test_public_auth_method(dvc): diff --git a/tests/unit/remote/test_oss.py b/tests/unit/remote/test_oss.py index 8cf473e38..fbb0e3c86 100644 --- a/tests/unit/remote/test_oss.py +++ b/tests/unit/remote/test_oss.py @@ -1,4 +1,4 @@ -from dvc.fs.oss import OSSFileSystem +from dvc.fs import OSSFileSystem bucket_name = "bucket-name" endpoint = "endpoint" diff --git a/tests/unit/remote/test_remote.py b/tests/unit/remote/test_remote.py index b169d6cce..a2cf3dd07 100644 --- a/tests/unit/remote/test_remote.py +++ b/tests/unit/remote/test_remote.py @@ -3,9 +3,7 @@ import os import pytest from dvc.data.db import get_index -from dvc.fs import get_cloud_fs -from dvc.fs.gs import GSFileSystem -from dvc.fs.s3 import S3FileSystem +from dvc.fs import GSFileSystem, S3FileSystem, get_cloud_fs def test_remote_with_hash_jobs(dvc): diff --git a/tests/unit/remote/test_webdav.py b/tests/unit/remote/test_webdav.py index e207dfe97..c1b599ce4 100644 --- a/tests/unit/remote/test_webdav.py +++ b/tests/unit/remote/test_webdav.py @@ -2,8 +2,7 @@ from unittest.mock import patch import pytest -from dvc.fs import get_cloud_fs -from dvc.fs.webdav import WebDAVFileSystem, WebDAVSFileSystem +from dvc.fs import WebDAVFileSystem, WebDAVSFileSystem, get_cloud_fs from tests.utils.asserts import issubset url_fmt = "{scheme}://{user}@example.com/public.php/webdav" @@ -66,7 +65,7 @@ def test_token(): ) -@patch("dvc.fs.webdav.ask_password") +@patch("dvc.objects.fs.webdav.ask_password") def test_ask_password(ask_password_mocked): ask_password_mocked.return_value = "pass" host = "host" diff --git a/tests/unit/remote/test_webhdfs.py b/tests/unit/remote/test_webhdfs.py index 341aa4928..91b33d55a 100644 --- a/tests/unit/remote/test_webhdfs.py +++ b/tests/unit/remote/test_webhdfs.py @@ -3,7 +3,7 @@ from unittest.mock import Mock, create_autospec import pytest import requests -from dvc.fs.webhdfs import WebHDFSFileSystem +from dvc.fs import WebHDFSFileSystem host = "host" kerberos = False diff --git a/tests/unit/stage/test_utils.py b/tests/unit/stage/test_utils.py index 1b87c0ad0..86d845653 100644 --- a/tests/unit/stage/test_utils.py +++ b/tests/unit/stage/test_utils.py @@ -1,6 +1,6 @@ import os -from dvc.fs.local import localfs +from dvc.fs import localfs from dvc.stage.utils import resolve_paths diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index ec3e7c84a..e15586fac 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -4,7 +4,7 @@ import textwrap import pytest from dvc.config import Config -from dvc.fs.local import LocalFileSystem +from dvc.fs import LocalFileSystem @pytest.mark.parametrize( diff --git a/tests/unit/test_context.py b/tests/unit/test_context.py index d89e2ed08..a86088498 100644 --- a/tests/unit/test_context.py +++ b/tests/unit/test_context.py @@ -3,7 +3,7 @@ from math import pi import pytest -from dvc.fs.local import LocalFileSystem +from dvc.fs import LocalFileSystem from dvc.parsing import DEFAULT_PARAMS_FILE from dvc.parsing.context import ( Context, diff --git a/tests/unit/test_ignore.py b/tests/unit/test_ignore.py index 3a6be60ef..c5f4b5a3d 100644 --- a/tests/unit/test_ignore.py +++ b/tests/unit/test_ignore.py @@ -7,7 +7,7 @@ from dvc.ignore import DvcIgnorePatterns def mock_dvcignore(dvcignore_path, patterns): - from dvc.fs.local import localfs + from dvc.fs import localfs fs = MagicMock() fs.path = localfs.path diff --git a/tests/unit/utils/test_fs.py b/tests/unit/utils/test_fs.py index 191b74e4f..f06ba3c39 100644 --- a/tests/unit/utils/test_fs.py +++ b/tests/unit/utils/test_fs.py @@ -4,8 +4,7 @@ import os import pytest import dvc -from dvc.fs import system -from dvc.fs.local import LocalFileSystem +from dvc.fs import LocalFileSystem, system from dvc.objects.utils import get_mtime_and_size from dvc.utils import relpath from dvc.utils.fs import ( diff --git a/tests/unit/utils/test_stream.py b/tests/unit/utils/test_stream.py index edbb15774..07869fa3e 100644 --- a/tests/unit/utils/test_stream.py +++ b/tests/unit/utils/test_stream.py @@ -1,7 +1,7 @@ import pytest from dvc.data.stream import HashedStreamReader -from dvc.fs.local import LocalFileSystem +from dvc.fs import LocalFileSystem from dvc.objects.hash import file_md5 from dvc.objects.istextfile import DEFAULT_CHUNK_SIZE, istextfile diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 111a88015..7befade7a 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -46,7 +46,7 @@ def dump_sv(stream, metrics, delimiter=",", header=True): def clean_staging(): from dvc.data.stage import _STAGING_MEMFS_PATH - from dvc.fs.memory import MemoryFileSystem + from dvc.fs import MemoryFileSystem try: MemoryFileSystem().fs.rm(