fs: move filesystems to dvc.objects.fs.implementations

dvc/data/db/index.py

@@ -89,7 +89,7 @@ class ObjectDBIndex(ObjectDBIndexBase):
         name: str,
     ):  # pylint: disable=super-init-not-called
         from dvc.objects.cache import Cache, Index
-        from dvc.objects.fs.local import LocalFileSystem
+        from dvc.objects.fs import LocalFileSystem
         from dvc.objects.fs.utils import makedirs
 
         self.index_dir = os.path.join(tmp_dir, self.INDEX_DIR, name)

dvc/data/stage.py

@@ -194,7 +194,7 @@ def _get_staging(odb: "ObjectDB") -> "ReferenceObjectDB":
     Staging will be a reference ODB stored in the the global memfs.
     """
 
-    from dvc.objects.fs.memory import MemoryFileSystem
+    from dvc.objects.fs import MemoryFileSystem
 
     fs = MemoryFileSystem()
     fs_path = _make_staging_url(fs, odb, odb.fs_path)

dvc/data/tree.py

@@ -55,7 +55,7 @@ class Tree(HashFile):
         self._dict[key] = (meta, oid)
 
     def digest(self, hash_info: Optional["HashInfo"] = None):
-        from dvc.objects.fs.memory import MemoryFileSystem
+        from dvc.objects.fs import MemoryFileSystem
         from dvc.objects.fs.utils import tmp_fname
 
         memfs = MemoryFileSystem()

dvc/fs/__init__.py

@@ -4,11 +4,13 @@ from dvc.objects.fs import (  # noqa: F401
     FS_MAP,
     AzureFileSystem,
     GDriveFileSystem,
+    GitFileSystem,
     GSFileSystem,
     HDFSFileSystem,
     HTTPFileSystem,
     HTTPSFileSystem,
     LocalFileSystem,
+    MemoryFileSystem,
     OSSFileSystem,
     S3FileSystem,
     SSHFileSystem,
@@ -22,17 +24,17 @@ from dvc.objects.fs import (  # noqa: F401
     get_fs_config,
     system,
 )
-from dvc.objects.fs.azure import AzureAuthError  # noqa: F401
 from dvc.objects.fs.base import (  # noqa: F401
     AnyFSPath,
     FileSystem,
     RemoteMissingDepsError,
 )
-from dvc.objects.fs.gdrive import GDriveAuthError  # noqa: F401
-from dvc.objects.fs.git import GitFileSystem  # noqa: F401
-from dvc.objects.fs.local import localfs  # noqa: F401
-from dvc.objects.fs.memory import MemoryFileSystem  # noqa: F401
+from dvc.objects.fs.implementations.azure import AzureAuthError  # noqa: F401
+from dvc.objects.fs.implementations.gdrive import GDriveAuthError  # noqa: F401
+from dvc.objects.fs.implementations.local import localfs  # noqa: F401
+from dvc.objects.fs.implementations.ssh import (  # noqa: F401
+    DEFAULT_PORT as DEFAULT_SSH_PORT,
+)
 from dvc.objects.fs.path import Path  # noqa: F401
-from dvc.objects.fs.ssh import DEFAULT_PORT as DEFAULT_SSH_PORT  # noqa: F401
 
 # pylint: enable=unused-import

dvc/objects/fs/__init__.py

@@ -3,18 +3,20 @@ from urllib.parse import urlparse
 from dvc.scheme import Schemes
 
 from . import generic  # noqa: F401
-from .azure import AzureFileSystem
-from .gdrive import GDriveFileSystem
-from .gs import GSFileSystem
-from .hdfs import HDFSFileSystem
-from .http import HTTPFileSystem
-from .https import HTTPSFileSystem
-from .local import LocalFileSystem
-from .oss import OSSFileSystem
-from .s3 import S3FileSystem
-from .ssh import SSHFileSystem
-from .webdav import WebDAVFileSystem, WebDAVSFileSystem
-from .webhdfs import WebHDFSFileSystem
+from .implementations.azure import AzureFileSystem
+from .implementations.gdrive import GDriveFileSystem
+from .implementations.git import GitFileSystem  # noqa: F401
+from .implementations.gs import GSFileSystem
+from .implementations.hdfs import HDFSFileSystem
+from .implementations.http import HTTPFileSystem
+from .implementations.https import HTTPSFileSystem
+from .implementations.local import LocalFileSystem
+from .implementations.memory import MemoryFileSystem  # noqa: F401
+from .implementations.oss import OSSFileSystem
+from .implementations.s3 import S3FileSystem
+from .implementations.ssh import SSHFileSystem
+from .implementations.webdav import WebDAVFileSystem, WebDAVSFileSystem
+from .implementations.webhdfs import WebHDFSFileSystem
 
 FS_MAP = {
     Schemes.AZURE: AzureFileSystem,

dvc/objects/fs/base.py

@@ -474,7 +474,7 @@ class FileSystem:
     ) -> None:
         # Currently, the implementation is non-recursive if the paths are
         # provided as a list, and recursive if it's a single path.
-        from .local import localfs
+        from .implementations.local import localfs
 
         def get_file(rpath, lpath, **kwargs):
             localfs.makedirs(localfs.path.parent(lpath), exist_ok=True)

dvc/objects/fs/generic.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, List, Optional
 
 from ._callback import DEFAULT_CALLBACK
 from .base import RemoteActionNotImplemented
-from .local import LocalFileSystem
+from .implementations.local import LocalFileSystem
 from .utils import as_atomic
 
 if TYPE_CHECKING:
@@ -47,7 +47,7 @@ def copy(
         return to_fs.put_file(from_path, to_path, callback=callback)
 
     if isinstance(to_fs, LocalFileSystem):
-        from .local import localfs
+        from .implementations.local import localfs
 
         with as_atomic(localfs, to_path, create_parents=True) as tmp_file:
             return from_fs.get_file(from_path, tmp_file, callback=callback)

dvc/objects/fs/implementations/azure.py

@@ -10,7 +10,7 @@ from dvc.exceptions import DvcException
 from dvc.scheme import Schemes
 from dvc.utils import format_link
 
-from .base import ObjectFileSystem
+from ..base import ObjectFileSystem
 
 logger = logging.getLogger(__name__)
 _DEFAULT_CREDS_STEPS = (

dvc/objects/fs/implementations/gdrive.py

@@ -8,8 +8,8 @@ from dvc.exceptions import DvcException
 from dvc.scheme import Schemes
 from dvc.utils import format_link
 
-from .base import FileSystem
-from .utils import tmp_fname
+from ..base import FileSystem
+from ..utils import tmp_fname
 
 logger = logging.getLogger(__name__)
 FOLDER_MIME_TYPE = "application/vnd.google-apps.folder"

dvc/objects/fs/implementations/git.py

@@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Any
 
 from funcy import cached_property, wrap_prop
 
-from .base import FileSystem
+from ..base import FileSystem
 
 if TYPE_CHECKING:
     from scmrepo.fs import GitFileSystem as FsspecGitFileSystem

dvc/objects/fs/implementations/gs.py

@@ -5,7 +5,7 @@ from funcy import cached_property, wrap_prop
 from dvc.scheme import Schemes
 
 # pylint:disable=abstract-method
-from .base import ObjectFileSystem
+from ..base import ObjectFileSystem
 
 
 class GSFileSystem(ObjectFileSystem):

dvc/objects/fs/implementations/hdfs.py

@@ -8,7 +8,7 @@ from funcy import cached_property, wrap_prop
 from dvc.scheme import Schemes
 from dvc.utils import fix_env
 
-from .base import FileSystem
+from ..base import FileSystem
 
 CHECKSUM_REGEX = re.compile(r".*\t.*\t(?P<checksum>.*)")
 

dvc/objects/fs/implementations/http.py

@@ -6,8 +6,8 @@ from funcy import cached_property, memoize, wrap_with
 from dvc import prompt
 from dvc.scheme import Schemes
 
-from ._callback import DEFAULT_CALLBACK, FsspecCallback
-from .base import AnyFSPath, FileSystem
+from .._callback import DEFAULT_CALLBACK, FsspecCallback
+from ..base import AnyFSPath, FileSystem
 
 
 @wrap_with(threading.Lock())

dvc/objects/fs/implementations/local.py

@@ -8,9 +8,9 @@ from funcy import cached_property, wrap_prop
 
 from dvc.scheme import Schemes
 
-from . import system
-from .base import FileSystem
-from .utils import copyfile, makedirs, move, remove, tmp_fname
+from .. import system
+from ..base import FileSystem
+from ..utils import copyfile, makedirs, move, remove, tmp_fname
 
 logger = logging.getLogger(__name__)
 
@@ -180,7 +180,7 @@ class LocalFileSystem(FileSystem):
 
     @cached_property
     def path(self):
-        from .path import Path
+        from ..path import Path
 
         return Path(self.sep, getcwd=os.getcwd, realpath=os.path.realpath)
 

dvc/objects/fs/implementations/memory.py

@@ -4,7 +4,7 @@ from funcy import cached_property, wrap_prop
 
 from dvc.scheme import Schemes
 
-from .base import FileSystem
+from ..base import FileSystem
 
 
 class MemoryFileSystem(FileSystem):  # pylint:disable=abstract-method

dvc/objects/fs/implementations/oss.py

@@ -6,7 +6,7 @@ from funcy import cached_property, wrap_prop
 
 from dvc.scheme import Schemes
 
-from .base import ObjectFileSystem
+from ..base import ObjectFileSystem
 
 logger = logging.getLogger(__name__)
 

dvc/objects/fs/implementations/s3.py

@@ -6,7 +6,7 @@ from funcy import cached_property, wrap_prop
 
 from dvc.scheme import Schemes
 
-from .base import ObjectFileSystem
+from ..base import ObjectFileSystem
 
 _AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config")
 
@@ -38,7 +38,7 @@ class S3FileSystem(ObjectFileSystem):
 
         from boto3.s3.transfer import TransferConfig
 
-        from .utils import human_readable_to_bytes
+        from ..utils import human_readable_to_bytes
 
         config, transfer_config = {}, {}
         for key, value in s3_config.items():

dvc/objects/fs/implementations/ssh.py

@@ -7,9 +7,9 @@ from funcy import cached_property, memoize, silent, wrap_prop, wrap_with
 from dvc import prompt
 from dvc.scheme import Schemes
 
-from ._callback import DEFAULT_CALLBACK
-from .base import FileSystem
-from .utils import as_atomic
+from .._callback import DEFAULT_CALLBACK
+from ..base import FileSystem
+from ..utils import as_atomic
 
 DEFAULT_PORT = 22
 

dvc/objects/fs/implementations/webdav.py

@@ -5,7 +5,7 @@ from funcy import cached_property, wrap_prop
 
 from dvc.scheme import Schemes
 
-from .base import FileSystem
+from ..base import FileSystem
 from .http import ask_password
 
 logger = logging.getLogger(__name__)

dvc/objects/fs/implementations/webhdfs.py

@@ -5,7 +5,7 @@ from funcy import cached_property, wrap_prop
 from dvc.scheme import Schemes
 
 # pylint:disable=abstract-method
-from .base import FileSystem
+from ..base import FileSystem
 
 
 class WebHDFSFileSystem(FileSystem):

tests/unit/fs/test_ssh.py

@@ -44,7 +44,9 @@ def test_init():
 
 
 def test_ssh_ask_password(mocker):
-    mocker.patch("dvc.objects.fs.ssh.ask_password", return_value="fish")
+    mocker.patch(
+        "dvc.objects.fs.implementations.ssh.ask_password", return_value="fish"
+    )
     fs = SSHFileSystem(user="test", host="2.2.2.2", ask_password=True)
     assert fs.fs_args["password"] == fs.fs_args["passphrase"] == "fish"
 

tests/unit/remote/test_webdav.py

@@ -65,7 +65,7 @@ def test_token():
     )
 
 
-@patch("dvc.objects.fs.webdav.ask_password")
+@patch("dvc.objects.fs.implementations.webdav.ask_password")
 def test_ask_password(ask_password_mocked):
     ask_password_mocked.return_value = "pass"
     host = "host"