diff --git a/dvc/data/db/index.py b/dvc/data/db/index.py index b2faf114f..7c8c5623f 100644 --- a/dvc/data/db/index.py +++ b/dvc/data/db/index.py @@ -89,7 +89,7 @@ class ObjectDBIndex(ObjectDBIndexBase): name: str, ): # pylint: disable=super-init-not-called from dvc.objects.cache import Cache, Index - from dvc.objects.fs.local import LocalFileSystem + from dvc.objects.fs import LocalFileSystem from dvc.objects.fs.utils import makedirs self.index_dir = os.path.join(tmp_dir, self.INDEX_DIR, name) diff --git a/dvc/data/stage.py b/dvc/data/stage.py index 67b48fa87..a451066b8 100644 --- a/dvc/data/stage.py +++ b/dvc/data/stage.py @@ -194,7 +194,7 @@ def _get_staging(odb: "ObjectDB") -> "ReferenceObjectDB": Staging will be a reference ODB stored in the the global memfs. """ - from dvc.objects.fs.memory import MemoryFileSystem + from dvc.objects.fs import MemoryFileSystem fs = MemoryFileSystem() fs_path = _make_staging_url(fs, odb, odb.fs_path) diff --git a/dvc/data/tree.py b/dvc/data/tree.py index f7c90cc56..e1b2b7269 100644 --- a/dvc/data/tree.py +++ b/dvc/data/tree.py @@ -55,7 +55,7 @@ class Tree(HashFile): self._dict[key] = (meta, oid) def digest(self, hash_info: Optional["HashInfo"] = None): - from dvc.objects.fs.memory import MemoryFileSystem + from dvc.objects.fs import MemoryFileSystem from dvc.objects.fs.utils import tmp_fname memfs = MemoryFileSystem() diff --git a/dvc/fs/__init__.py b/dvc/fs/__init__.py index df0426aed..6f6f3ed99 100644 --- a/dvc/fs/__init__.py +++ b/dvc/fs/__init__.py @@ -4,11 +4,13 @@ from dvc.objects.fs import ( # noqa: F401 FS_MAP, AzureFileSystem, GDriveFileSystem, + GitFileSystem, GSFileSystem, HDFSFileSystem, HTTPFileSystem, HTTPSFileSystem, LocalFileSystem, + MemoryFileSystem, OSSFileSystem, S3FileSystem, SSHFileSystem, @@ -22,17 +24,17 @@ from dvc.objects.fs import ( # noqa: F401 get_fs_config, system, ) -from dvc.objects.fs.azure import AzureAuthError # noqa: F401 from dvc.objects.fs.base import ( # noqa: F401 AnyFSPath, FileSystem, RemoteMissingDepsError, ) -from dvc.objects.fs.gdrive import GDriveAuthError # noqa: F401 -from dvc.objects.fs.git import GitFileSystem # noqa: F401 -from dvc.objects.fs.local import localfs # noqa: F401 -from dvc.objects.fs.memory import MemoryFileSystem # noqa: F401 +from dvc.objects.fs.implementations.azure import AzureAuthError # noqa: F401 +from dvc.objects.fs.implementations.gdrive import GDriveAuthError # noqa: F401 +from dvc.objects.fs.implementations.local import localfs # noqa: F401 +from dvc.objects.fs.implementations.ssh import ( # noqa: F401 + DEFAULT_PORT as DEFAULT_SSH_PORT, +) from dvc.objects.fs.path import Path # noqa: F401 -from dvc.objects.fs.ssh import DEFAULT_PORT as DEFAULT_SSH_PORT # noqa: F401 # pylint: enable=unused-import diff --git a/dvc/objects/fs/__init__.py b/dvc/objects/fs/__init__.py index e183953d8..4cb5384cc 100644 --- a/dvc/objects/fs/__init__.py +++ b/dvc/objects/fs/__init__.py @@ -3,18 +3,20 @@ from urllib.parse import urlparse from dvc.scheme import Schemes from . import generic # noqa: F401 -from .azure import AzureFileSystem -from .gdrive import GDriveFileSystem -from .gs import GSFileSystem -from .hdfs import HDFSFileSystem -from .http import HTTPFileSystem -from .https import HTTPSFileSystem -from .local import LocalFileSystem -from .oss import OSSFileSystem -from .s3 import S3FileSystem -from .ssh import SSHFileSystem -from .webdav import WebDAVFileSystem, WebDAVSFileSystem -from .webhdfs import WebHDFSFileSystem +from .implementations.azure import AzureFileSystem +from .implementations.gdrive import GDriveFileSystem +from .implementations.git import GitFileSystem # noqa: F401 +from .implementations.gs import GSFileSystem +from .implementations.hdfs import HDFSFileSystem +from .implementations.http import HTTPFileSystem +from .implementations.https import HTTPSFileSystem +from .implementations.local import LocalFileSystem +from .implementations.memory import MemoryFileSystem # noqa: F401 +from .implementations.oss import OSSFileSystem +from .implementations.s3 import S3FileSystem +from .implementations.ssh import SSHFileSystem +from .implementations.webdav import WebDAVFileSystem, WebDAVSFileSystem +from .implementations.webhdfs import WebHDFSFileSystem FS_MAP = { Schemes.AZURE: AzureFileSystem, diff --git a/dvc/objects/fs/base.py b/dvc/objects/fs/base.py index a38e9c343..47b528c42 100644 --- a/dvc/objects/fs/base.py +++ b/dvc/objects/fs/base.py @@ -474,7 +474,7 @@ class FileSystem: ) -> None: # Currently, the implementation is non-recursive if the paths are # provided as a list, and recursive if it's a single path. - from .local import localfs + from .implementations.local import localfs def get_file(rpath, lpath, **kwargs): localfs.makedirs(localfs.path.parent(lpath), exist_ok=True) diff --git a/dvc/objects/fs/generic.py b/dvc/objects/fs/generic.py index 3c4b8fbac..6234ec586 100644 --- a/dvc/objects/fs/generic.py +++ b/dvc/objects/fs/generic.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, List, Optional from ._callback import DEFAULT_CALLBACK from .base import RemoteActionNotImplemented -from .local import LocalFileSystem +from .implementations.local import LocalFileSystem from .utils import as_atomic if TYPE_CHECKING: @@ -47,7 +47,7 @@ def copy( return to_fs.put_file(from_path, to_path, callback=callback) if isinstance(to_fs, LocalFileSystem): - from .local import localfs + from .implementations.local import localfs with as_atomic(localfs, to_path, create_parents=True) as tmp_file: return from_fs.get_file(from_path, tmp_file, callback=callback) diff --git a/dvc/objects/fs/implementations/__init__.py b/dvc/objects/fs/implementations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dvc/objects/fs/azure.py b/dvc/objects/fs/implementations/azure.py similarity index 99% rename from dvc/objects/fs/azure.py rename to dvc/objects/fs/implementations/azure.py index 783ebad69..4c9434ea7 100644 --- a/dvc/objects/fs/azure.py +++ b/dvc/objects/fs/implementations/azure.py @@ -10,7 +10,7 @@ from dvc.exceptions import DvcException from dvc.scheme import Schemes from dvc.utils import format_link -from .base import ObjectFileSystem +from ..base import ObjectFileSystem logger = logging.getLogger(__name__) _DEFAULT_CREDS_STEPS = ( diff --git a/dvc/objects/fs/gdrive.py b/dvc/objects/fs/implementations/gdrive.py similarity index 99% rename from dvc/objects/fs/gdrive.py rename to dvc/objects/fs/implementations/gdrive.py index 8f2cec267..e766cfef7 100644 --- a/dvc/objects/fs/gdrive.py +++ b/dvc/objects/fs/implementations/gdrive.py @@ -8,8 +8,8 @@ from dvc.exceptions import DvcException from dvc.scheme import Schemes from dvc.utils import format_link -from .base import FileSystem -from .utils import tmp_fname +from ..base import FileSystem +from ..utils import tmp_fname logger = logging.getLogger(__name__) FOLDER_MIME_TYPE = "application/vnd.google-apps.folder" diff --git a/dvc/objects/fs/git.py b/dvc/objects/fs/implementations/git.py similarity index 97% rename from dvc/objects/fs/git.py rename to dvc/objects/fs/implementations/git.py index 3b20f363d..cb2cd85bb 100644 --- a/dvc/objects/fs/git.py +++ b/dvc/objects/fs/implementations/git.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Any from funcy import cached_property, wrap_prop -from .base import FileSystem +from ..base import FileSystem if TYPE_CHECKING: from scmrepo.fs import GitFileSystem as FsspecGitFileSystem diff --git a/dvc/objects/fs/gs.py b/dvc/objects/fs/implementations/gs.py similarity index 94% rename from dvc/objects/fs/gs.py rename to dvc/objects/fs/implementations/gs.py index e3eef53f1..cbfcc3676 100644 --- a/dvc/objects/fs/gs.py +++ b/dvc/objects/fs/implementations/gs.py @@ -5,7 +5,7 @@ from funcy import cached_property, wrap_prop from dvc.scheme import Schemes # pylint:disable=abstract-method -from .base import ObjectFileSystem +from ..base import ObjectFileSystem class GSFileSystem(ObjectFileSystem): diff --git a/dvc/objects/fs/hdfs.py b/dvc/objects/fs/implementations/hdfs.py similarity index 98% rename from dvc/objects/fs/hdfs.py rename to dvc/objects/fs/implementations/hdfs.py index 163b32691..6cb68a84c 100644 --- a/dvc/objects/fs/hdfs.py +++ b/dvc/objects/fs/implementations/hdfs.py @@ -8,7 +8,7 @@ from funcy import cached_property, wrap_prop from dvc.scheme import Schemes from dvc.utils import fix_env -from .base import FileSystem +from ..base import FileSystem CHECKSUM_REGEX = re.compile(r".*\t.*\t(?P.*)") diff --git a/dvc/objects/fs/http.py b/dvc/objects/fs/implementations/http.py similarity index 98% rename from dvc/objects/fs/http.py rename to dvc/objects/fs/implementations/http.py index c0c9530ad..1ae775f66 100644 --- a/dvc/objects/fs/http.py +++ b/dvc/objects/fs/implementations/http.py @@ -6,8 +6,8 @@ from funcy import cached_property, memoize, wrap_with from dvc import prompt from dvc.scheme import Schemes -from ._callback import DEFAULT_CALLBACK, FsspecCallback -from .base import AnyFSPath, FileSystem +from .._callback import DEFAULT_CALLBACK, FsspecCallback +from ..base import AnyFSPath, FileSystem @wrap_with(threading.Lock()) diff --git a/dvc/objects/fs/https.py b/dvc/objects/fs/implementations/https.py similarity index 100% rename from dvc/objects/fs/https.py rename to dvc/objects/fs/implementations/https.py diff --git a/dvc/objects/fs/local.py b/dvc/objects/fs/implementations/local.py similarity index 97% rename from dvc/objects/fs/local.py rename to dvc/objects/fs/implementations/local.py index 3bdf922ac..0791386ce 100644 --- a/dvc/objects/fs/local.py +++ b/dvc/objects/fs/implementations/local.py @@ -8,9 +8,9 @@ from funcy import cached_property, wrap_prop from dvc.scheme import Schemes -from . import system -from .base import FileSystem -from .utils import copyfile, makedirs, move, remove, tmp_fname +from .. import system +from ..base import FileSystem +from ..utils import copyfile, makedirs, move, remove, tmp_fname logger = logging.getLogger(__name__) @@ -180,7 +180,7 @@ class LocalFileSystem(FileSystem): @cached_property def path(self): - from .path import Path + from ..path import Path return Path(self.sep, getcwd=os.getcwd, realpath=os.path.realpath) diff --git a/dvc/objects/fs/memory.py b/dvc/objects/fs/implementations/memory.py similarity index 95% rename from dvc/objects/fs/memory.py rename to dvc/objects/fs/implementations/memory.py index 1b6130720..a8e1b81d3 100644 --- a/dvc/objects/fs/memory.py +++ b/dvc/objects/fs/implementations/memory.py @@ -4,7 +4,7 @@ from funcy import cached_property, wrap_prop from dvc.scheme import Schemes -from .base import FileSystem +from ..base import FileSystem class MemoryFileSystem(FileSystem): # pylint:disable=abstract-method diff --git a/dvc/objects/fs/oss.py b/dvc/objects/fs/implementations/oss.py similarity index 96% rename from dvc/objects/fs/oss.py rename to dvc/objects/fs/implementations/oss.py index 92cb1caf0..2bd2d5c37 100644 --- a/dvc/objects/fs/oss.py +++ b/dvc/objects/fs/implementations/oss.py @@ -6,7 +6,7 @@ from funcy import cached_property, wrap_prop from dvc.scheme import Schemes -from .base import ObjectFileSystem +from ..base import ObjectFileSystem logger = logging.getLogger(__name__) diff --git a/dvc/objects/fs/s3.py b/dvc/objects/fs/implementations/s3.py similarity index 98% rename from dvc/objects/fs/s3.py rename to dvc/objects/fs/implementations/s3.py index 3a9c2edfa..38d97f3d8 100644 --- a/dvc/objects/fs/s3.py +++ b/dvc/objects/fs/implementations/s3.py @@ -6,7 +6,7 @@ from funcy import cached_property, wrap_prop from dvc.scheme import Schemes -from .base import ObjectFileSystem +from ..base import ObjectFileSystem _AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config") @@ -38,7 +38,7 @@ class S3FileSystem(ObjectFileSystem): from boto3.s3.transfer import TransferConfig - from .utils import human_readable_to_bytes + from ..utils import human_readable_to_bytes config, transfer_config = {}, {} for key, value in s3_config.items(): diff --git a/dvc/objects/fs/ssh.py b/dvc/objects/fs/implementations/ssh.py similarity index 97% rename from dvc/objects/fs/ssh.py rename to dvc/objects/fs/implementations/ssh.py index 6d983a83a..9c4810488 100644 --- a/dvc/objects/fs/ssh.py +++ b/dvc/objects/fs/implementations/ssh.py @@ -7,9 +7,9 @@ from funcy import cached_property, memoize, silent, wrap_prop, wrap_with from dvc import prompt from dvc.scheme import Schemes -from ._callback import DEFAULT_CALLBACK -from .base import FileSystem -from .utils import as_atomic +from .._callback import DEFAULT_CALLBACK +from ..base import FileSystem +from ..utils import as_atomic DEFAULT_PORT = 22 diff --git a/dvc/objects/fs/webdav.py b/dvc/objects/fs/implementations/webdav.py similarity index 98% rename from dvc/objects/fs/webdav.py rename to dvc/objects/fs/implementations/webdav.py index fb643014d..174bbaa74 100644 --- a/dvc/objects/fs/webdav.py +++ b/dvc/objects/fs/implementations/webdav.py @@ -5,7 +5,7 @@ from funcy import cached_property, wrap_prop from dvc.scheme import Schemes -from .base import FileSystem +from ..base import FileSystem from .http import ask_password logger = logging.getLogger(__name__) diff --git a/dvc/objects/fs/webhdfs.py b/dvc/objects/fs/implementations/webhdfs.py similarity index 98% rename from dvc/objects/fs/webhdfs.py rename to dvc/objects/fs/implementations/webhdfs.py index f7661a690..97a7d24c5 100644 --- a/dvc/objects/fs/webhdfs.py +++ b/dvc/objects/fs/implementations/webhdfs.py @@ -5,7 +5,7 @@ from funcy import cached_property, wrap_prop from dvc.scheme import Schemes # pylint:disable=abstract-method -from .base import FileSystem +from ..base import FileSystem class WebHDFSFileSystem(FileSystem): diff --git a/tests/unit/fs/test_ssh.py b/tests/unit/fs/test_ssh.py index e31f3f580..d8ef3c3f3 100644 --- a/tests/unit/fs/test_ssh.py +++ b/tests/unit/fs/test_ssh.py @@ -44,7 +44,9 @@ def test_init(): def test_ssh_ask_password(mocker): - mocker.patch("dvc.objects.fs.ssh.ask_password", return_value="fish") + mocker.patch( + "dvc.objects.fs.implementations.ssh.ask_password", return_value="fish" + ) fs = SSHFileSystem(user="test", host="2.2.2.2", ask_password=True) assert fs.fs_args["password"] == fs.fs_args["passphrase"] == "fish" diff --git a/tests/unit/remote/test_webdav.py b/tests/unit/remote/test_webdav.py index c1b599ce4..d99511396 100644 --- a/tests/unit/remote/test_webdav.py +++ b/tests/unit/remote/test_webdav.py @@ -65,7 +65,7 @@ def test_token(): ) -@patch("dvc.objects.fs.webdav.ask_password") +@patch("dvc.objects.fs.implementations.webdav.ask_password") def test_ask_password(ask_password_mocked): ask_password_mocked.return_value = "pass" host = "host"