diff --git a/dvc/data/checkout.py b/dvc/data/checkout.py index df65cb271..cf0c9e676 100644 --- a/dvc/data/checkout.py +++ b/dvc/data/checkout.py @@ -4,7 +4,7 @@ from itertools import chain from dvc import prompt from dvc.exceptions import CacheLinkError, CheckoutError, ConfirmRemoveError from dvc.fs._callback import FsspecCallback -from dvc.fs.utils import test_links, transfer +from dvc.fs.generic import test_links, transfer from dvc.ignore import DvcIgnoreFilter from dvc.types import Optional diff --git a/dvc/fs/__init__.py b/dvc/fs/__init__.py index 6d289f2cf..a8b56bac1 100644 --- a/dvc/fs/__init__.py +++ b/dvc/fs/__init__.py @@ -1,7 +1,7 @@ from urllib.parse import urlparse from ..scheme import Schemes -from . import utils # noqa: F401 +from . import generic # noqa: F401 from .azure import AzureFileSystem from .gdrive import GDriveFileSystem from .gs import GSFileSystem diff --git a/dvc/fs/utils.py b/dvc/fs/generic.py similarity index 100% rename from dvc/fs/utils.py rename to dvc/fs/generic.py diff --git a/dvc/info.py b/dvc/info.py index 4c4b9af0e..37e9a99f8 100644 --- a/dvc/info.py +++ b/dvc/info.py @@ -8,7 +8,7 @@ import psutil from dvc import __version__ from dvc.exceptions import NotDvcRepoError from dvc.fs import FS_MAP, get_fs_cls, get_fs_config -from dvc.fs.utils import test_links +from dvc.fs.generic import test_links from dvc.repo import Repo from dvc.scm import SCMError from dvc.utils import error_link diff --git a/dvc/objects/db.py b/dvc/objects/db.py index f2f644735..f567314d0 100644 --- a/dvc/objects/db.py +++ b/dvc/objects/db.py @@ -99,7 +99,7 @@ class ObjectDB: from dvc.fs._callback import FsspecCallback self.makedirs(self.fs.path.parent(to_info)) - return fs.utils.transfer( + return fs.generic.transfer( from_fs, from_info, self.fs, diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index c2a32050c..6d2131aa7 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -217,7 +217,7 @@ class StageCache: from_fs = from_odb.fs to_fs = from_odb.fs - func = _log_exceptions(fs.utils.copy) + func = _log_exceptions(fs.generic.copy) runs = from_fs.path.join(from_odb.fs_path, "runs") ret = [] diff --git a/tests/func/objects/db/test_index.py b/tests/func/objects/db/test_index.py index 4aa6d3ca3..af463a763 100644 --- a/tests/func/objects/db/test_index.py +++ b/tests/func/objects/db/test_index.py @@ -66,7 +66,7 @@ def test_clear_on_download_err(tmp_dir, dvc, index, mocker): assert list(index.hashes()) - mocker.patch("dvc.fs.utils.transfer", side_effect=Exception) + mocker.patch("dvc.fs.generic.transfer", side_effect=Exception) with pytest.raises(DownloadError): dvc.pull() assert not list(index.hashes()) @@ -83,7 +83,7 @@ def test_partial_upload(tmp_dir, dvc, index, mocker): raise Exception("stop baz") return original(self, from_file, to_info, name, **kwargs) - mocker.patch("dvc.fs.utils.transfer", unreliable_upload) + mocker.patch("dvc.fs.generic.transfer", unreliable_upload) with pytest.raises(UploadError): dvc.push() assert not list(index.hashes()) diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index 479cf219a..df49696dc 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -47,14 +47,14 @@ def test_source_change(erepo_dir): def test_cache_reused(erepo_dir, mocker, local_cloud): - import dvc.fs.utils + from dvc import fs erepo_dir.add_remote(config=local_cloud.config) with erepo_dir.chdir(): erepo_dir.dvc_gen("file", "text", commit="add file") erepo_dir.dvc.push() - download_spy = mocker.spy(dvc.fs.utils, "transfer") + download_spy = mocker.spy(fs.generic, "transfer") # Use URL to prevent any fishy optimizations url = f"file://{erepo_dir}" diff --git a/tests/func/test_import.py b/tests/func/test_import.py index 5cc8e2c71..decf5ba93 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -307,9 +307,9 @@ def test_download_error_pulling_imported_stage(tmp_dir, dvc, erepo_dir): remove("foo_imported") remove(dst_cache) - with patch("dvc.fs.utils.transfer", side_effect=Exception), pytest.raises( - DownloadError - ): + with patch( + "dvc.fs.generic.transfer", side_effect=Exception + ), pytest.raises(DownloadError): dvc.pull(["foo_imported.dvc"]) diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index dbfe080f9..73a2db87d 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -166,14 +166,14 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote): - import dvc.fs.utils as fs_utils + from dvc import fs foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0] bar = tmp_dir.dvc_gen({"bar": "bar content"})[0].outs[0] baz = tmp_dir.dvc_gen({"baz": {"foo": "foo content"}})[0].outs[0] # Faulty upload version, failing on foo - original = fs_utils.transfer + original = fs.generic.transfer odb = dvc.cloud.get_remote_odb("upstream") def unreliable_upload(from_fs, from_info, to_fs, to_info, **kwargs): @@ -183,7 +183,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote): raise Exception("stop foo") return original(from_fs, from_info, to_fs, to_info, **kwargs) - with patch.object(fs_utils, "transfer", unreliable_upload): + with patch.object(fs.generic, "transfer", unreliable_upload): with pytest.raises(UploadError) as upload_error_info: dvc.push() assert upload_error_info.value.amount == 2 @@ -197,7 +197,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote): remove(dvc.odb.local.cache_dir) baz._collect_used_dir_cache() - with patch.object(fs_utils, "transfer", side_effect=Exception): + with patch.object(fs.generic, "transfer", side_effect=Exception): with pytest.raises(DownloadError) as download_error_info: dvc.pull() # error count should be len(.dir + standalone file checksums) @@ -211,7 +211,7 @@ def test_raise_on_too_many_open_files( tmp_dir.dvc_gen({"file": "file content"}) mocker.patch( - "dvc.fs.utils.transfer", + "dvc.fs.generic.transfer", side_effect=OSError(errno.EMFILE, "Too many open files"), ) @@ -260,12 +260,12 @@ def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory): def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote): - import dvc.fs.utils as fs_utils + from dvc import fs foo = tmp_dir.dvc_gen({"foo": {"bar": "bar content"}})[0].outs[0] tmp_dir.dvc_gen({"baz": "baz content"}) - mocked_upload = mocker.spy(fs_utils, "transfer") + mocked_upload = mocker.spy(fs.generic, "transfer") dvc.push() # foo .dir file should be uploaded after bar