dvc.fs: rename fs.utils to fs.generic

In the spirit of fsspec.generic, it is for cross-filesystem operations.
Trying to avoid `fs.utils` as much as possible, but we have some
functions that are really utilities coming, so I am making a place
for them. :)

dvc/data/checkout.py

@@ -4,7 +4,7 @@ from itertools import chain
 from dvc import prompt
 from dvc.exceptions import CacheLinkError, CheckoutError, ConfirmRemoveError
 from dvc.fs._callback import FsspecCallback
-from dvc.fs.utils import test_links, transfer
+from dvc.fs.generic import test_links, transfer
 from dvc.ignore import DvcIgnoreFilter
 from dvc.types import Optional
 

dvc/fs/__init__.py

@@ -1,7 +1,7 @@
 from urllib.parse import urlparse
 
 from ..scheme import Schemes
-from . import utils  # noqa: F401
+from . import generic  # noqa: F401
 from .azure import AzureFileSystem
 from .gdrive import GDriveFileSystem
 from .gs import GSFileSystem

dvc/info.py

@@ -8,7 +8,7 @@ import psutil
 from dvc import __version__
 from dvc.exceptions import NotDvcRepoError
 from dvc.fs import FS_MAP, get_fs_cls, get_fs_config
-from dvc.fs.utils import test_links
+from dvc.fs.generic import test_links
 from dvc.repo import Repo
 from dvc.scm import SCMError
 from dvc.utils import error_link

dvc/objects/db.py

@@ -99,7 +99,7 @@ class ObjectDB:
         from dvc.fs._callback import FsspecCallback
 
         self.makedirs(self.fs.path.parent(to_info))
-        return fs.utils.transfer(
+        return fs.generic.transfer(
             from_fs,
             from_info,
             self.fs,

dvc/stage/cache.py

@@ -217,7 +217,7 @@ class StageCache:
 
         from_fs = from_odb.fs
         to_fs = from_odb.fs
-        func = _log_exceptions(fs.utils.copy)
+        func = _log_exceptions(fs.generic.copy)
         runs = from_fs.path.join(from_odb.fs_path, "runs")
 
         ret = []

tests/func/objects/db/test_index.py

@@ -66,7 +66,7 @@ def test_clear_on_download_err(tmp_dir, dvc, index, mocker):
 
     assert list(index.hashes())
 
-    mocker.patch("dvc.fs.utils.transfer", side_effect=Exception)
+    mocker.patch("dvc.fs.generic.transfer", side_effect=Exception)
     with pytest.raises(DownloadError):
         dvc.pull()
     assert not list(index.hashes())
@@ -83,7 +83,7 @@ def test_partial_upload(tmp_dir, dvc, index, mocker):
             raise Exception("stop baz")
         return original(self, from_file, to_info, name, **kwargs)
 
-    mocker.patch("dvc.fs.utils.transfer", unreliable_upload)
+    mocker.patch("dvc.fs.generic.transfer", unreliable_upload)
     with pytest.raises(UploadError):
         dvc.push()
     assert not list(index.hashes())

tests/func/test_external_repo.py

@@ -47,14 +47,14 @@ def test_source_change(erepo_dir):
 
 
 def test_cache_reused(erepo_dir, mocker, local_cloud):
-    import dvc.fs.utils
+    from dvc import fs
 
     erepo_dir.add_remote(config=local_cloud.config)
     with erepo_dir.chdir():
         erepo_dir.dvc_gen("file", "text", commit="add file")
     erepo_dir.dvc.push()
 
-    download_spy = mocker.spy(dvc.fs.utils, "transfer")
+    download_spy = mocker.spy(fs.generic, "transfer")
 
     # Use URL to prevent any fishy optimizations
     url = f"file://{erepo_dir}"

tests/func/test_import.py

@@ -307,9 +307,9 @@ def test_download_error_pulling_imported_stage(tmp_dir, dvc, erepo_dir):
     remove("foo_imported")
     remove(dst_cache)
 
-    with patch("dvc.fs.utils.transfer", side_effect=Exception), pytest.raises(
-        DownloadError
-    ):
+    with patch(
+        "dvc.fs.generic.transfer", side_effect=Exception
+    ), pytest.raises(DownloadError):
         dvc.pull(["foo_imported.dvc"])
 
 

tests/func/test_remote.py

@@ -166,14 +166,14 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
 
 
 def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
-    import dvc.fs.utils as fs_utils
+    from dvc import fs
 
     foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0]
     bar = tmp_dir.dvc_gen({"bar": "bar content"})[0].outs[0]
     baz = tmp_dir.dvc_gen({"baz": {"foo": "foo content"}})[0].outs[0]
 
     # Faulty upload version, failing on foo
-    original = fs_utils.transfer
+    original = fs.generic.transfer
     odb = dvc.cloud.get_remote_odb("upstream")
 
     def unreliable_upload(from_fs, from_info, to_fs, to_info, **kwargs):
@@ -183,7 +183,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
             raise Exception("stop foo")
         return original(from_fs, from_info, to_fs, to_info, **kwargs)
 
-    with patch.object(fs_utils, "transfer", unreliable_upload):
+    with patch.object(fs.generic, "transfer", unreliable_upload):
         with pytest.raises(UploadError) as upload_error_info:
             dvc.push()
         assert upload_error_info.value.amount == 2
@@ -197,7 +197,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
     remove(dvc.odb.local.cache_dir)
 
     baz._collect_used_dir_cache()
-    with patch.object(fs_utils, "transfer", side_effect=Exception):
+    with patch.object(fs.generic, "transfer", side_effect=Exception):
         with pytest.raises(DownloadError) as download_error_info:
             dvc.pull()
         # error count should be len(.dir + standalone file checksums)
@@ -211,7 +211,7 @@ def test_raise_on_too_many_open_files(
     tmp_dir.dvc_gen({"file": "file content"})
 
     mocker.patch(
-        "dvc.fs.utils.transfer",
+        "dvc.fs.generic.transfer",
         side_effect=OSError(errno.EMFILE, "Too many open files"),
     )
 
@@ -260,12 +260,12 @@ def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory):
 
 
 def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote):
-    import dvc.fs.utils as fs_utils
+    from dvc import fs
 
     foo = tmp_dir.dvc_gen({"foo": {"bar": "bar content"}})[0].outs[0]
     tmp_dir.dvc_gen({"baz": "baz content"})
 
-    mocked_upload = mocker.spy(fs_utils, "transfer")
+    mocked_upload = mocker.spy(fs.generic, "transfer")
     dvc.push()
 
     # foo .dir file should be uploaded after bar