diff --git a/dvc/fs/local.py b/dvc/fs/local.py index aea332299..5d12444dc 100644 --- a/dvc/fs/local.py +++ b/dvc/fs/local.py @@ -5,8 +5,8 @@ import threading from fsspec import AbstractFileSystem from funcy import cached_property, wrap_prop +from dvc.fs.system import System from dvc.scheme import Schemes -from dvc.system import System from dvc.utils import tmp_fname from dvc.utils.fs import copy_fobj_to_file, copyfile, makedirs, move, remove diff --git a/dvc/system.py b/dvc/fs/system.py similarity index 98% rename from dvc/system.py rename to dvc/fs/system.py index 36e351e70..546891d19 100644 --- a/dvc/system.py +++ b/dvc/fs/system.py @@ -86,8 +86,7 @@ class System: @staticmethod def _reflink_linux(src, dst): import fcntl # pylint: disable=import-error - - from funcy import suppress + from contextlib import suppress FICLONE = 0x40049409 diff --git a/dvc/objects/db.py b/dvc/objects/db.py index f567314d0..0d651e83b 100644 --- a/dvc/objects/db.py +++ b/dvc/objects/db.py @@ -310,8 +310,6 @@ class ObjectDB: sense to use a generator to gradually iterate over it, without keeping all of it in memory. """ - from funcy import collecting - num_pages = remote_size / self.fs.LIST_OBJECT_PAGE_SIZE if num_pages < 256 / self.fs.jobs: # Fetching prefixes in parallel requires at least 255 more @@ -330,9 +328,8 @@ class ObjectDB: for i in range(1, pow(16, self.fs.TRAVERSE_PREFIX_LEN - 2)) ] - list_hashes = collecting(self._list_hashes) with ThreadPoolExecutor(max_workers=jobs or self.fs.jobs) as executor: - in_remote = executor.map(list_hashes, traverse_prefixes) + in_remote = executor.map(self._list_hashes, traverse_prefixes) yield from itertools.chain.from_iterable(in_remote) def all(self, jobs=None): diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 8e1b61bdb..f50df9a4e 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -371,10 +371,10 @@ def resolve_paths(repo, out, always_local=False): from urllib.parse import urlparse from dvc.fs.local import localfs + from dvc.fs.system import System from ..dvcfile import DVC_FILE_SUFFIX from ..exceptions import DvcException - from ..system import System from .fs import contains_symlink_up_to abspath = os.path.abspath(out) diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index f7cb65ab6..ba2ab5a1a 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -8,7 +8,7 @@ from contextlib import contextmanager, suppress from typing import TYPE_CHECKING from dvc.exceptions import DvcException -from dvc.system import System +from dvc.fs.system import System from dvc.utils import dict_md5 if TYPE_CHECKING: diff --git a/tests/func/test_add.py b/tests/func/test_add.py index dcd2bed64..51b128e9f 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -22,6 +22,7 @@ from dvc.exceptions import ( RecursiveAddingWhileUsingFilename, ) from dvc.fs.local import LocalFileSystem +from dvc.fs.system import System from dvc.objects.hash_info import HashInfo from dvc.output import ( OutputAlreadyTrackedError, @@ -33,7 +34,6 @@ from dvc.stage.exceptions import ( StageExternalOutputsError, StagePathNotFoundError, ) -from dvc.system import System from dvc.testing.test_workspace import TestAdd from dvc.utils import LARGE_DIR_SIZE, file_md5, relpath from dvc.utils.fs import path_isin diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index 865e8d2e0..99c2fbac1 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -19,9 +19,9 @@ from dvc.exceptions import ( NoOutputOrStageError, ) from dvc.fs.local import LocalFileSystem +from dvc.fs.system import System from dvc.stage import Stage from dvc.stage.exceptions import StageFileDoesNotExistError -from dvc.system import System from dvc.utils import relpath from dvc.utils.fs import remove, walk_files from dvc.utils.serialize import dump_yaml, load_yaml diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 75b0774dc..8798558de 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -5,9 +5,9 @@ import pytest from dvc.cli import main from dvc.data.db import ODBManager +from dvc.fs.system import System from dvc.repo import Repo from dvc.repo.get import GetDVCFileError -from dvc.system import System from dvc.utils.fs import makedirs from tests.unit.fs.test_dvc import make_subrepo diff --git a/tests/func/test_import.py b/tests/func/test_import.py index decf5ba93..42e9ab707 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -10,8 +10,8 @@ from dvc.config import NoRemoteError from dvc.data.db import ODBManager from dvc.dvcfile import Dvcfile from dvc.exceptions import DownloadError, PathMissingError +from dvc.fs.system import System from dvc.stage.exceptions import StagePathNotFoundError -from dvc.system import System from dvc.utils.fs import makedirs, remove from tests.unit.fs.test_dvc import make_subrepo from tests.utils import clean_staging diff --git a/tests/func/test_ls.py b/tests/func/test_ls.py index 569ecd950..9b1fa519f 100644 --- a/tests/func/test_ls.py +++ b/tests/func/test_ls.py @@ -573,7 +573,7 @@ def test_subrepo(dvc_top_level, erepo): def test_broken_symlink(tmp_dir, dvc): - from dvc.system import System + from dvc.fs.system import System tmp_dir.gen("file", "content") System.symlink("file", "link") diff --git a/tests/func/test_remove.py b/tests/func/test_remove.py index e2039d2cd..1150fa96c 100644 --- a/tests/func/test_remove.py +++ b/tests/func/test_remove.py @@ -4,8 +4,8 @@ import pytest from dvc.cli import main from dvc.exceptions import DvcException +from dvc.fs.system import System from dvc.stage.exceptions import StageFileDoesNotExistError -from dvc.system import System from dvc.utils.fs import remove from tests.utils import get_gitignore_content diff --git a/tests/func/test_repo.py b/tests/func/test_repo.py index cbf7cb2cd..595704bca 100644 --- a/tests/func/test_repo.py +++ b/tests/func/test_repo.py @@ -1,6 +1,6 @@ from dvc.data.db import ODBManager from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK -from dvc.system import System +from dvc.fs.system import System def test_destroy(tmp_dir, dvc, run_copy): diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index 4adb7e0fa..b48729dfa 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -15,10 +15,10 @@ from dvc.exceptions import ( StagePathAsOutputError, ) from dvc.fs.local import LocalFileSystem +from dvc.fs.system import System from dvc.output import Output from dvc.stage import Stage from dvc.stage.exceptions import StageFileDoesNotExistError -from dvc.system import System from dvc.utils import file_md5, relpath from dvc.utils.fs import remove from dvc.utils.serialize import dump_yaml, load_yaml diff --git a/tests/func/test_run_single_stage.py b/tests/func/test_run_single_stage.py index a4db9b131..f211fe32c 100644 --- a/tests/func/test_run_single_stage.py +++ b/tests/func/test_run_single_stage.py @@ -19,6 +19,7 @@ from dvc.exceptions import ( OverlappingOutputPathsError, StagePathAsOutputError, ) +from dvc.fs.system import System from dvc.output import Output, OutputIsStageFileError from dvc.repo import Repo as DvcRepo from dvc.stage import Stage @@ -29,7 +30,6 @@ from dvc.stage.exceptions import ( StagePathNotFoundError, StagePathOutsideError, ) -from dvc.system import System from dvc.utils import file_md5 from dvc.utils.serialize import load_yaml from tests.basic_env import TestDvc, TestDvcGit diff --git a/tests/unit/utils/test_fs.py b/tests/unit/utils/test_fs.py index f40faf82b..e07ddcf50 100644 --- a/tests/unit/utils/test_fs.py +++ b/tests/unit/utils/test_fs.py @@ -5,7 +5,7 @@ import pytest import dvc from dvc.fs.local import LocalFileSystem -from dvc.system import System +from dvc.fs.system import System from dvc.utils import relpath from dvc.utils.fs import ( BasePathNotInCheckedPathException,