diff --git a/dvc/objects/__init__.py b/dvc/data/__init__.py similarity index 89% rename from dvc/objects/__init__.py rename to dvc/data/__init__.py index ad45ba490..9a00f3737 100644 --- a/dvc/objects/__init__.py +++ b/dvc/data/__init__.py @@ -5,9 +5,8 @@ from .tree import Tree if TYPE_CHECKING: from dvc.hash_info import HashInfo - - from .db.base import ObjectDB - from .file import HashFile + from dvc.objects.db import ObjectDB + from dvc.objects.file import HashFile logger = logging.getLogger(__name__) diff --git a/dvc/objects/checkout.py b/dvc/data/checkout.py similarity index 96% rename from dvc/objects/checkout.py rename to dvc/data/checkout.py index 6c700eede..3097ffd77 100644 --- a/dvc/objects/checkout.py +++ b/dvc/data/checkout.py @@ -5,14 +5,13 @@ from dvc import prompt from dvc.exceptions import CacheLinkError, CheckoutError, ConfirmRemoveError from dvc.fs.utils import test_links, transfer from dvc.ignore import DvcIgnoreFilter -from dvc.objects.db.slow_link_detection import ( # type: ignore[attr-defined] - slow_link_guard, -) -from dvc.objects.diff import ROOT -from dvc.objects.diff import diff as odiff -from dvc.objects.stage import stage from dvc.types import Optional +from .diff import ROOT +from .diff import diff as odiff +from .slow_link_detection import slow_link_guard # type: ignore[attr-defined] +from .stage import stage + logger = logging.getLogger(__name__) diff --git a/dvc/objects/db/__init__.py b/dvc/data/db/__init__.py similarity index 98% rename from dvc/objects/db/__init__.py rename to dvc/data/db/__init__.py index 4890ae144..1210cdf10 100644 --- a/dvc/objects/db/__init__.py +++ b/dvc/data/db/__init__.py @@ -7,7 +7,8 @@ if TYPE_CHECKING: def get_odb(fs, fs_path, **config): - from .base import ObjectDB + from dvc.objects.db import ObjectDB + from .local import LocalObjectDB if fs.scheme == Schemes.LOCAL: diff --git a/dvc/objects/db/index.py b/dvc/data/db/index.py similarity index 98% rename from dvc/objects/db/index.py rename to dvc/data/db/index.py index 2b65f07b7..ee02746b2 100644 --- a/dvc/objects/db/index.py +++ b/dvc/data/db/index.py @@ -3,7 +3,7 @@ import os from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Iterable, Set -from ..errors import ObjectDBError +from dvc.objects.errors import ObjectDBError if TYPE_CHECKING: from dvc.types import StrPath diff --git a/dvc/objects/db/local.py b/dvc/data/db/local.py similarity index 99% rename from dvc/objects/db/local.py rename to dvc/data/db/local.py index 2c737b226..f7cb63954 100644 --- a/dvc/objects/db/local.py +++ b/dvc/data/db/local.py @@ -6,13 +6,12 @@ from funcy import cached_property from shortuuid import uuid from dvc.hash_info import HashInfo +from dvc.objects.db import ObjectDB from dvc.objects.errors import ObjectFormatError from dvc.progress import Tqdm from dvc.utils import relpath from dvc.utils.fs import copyfile, remove, umask, walk_files -from .base import ObjectDB - logger = logging.getLogger(__name__) diff --git a/dvc/objects/db/reference.py b/dvc/data/db/reference.py similarity index 96% rename from dvc/objects/db/reference.py rename to dvc/data/db/reference.py index 517c46c76..b9fde6be0 100644 --- a/dvc/objects/db/reference.py +++ b/dvc/data/db/reference.py @@ -3,11 +3,11 @@ import logging import os from typing import TYPE_CHECKING, Dict +from dvc.objects.db import ObjectDB +from dvc.objects.errors import ObjectFormatError from dvc.scheme import Schemes -from ..errors import ObjectFormatError from ..reference import ReferenceHashFile -from .base import ObjectDB if TYPE_CHECKING: from dvc.fs.base import FileSystem diff --git a/dvc/objects/diff.py b/dvc/data/diff.py similarity index 96% rename from dvc/objects/diff.py rename to dvc/data/diff.py index 52ee299b7..d20212e7c 100644 --- a/dvc/objects/diff.py +++ b/dvc/data/diff.py @@ -3,8 +3,7 @@ from typing import TYPE_CHECKING, List, Optional, Tuple if TYPE_CHECKING: from dvc.hash_info import HashInfo - - from .file import HashFile + from dvc.objects.file import HashFile ADD = "add" MODIFY = "modify" @@ -96,7 +95,7 @@ def diff( return entry_obj.hash_info if entry_obj else None def _in_cache(oid, cache): - from .errors import ObjectFormatError + from dvc.objects.errors import ObjectFormatError if not oid: return False diff --git a/dvc/objects/meta.py b/dvc/data/meta.py similarity index 100% rename from dvc/objects/meta.py rename to dvc/data/meta.py diff --git a/dvc/objects/reference.py b/dvc/data/reference.py similarity index 96% rename from dvc/objects/reference.py rename to dvc/data/reference.py index 7fe7c414d..a3438e390 100644 --- a/dvc/objects/reference.py +++ b/dvc/data/reference.py @@ -4,16 +4,15 @@ import os import pickle from typing import TYPE_CHECKING, Optional -from .errors import ObjectFormatError -from .file import HashFile +from dvc.objects.errors import ObjectFormatError +from dvc.objects.file import HashFile if TYPE_CHECKING: from dvc.fs.base import FileSystem from dvc.hash_info import HashInfo + from dvc.objects.db import ObjectDB from dvc.types import AnyPath - from .db.base import ObjectDB - logger = logging.getLogger(__name__) diff --git a/dvc/objects/db/slow_link_detection.py b/dvc/data/slow_link_detection.py similarity index 100% rename from dvc/objects/db/slow_link_detection.py rename to dvc/data/slow_link_detection.py diff --git a/dvc/objects/stage.py b/dvc/data/stage.py similarity index 98% rename from dvc/objects/stage.py rename to dvc/data/stage.py index d487af53d..a34ce6250 100644 --- a/dvc/objects/stage.py +++ b/dvc/data/stage.py @@ -8,19 +8,18 @@ from typing import TYPE_CHECKING, Dict, Optional, Tuple from dvc.exceptions import DvcIgnoreInCollectedDirError from dvc.hash_info import HashInfo from dvc.ignore import DvcIgnore +from dvc.objects.file import HashFile from dvc.progress import Tqdm from dvc.utils import file_md5 from .db.reference import ReferenceObjectDB -from .file import HashFile from .meta import Meta if TYPE_CHECKING: from dvc.fs.base import FileSystem + from dvc.objects.db import ObjectDB from dvc.types import AnyPath - from .db.base import ObjectDB - logger = logging.getLogger(__name__) @@ -245,8 +244,9 @@ def _get_staging(odb: "ObjectDB") -> "ObjectDB": def _load_from_state(odb, staging, fs_path, fs, name): + from dvc.objects.errors import ObjectFormatError + from . import check, load - from .errors import ObjectFormatError from .tree import Tree state = odb.state diff --git a/dvc/objects/status.py b/dvc/data/status.py similarity index 98% rename from dvc/objects/status.py rename to dvc/data/status.py index 947d841a7..400b6f163 100644 --- a/dvc/objects/status.py +++ b/dvc/data/status.py @@ -7,9 +7,10 @@ from dvc.scheme import Schemes from .tree import Tree if TYPE_CHECKING: - from .db.base import ObjectDB + from dvc.objects.db import ObjectDB + from dvc.objects.file import HashFile + from .db.index import ObjectDBIndexBase - from .file import HashFile logger = logging.getLogger(__name__) diff --git a/dvc/objects/transfer.py b/dvc/data/transfer.py similarity index 98% rename from dvc/objects/transfer.py rename to dvc/data/transfer.py index de1f9d02c..dbb6cc6df 100644 --- a/dvc/objects/transfer.py +++ b/dvc/data/transfer.py @@ -10,8 +10,8 @@ from dvc.utils.threadpool import ThreadPoolExecutor if TYPE_CHECKING: from dvc.hash_info import HashInfo + from dvc.objects.db import ObjectDB - from .db.base import ObjectDB from .db.index import ObjectDBIndexBase from .tree import Tree @@ -40,7 +40,8 @@ def _log_exceptions(func): def find_tree_by_obj_id( odbs: Iterable[Optional["ObjectDB"]], obj_id: "HashInfo" ) -> Optional["Tree"]: - from .errors import ObjectFormatError + from dvc.objects.errors import ObjectFormatError + from .tree import Tree for odb in odbs: diff --git a/dvc/objects/tree.py b/dvc/data/tree.py similarity index 98% rename from dvc/objects/tree.py rename to dvc/data/tree.py index f985ddb34..02114a5e6 100644 --- a/dvc/objects/tree.py +++ b/dvc/data/tree.py @@ -5,8 +5,9 @@ from typing import TYPE_CHECKING, Dict, Optional, Tuple from funcy import cached_property -from .errors import ObjectFormatError -from .file import HashFile +from dvc.objects.errors import ObjectFormatError +from dvc.objects.file import HashFile + from .stage import get_file_hash if TYPE_CHECKING: diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index b3176ff34..03a258547 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -3,11 +3,11 @@ import logging from typing import TYPE_CHECKING, Iterable, Optional -from dvc.objects.db import get_index +from dvc.data.db import get_index if TYPE_CHECKING: from dvc.hash_info import HashInfo - from dvc.objects.db.base import ObjectDB + from dvc.objects.db import ObjectDB logger = logging.getLogger(__name__) @@ -55,8 +55,8 @@ class DataCloud: raise NoRemoteError(error_msg) def _init_odb(self, name): + from dvc.data.db import get_odb from dvc.fs import get_cloud_fs - from dvc.objects.db import get_odb cls, config, fs_path = get_cloud_fs(self.repo, name=name) config["tmp_dir"] = self.repo.index_db_dir @@ -78,7 +78,7 @@ class DataCloud: By default remote from core.remote config option is used. odb: optional ODB to push to. Overrides remote. """ - from dvc.objects.transfer import transfer + from dvc.data.transfer import transfer if not odb: odb = self.get_remote_odb(remote, "push") @@ -107,7 +107,7 @@ class DataCloud: By default remote from core.remote config option is used. odb: optional ODB to pull from. Overrides remote. """ - from dvc.objects.transfer import transfer + from dvc.data.transfer import transfer if not odb: odb = self.get_remote_odb(remote, "pull") @@ -141,7 +141,7 @@ class DataCloud: log_missing: log warning messages if file doesn't exist neither in cache, neither in cloud. """ - from dvc.objects.status import compare_status + from dvc.data.status import compare_status if not odb: odb = self.get_remote_odb(remote, "status") diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 172e20584..87fd38663 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -9,7 +9,7 @@ from .base import Dependency if TYPE_CHECKING: from dvc.hash_info import HashInfo - from dvc.objects.db.base import ObjectDB + from dvc.objects.db import ObjectDB from dvc.objects.file import HashFile @@ -61,7 +61,7 @@ class RepoDependency(Dependency): return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo} def download(self, to, jobs=None): - from dvc.objects.checkout import checkout + from dvc.data.checkout import checkout for odb, objs in self.get_used_objs().items(): self.repo.cloud.pull(objs, jobs=jobs, odb=odb) @@ -98,9 +98,9 @@ class RepoDependency(Dependency): self, obj_only=False, **kwargs ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "HashFile"]: from dvc.config import NoRemoteError + from dvc.data.stage import stage + from dvc.data.tree import Tree from dvc.exceptions import NoOutputOrStageError, PathMissingError - from dvc.objects.stage import stage - from dvc.objects.tree import Tree local_odb = self.repo.odb.local locked = kwargs.pop("locked", True) @@ -150,10 +150,10 @@ class RepoDependency(Dependency): return used_obj_ids, staged_obj def _check_circular_import(self, odb, obj_ids): + from dvc.data.db.reference import ReferenceObjectDB + from dvc.data.tree import Tree from dvc.exceptions import CircularImportError from dvc.fs.repo import RepoFileSystem - from dvc.objects.db.reference import ReferenceObjectDB - from dvc.objects.tree import Tree if not isinstance(odb, ReferenceObjectDB): return diff --git a/dvc/objects/db/base.py b/dvc/objects/db.py similarity index 99% rename from dvc/objects/db/base.py rename to dvc/objects/db.py index 485a41274..97b2dc7a2 100644 --- a/dvc/objects/db/base.py +++ b/dvc/objects/db.py @@ -357,7 +357,7 @@ class ObjectDB: pass def gc(self, used, jobs=None, cache_odb=None, shallow=True): - from ..tree import Tree + from dvc.data.tree import Tree if self.read_only: raise ObjectDBPermissionError("Cannot gc read-only ODB") diff --git a/dvc/objects/file.py b/dvc/objects/file.py index b4435ba37..eeeb45aaa 100644 --- a/dvc/objects/file.py +++ b/dvc/objects/file.py @@ -3,14 +3,12 @@ import logging import os from typing import TYPE_CHECKING, Optional -from .errors import ObjectFormatError - if TYPE_CHECKING: from dvc.fs.base import FileSystem from dvc.hash_info import HashInfo from dvc.types import AnyPath - from .db.base import ObjectDB + from .db import ObjectDB logger = logging.getLogger(__name__) @@ -68,7 +66,8 @@ class HashFile: self._check_hash(odb) def _check_hash(self, odb): - from .stage import get_file_hash + from dvc.data.stage import get_file_hash + from dvc.objects.errors import ObjectFormatError _, actual = get_file_hash( self.fs_path, self.fs, self.hash_info.name, odb.state diff --git a/dvc/output.py b/dvc/output.py index f3d954448..bc3c0964c 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -7,7 +7,7 @@ from urllib.parse import urlparse from funcy import collecting, project from voluptuous import And, Any, Coerce, Length, Lower, Required, SetTo -from dvc import objects, prompt +from dvc import prompt from dvc.exceptions import ( CheckoutError, CollectCacheError, @@ -15,25 +15,27 @@ from dvc.exceptions import ( MergeError, RemoteCacheRequiredError, ) -from dvc.objects.checkout import checkout +from .data import check as ocheck +from .data import load as oload +from .data.checkout import checkout +from .data.meta import Meta +from .data.stage import stage as ostage +from .data.transfer import transfer as otransfer +from .data.tree import Tree from .fs import get_cloud_fs from .fs.hdfs import HDFSFileSystem from .fs.local import LocalFileSystem from .fs.s3 import S3FileSystem from .hash_info import HashInfo from .istextfile import istextfile -from .objects import Tree from .objects.errors import ObjectFormatError -from .objects.meta import Meta -from .objects.stage import stage as ostage -from .objects.transfer import transfer as otransfer from .scheme import Schemes from .utils import relpath from .utils.fs import path_isin if TYPE_CHECKING: - from .objects.db.base import ObjectDB + from .objects.db import ObjectDB logger = logging.getLogger(__name__) @@ -456,7 +458,7 @@ class Output: return True try: - objects.check(self.odb, obj) + ocheck(self.odb, obj) return False except (FileNotFoundError, ObjectFormatError): return True @@ -710,7 +712,7 @@ class Output: obj = self.obj elif self.hash_info: try: - obj = objects.load(self.odb, self.hash_info) + obj = oload(self.odb, self.hash_info) except FileNotFoundError: return None else: @@ -857,7 +859,7 @@ class Output: obj = self.odb.get(self.hash_info) try: - objects.check(self.odb, obj) + ocheck(self.odb, obj) except FileNotFoundError: if self.remote: kwargs["remote"] = self.remote @@ -867,7 +869,7 @@ class Output: return self.obj try: - self.obj = objects.load(self.odb, self.hash_info) + self.obj = oload(self.odb, self.hash_info) except (FileNotFoundError, ObjectFormatError): self.obj = None @@ -884,7 +886,7 @@ class Output: logger.debug(f"failed to pull cache for '{self}'") try: - objects.check(self.odb, self.odb.get(self.hash_info)) + ocheck(self.odb, self.odb.get(self.hash_info)) except FileNotFoundError: msg = ( "Missing cache for directory '{}'. " @@ -1012,7 +1014,7 @@ class Output: ) def merge(self, ancestor, other): - from dvc.objects.tree import du, merge + from dvc.data.tree import du, merge assert other diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index c6f02688b..a79627373 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -150,11 +150,11 @@ class Repo: repo_factory=None, ): from dvc.config import Config + from dvc.data.db import ODBManager from dvc.data_cloud import DataCloud from dvc.fs.git import GitFileSystem from dvc.fs.local import localfs from dvc.lock import LockNoop, make_lock - from dvc.objects.db import ODBManager from dvc.repo.live import Live from dvc.repo.metrics import Metrics from dvc.repo.params import Params @@ -406,7 +406,7 @@ class Repo: def _add_suffix(objs: Set["HashFile"], suffix: str) -> None: from itertools import chain - from dvc.objects import iterobjs + from dvc.data import iterobjs for obj in chain.from_iterable(map(iterobjs, objs)): if obj.name is not None: diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index f2295aeeb..8689d2b3b 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -113,8 +113,8 @@ def _paths_checksums(repo, targets): def _output_paths(repo, targets): + from dvc.data.stage import stage as ostage from dvc.fs.local import LocalFileSystem - from dvc.objects.stage import stage as ostage on_working_fs = isinstance(repo.fs, LocalFileSystem) diff --git a/dvc/repo/experiments/executor/ssh.py b/dvc/repo/experiments/executor/ssh.py index 5aafb954c..3b1df9503 100644 --- a/dvc/repo/experiments/executor/ssh.py +++ b/dvc/repo/experiments/executor/ssh.py @@ -156,7 +156,7 @@ class SSHExecutor(BaseExecutor): return sshfs.fs.execute(f"cd {working_dir};{cmd}", **kwargs) def init_cache(self, dvc: "Repo", rev: str, run_cache: bool = True): - from dvc.objects.db import ODBManager, get_odb + from dvc.data.db import ODBManager, get_odb from dvc.repo import Repo from dvc.repo.push import push diff --git a/dvc/repo/gc.py b/dvc/repo/gc.py index 08717658b..549f315bb 100644 --- a/dvc/repo/gc.py +++ b/dvc/repo/gc.py @@ -44,7 +44,7 @@ def gc( from contextlib import ExitStack - from dvc.objects.db import get_index + from dvc.data.db import get_index from dvc.repo import Repo if not repos: diff --git a/dvc/schema.py b/dvc/schema.py index 0f5fcb1d7..5f437374f 100644 --- a/dvc/schema.py +++ b/dvc/schema.py @@ -3,7 +3,7 @@ from collections.abc import Mapping from voluptuous import Any, Optional, Required, Schema from dvc import dependency, output -from dvc.objects.meta import Meta +from dvc.data.meta import Meta from dvc.output import CHECKSUMS_SCHEMA, Output from dvc.parsing import DO_KWD, FOREACH_KWD, VARS_KWD from dvc.parsing.versions import SCHEMA_KWD, lockfile_version_schema diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index 5c58d4453..04e225f93 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -21,7 +21,7 @@ class RunCacheNotFoundError(DvcException): def _get_cache_hash(cache, key=False): - from dvc.objects.meta import Meta + from dvc.data.meta import Meta if key: cache["outs"] = [out["path"] for out in cache.get("outs", [])] @@ -236,7 +236,7 @@ class StageCache: return ret def push(self, remote: Optional[str], odb: Optional["ObjectDB"] = None): - from dvc.objects.transfer import _log_exceptions + from dvc.data.transfer import _log_exceptions if odb is None: odb = self.repo.cloud.get_remote_odb(remote) @@ -247,7 +247,7 @@ class StageCache: ) def pull(self, remote: Optional[str]): - from dvc.objects.transfer import _log_exceptions + from dvc.data.transfer import _log_exceptions odb = self.repo.cloud.get_remote_odb(remote) return self._transfer( diff --git a/dvc/stage/loader.py b/dvc/stage/loader.py index 5f263fb9e..f5959c35c 100644 --- a/dvc/stage/loader.py +++ b/dvc/stage/loader.py @@ -6,8 +6,8 @@ from itertools import chain from funcy import cached_property, get_in, lcat, once, project from dvc import dependency, output +from dvc.data.meta import Meta from dvc.hash_info import HashInfo -from dvc.objects.meta import Meta from dvc.parsing import FOREACH_KWD, JOIN, DataResolver, EntryNotFound from dvc.parsing.versions import LOCKFILE_VERSION diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index d869ce8f6..fbba5fb28 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -4,8 +4,8 @@ from typing import TYPE_CHECKING, Union from funcy import concat, first, lsplit, rpartial, without +from dvc.data.meta import Meta from dvc.exceptions import InvalidArgumentError -from dvc.objects.meta import Meta from .exceptions import ( MissingDataSource, diff --git a/dvc/state.py b/dvc/state.py index 03145c736..6f8c86d90 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -103,7 +103,7 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes HashInfo or None: hash for the specified path info or None if it doesn't exist in the state database. """ - from .objects.meta import Meta + from .data.meta import Meta if not isinstance(fs, LocalFileSystem): return None, None diff --git a/dvc/testing/fixtures.py b/dvc/testing/fixtures.py index b3bedacce..3c9c541f0 100644 --- a/dvc/testing/fixtures.py +++ b/dvc/testing/fixtures.py @@ -149,7 +149,7 @@ def local_remote(make_remote): @pytest.fixture def make_workspace(tmp_dir, dvc, make_cloud): def _make_workspace(name, typ="local"): - from dvc.objects.db import ODBManager + from dvc.data.db import ODBManager cloud = make_cloud(typ) # pylint: disable=W0621 diff --git a/dvc/testing/test_workspace.py b/dvc/testing/test_workspace.py index 0151bc76f..f6225a2fb 100644 --- a/dvc/testing/test_workspace.py +++ b/dvc/testing/test_workspace.py @@ -20,7 +20,7 @@ class TestImport: pytest.skip() def test_import_dir(self, tmp_dir, dvc, workspace, stage_md5, dir_md5): - from dvc.objects.db import ODBManager + from dvc.data.db import ODBManager workspace.gen( {"dir": {"file": "file", "subdir": {"subfile": "subfile"}}} diff --git a/tests/func/objects/db/test_index.py b/tests/func/objects/db/test_index.py index 651dc781f..fd41aafc3 100644 --- a/tests/func/objects/db/test_index.py +++ b/tests/func/objects/db/test_index.py @@ -1,8 +1,8 @@ import pytest +from dvc.data.db import get_index from dvc.exceptions import DownloadError, UploadError from dvc.fs.local import LocalFileSystem -from dvc.objects.db import get_index from dvc.utils.fs import remove from tests.utils import clean_staging diff --git a/tests/func/test_add.py b/tests/func/test_add.py index deb4cedbb..5b7af80b0 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -11,6 +11,7 @@ import colorama import pytest import dvc as dvc_module +from dvc.data.db import ODBManager from dvc.dvcfile import DVC_FILE_SUFFIX from dvc.exceptions import ( DvcException, @@ -22,7 +23,6 @@ from dvc.exceptions import ( from dvc.fs.local import LocalFileSystem from dvc.hash_info import HashInfo from dvc.main import main -from dvc.objects.db import ODBManager from dvc.output import ( OutputAlreadyTrackedError, OutputDoesNotExistError, @@ -102,7 +102,7 @@ def test_add_unsupported_file(dvc): def test_add_directory(tmp_dir, dvc): - from dvc.objects import load + from dvc.data import load (stage,) = tmp_dir.dvc_gen({"dir": {"file": "file"}}) @@ -378,7 +378,7 @@ class TestDoubleAddUnchanged(TestDvc): def test_should_update_state_entry_for_file_after_add(mocker, dvc, tmp_dir): - file_md5_counter = mocker.spy(dvc_module.objects.stage, "file_md5") + file_md5_counter = mocker.spy(dvc_module.data.stage, "file_md5") tmp_dir.gen("foo", "foo") ret = main(["config", "cache.type", "copy"]) @@ -409,7 +409,7 @@ def test_should_update_state_entry_for_file_after_add(mocker, dvc, tmp_dir): def test_should_update_state_entry_for_directory_after_add( mocker, dvc, tmp_dir ): - file_md5_counter = mocker.spy(dvc_module.objects.stage, "file_md5") + file_md5_counter = mocker.spy(dvc_module.data.stage, "file_md5") tmp_dir.gen({"data/data": "foo", "data/data_sub/sub_data": "foo"}) @@ -456,7 +456,7 @@ class TestAddCommit(TestDvc): def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc): tmp_dir.gen({"data/data": "foo"}) - counter = mocker.spy(dvc_module.objects.stage, "_stage_tree") + counter = mocker.spy(dvc_module.data.stage, "_stage_tree") ret = main(["add", "data"]) assert ret == 0 assert counter.mock.call_count == 1 @@ -880,7 +880,7 @@ def test_add_with_cache_link_error(tmp_dir, dvc, mocker, capsys): tmp_dir.gen("foo", "foo") mocker.patch( - "dvc.objects.checkout.test_links", + "dvc.data.checkout.test_links", return_value=[], ) dvc.add("foo") diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index ffd30232e..508a29d6b 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -85,7 +85,7 @@ class TestCheckoutCorruptedCacheFile(TestRepro): class TestCheckoutCorruptedCacheDir(TestDvc): def test(self): - from dvc.objects import load + from dvc.data import load # NOTE: using 'copy' so that cache and link don't have same inode ret = main(["config", "cache.type", "copy"]) diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 08311ef8f..3372332b1 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -107,7 +107,7 @@ def test_warn_on_outdated_stage(tmp_dir, dvc, local_remote, caplog): def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote): tmp_dir.gen({"foo": "foo"}) - test_file_md5 = mocker.spy(dvc_module.objects.stage, "file_md5") + test_file_md5 = mocker.spy(dvc_module.data.stage, "file_md5") ret = main(["config", "cache.type", "hardlink"]) assert ret == 0 ret = main(["add", "foo"]) @@ -169,7 +169,7 @@ def test_verify_hashes( remove("dir") remove(dvc.odb.local.cache_dir) - hash_spy = mocker.spy(dvc_module.objects.stage, "file_md5") + hash_spy = mocker.spy(dvc_module.data.stage, "file_md5") dvc.pull() assert hash_spy.call_count == 0 diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index c1a267979..068a20c1f 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -3,9 +3,9 @@ from unittest.mock import ANY, patch from scmrepo.git import Git +from dvc.data.stage import stage +from dvc.data.transfer import transfer from dvc.external_repo import CLONES, external_repo -from dvc.objects.stage import stage -from dvc.objects.transfer import transfer from dvc.utils import relpath from dvc.utils.fs import makedirs, remove from tests.unit.fs.test_repo import make_subrepo diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index 1e0885f45..680651666 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -6,10 +6,10 @@ import configobj import pytest from git import Repo +from dvc.data.db.local import LocalObjectDB from dvc.exceptions import CollectCacheError from dvc.fs.local import LocalFileSystem from dvc.main import main -from dvc.objects.db.local import LocalObjectDB from dvc.repo import Repo as DvcRepo from dvc.utils.fs import remove from tests.basic_env import TestDir, TestDvcGit diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 36a1d6dc1..edc30e9b2 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -3,8 +3,8 @@ import os import pytest +from dvc.data.db import ODBManager from dvc.main import main -from dvc.objects.db import ODBManager from dvc.repo import Repo from dvc.repo.get import GetDVCFileError from dvc.system import System diff --git a/tests/func/test_import.py b/tests/func/test_import.py index 0bc6f8b5b..e8f34f359 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -7,9 +7,9 @@ from funcy import first from scmrepo.git import Git from dvc.config import NoRemoteError +from dvc.data.db import ODBManager from dvc.dvcfile import Dvcfile from dvc.exceptions import DownloadError, PathMissingError -from dvc.objects.db import ODBManager from dvc.stage.exceptions import StagePathNotFoundError from dvc.system import System from dvc.utils.fs import makedirs, remove @@ -529,7 +529,7 @@ def test_import_with_no_exec(tmp_dir, dvc, erepo_dir): def test_import_with_jobs(mocker, dvc, erepo_dir): - import dvc.objects.transfer as otransfer + import dvc.data.transfer as otransfer with erepo_dir.chdir(): erepo_dir.dvc_gen( diff --git a/tests/func/test_odb.py b/tests/func/test_odb.py index a1206db85..5c3d3674f 100644 --- a/tests/func/test_odb.py +++ b/tests/func/test_odb.py @@ -4,9 +4,9 @@ import stat import configobj import pytest +from dvc.data.db import ODBManager from dvc.hash_info import HashInfo from dvc.main import main -from dvc.objects.db import ODBManager from dvc.objects.errors import ObjectFormatError from dvc.utils import relpath from tests.basic_env import TestDir, TestDvc @@ -47,7 +47,7 @@ class TestCacheLoadBadDirCache(TestDvc): self.assertEqual(len(ret), 0) def test(self): - from dvc.objects import load + from dvc.data import load dir_hash = "123.dir" fname = os.fspath(self.dvc.odb.local.hash_to_path(dir_hash)) diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index e764aef9f..a5b72563e 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -139,8 +139,8 @@ def test_upper_case_remote(tmp_dir, dvc, local_cloud): def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): - from dvc.objects.stage import stage - from dvc.objects.tree import Tree + from dvc.data.stage import stage + from dvc.data.tree import Tree tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}}) @@ -150,7 +150,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): [{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}] ) tree.digest() - with patch("dvc.objects.stage._stage_tree", return_value=(None, tree)): + with patch("dvc.data.stage._stage_tree", return_value=(None, tree)): _, _, obj = stage(dvc.odb.local, path, dvc.odb.local.fs, "md5") hash1 = obj.hash_info @@ -158,7 +158,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): [{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}] ) tree.digest() - with patch("dvc.objects.stage._stage_tree", return_value=(None, tree)): + with patch("dvc.data.stage._stage_tree", return_value=(None, tree)): _, _, obj = stage(dvc.odb.local, path, dvc.odb.local.fs, "md5") hash2 = obj.hash_info diff --git a/tests/func/test_repo.py b/tests/func/test_repo.py index fd4239d66..c8c6f78c5 100644 --- a/tests/func/test_repo.py +++ b/tests/func/test_repo.py @@ -1,5 +1,5 @@ +from dvc.data.db import ODBManager from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK -from dvc.objects.db import ODBManager from dvc.system import System diff --git a/tests/unit/fs/test_dvc.py b/tests/unit/fs/test_dvc.py index 68fa73818..bb8dd9cfd 100644 --- a/tests/unit/fs/test_dvc.py +++ b/tests/unit/fs/test_dvc.py @@ -4,9 +4,9 @@ import shutil import pytest from dvc.config import NoRemoteError +from dvc.data.stage import stage from dvc.fs.dvc import DvcFileSystem from dvc.hash_info import HashInfo -from dvc.objects.stage import stage from dvc.utils.fs import remove @@ -227,7 +227,7 @@ def test_get_hash_dir(tmp_dir, dvc, mocker): {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} ) fs = DvcFileSystem(repo=dvc) - get_file_hash_spy = mocker.spy(dvc_module.objects.stage, "get_file_hash") + get_file_hash_spy = mocker.spy(dvc_module.data.stage, "get_file_hash") assert ( fs.info((tmp_dir / "dir").fs_path)["md5"] == "8761c4e9acad696bee718615e23e22db.dir" diff --git a/tests/unit/fs/test_repo.py b/tests/unit/fs/test_repo.py index 47a3733e2..4bfe24c10 100644 --- a/tests/unit/fs/test_repo.py +++ b/tests/unit/fs/test_repo.py @@ -4,9 +4,9 @@ from unittest import mock import pytest +from dvc.data.stage import stage from dvc.fs.repo import RepoFileSystem from dvc.hash_info import HashInfo -from dvc.objects.stage import stage from tests.utils import clean_staging @@ -584,9 +584,9 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc): def test_get_hash_dirty_file(tmp_dir, dvc): - from dvc.objects import check + from dvc.data import check + from dvc.data.stage import get_file_hash from dvc.objects.errors import ObjectFormatError - from dvc.objects.stage import get_file_hash tmp_dir.dvc_gen("file", "file") file_hash_info = HashInfo("md5", "8c7dd922ad47494fc02c388e12c00eac") diff --git a/tests/unit/objects/db/test_local.py b/tests/unit/objects/db/test_local.py index 86ebb1e62..57b452bc7 100644 --- a/tests/unit/objects/db/test_local.py +++ b/tests/unit/objects/db/test_local.py @@ -3,9 +3,9 @@ import os import pytest +from dvc.data.db.local import LocalObjectDB from dvc.fs.local import LocalFileSystem from dvc.hash_info import HashInfo -from dvc.objects.db.local import LocalObjectDB def test_status_download_optimization(mocker, dvc): @@ -13,7 +13,7 @@ def test_status_download_optimization(mocker, dvc): And the desired files to fetch are already on the local cache, Don't check the existence of the desired files on the remote cache """ - from dvc.objects.status import compare_status + from dvc.data.status import compare_status odb = LocalObjectDB(LocalFileSystem(), os.getcwd()) obj_ids = { @@ -85,9 +85,9 @@ def test_set_exec_ignore_errors(tmp_dir, dvc, mocker, err): def test_staging_file(tmp_dir, dvc): - from dvc.objects import check - from dvc.objects.stage import stage - from dvc.objects.transfer import transfer + from dvc.data import check + from dvc.data.stage import stage + from dvc.data.transfer import transfer tmp_dir.gen("foo", "foo") fs = LocalFileSystem() @@ -113,9 +113,9 @@ def test_staging_file(tmp_dir, dvc): def test_staging_dir(tmp_dir, dvc): - from dvc.objects import check - from dvc.objects.stage import stage - from dvc.objects.transfer import transfer + from dvc.data import check + from dvc.data.stage import stage + from dvc.data.transfer import transfer tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) fs = LocalFileSystem() diff --git a/tests/unit/objects/test_tree.py b/tests/unit/objects/test_tree.py index 953d3d8a2..f765a6637 100644 --- a/tests/unit/objects/test_tree.py +++ b/tests/unit/objects/test_tree.py @@ -2,9 +2,9 @@ from operator import itemgetter import pytest +from dvc.data.meta import Meta +from dvc.data.tree import Tree, _merge from dvc.hash_info import HashInfo -from dvc.objects.meta import Meta -from dvc.objects.tree import Tree, _merge @pytest.mark.parametrize( diff --git a/tests/unit/output/test_local.py b/tests/unit/output/test_local.py index ddc703456..2cb3ce14b 100644 --- a/tests/unit/output/test_local.py +++ b/tests/unit/output/test_local.py @@ -1,8 +1,8 @@ import os from unittest.mock import patch +from dvc.data.meta import Meta from dvc.hash_info import HashInfo -from dvc.objects.meta import Meta from dvc.output import Output from dvc.stage import Stage from dvc.utils import relpath diff --git a/tests/unit/remote/test_base.py b/tests/unit/remote/test_base.py index 4370b17e5..02b38b955 100644 --- a/tests/unit/remote/test_base.py +++ b/tests/unit/remote/test_base.py @@ -3,7 +3,7 @@ import posixpath from unittest import mock from dvc.fs.base import FileSystem -from dvc.objects.db.base import ObjectDB +from dvc.objects.db import ObjectDB class _CallableOrNone: diff --git a/tests/unit/remote/test_index.py b/tests/unit/remote/test_index.py index 272aff5fa..ba57c8a75 100644 --- a/tests/unit/remote/test_index.py +++ b/tests/unit/remote/test_index.py @@ -3,7 +3,7 @@ import os import pytest from funcy import first -from dvc.objects.db.index import ObjectDBIndex +from dvc.data.db.index import ObjectDBIndex @pytest.fixture diff --git a/tests/unit/remote/test_remote.py b/tests/unit/remote/test_remote.py index e3896bca8..b169d6cce 100644 --- a/tests/unit/remote/test_remote.py +++ b/tests/unit/remote/test_remote.py @@ -2,10 +2,10 @@ import os import pytest +from dvc.data.db import get_index from dvc.fs import get_cloud_fs from dvc.fs.gs import GSFileSystem from dvc.fs.s3 import S3FileSystem -from dvc.objects.db import get_index def test_remote_with_hash_jobs(dvc): diff --git a/tests/unit/remote/test_slow_link_detection.py b/tests/unit/remote/test_slow_link_detection.py index 5913a90e3..46066e3ae 100644 --- a/tests/unit/remote/test_slow_link_detection.py +++ b/tests/unit/remote/test_slow_link_detection.py @@ -1,14 +1,12 @@ import pytest -import dvc.objects.db.slow_link_detection -from dvc.objects.db.slow_link_detection import slow_link_guard +import dvc.data.slow_link_detection +from dvc.data.slow_link_detection import slow_link_guard @pytest.fixture(autouse=True) def timeout_immediately(monkeypatch): - monkeypatch.setattr( - dvc.objects.db.slow_link_detection, "timeout_seconds", 0.0 - ) + monkeypatch.setattr(dvc.data.slow_link_detection, "timeout_seconds", 0.0) @pytest.fixture @@ -27,7 +25,7 @@ def test_show_warning_once(caplog, make_remote): slow_link_guard(lambda x, y: None)(None, remote) slow_link_guard(lambda x, y: None)(None, remote) - slow_link_detection = dvc.objects.db.slow_link_detection + slow_link_detection = dvc.data.slow_link_detection message = slow_link_detection.message # noqa, pylint: disable=no-member assert len(caplog.records) == 1 assert caplog.records[0].message == message diff --git a/tests/unit/stage/test_cache.py b/tests/unit/stage/test_cache.py index 233cf7b30..c1dbf3947 100644 --- a/tests/unit/stage/test_cache.py +++ b/tests/unit/stage/test_cache.py @@ -167,7 +167,7 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker): def test_shared_stage_cache(tmp_dir, dvc, run_copy): import stat - from dvc.objects.db import ODBManager + from dvc.data.db import ODBManager tmp_dir.gen("foo", "foo") diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 3b7937ab3..111a88015 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -45,8 +45,8 @@ def dump_sv(stream, metrics, delimiter=",", header=True): def clean_staging(): + from dvc.data.stage import _STAGING_MEMFS_PATH from dvc.fs.memory import MemoryFileSystem - from dvc.objects.stage import _STAGING_MEMFS_PATH try: MemoryFileSystem().fs.rm(