diff --git a/dvc/cli/__init__.py b/dvc/cli/__init__.py index e206befd6..2f700c086 100644 --- a/dvc/cli/__init__.py +++ b/dvc/cli/__init__.py @@ -121,7 +121,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]: ) return None - from dvc_data.build import IgnoreInCollectedDirError + from dvc_data.hashfile.build import IgnoreInCollectedDirError if isinstance(exc, IgnoreInCollectedDirError): logger.exception("") diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 36b3acb37..8f76a55f9 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -3,12 +3,12 @@ import logging from typing import TYPE_CHECKING, Iterable, Optional -from dvc_data.db import get_index +from dvc_data.hashfile.db import get_index if TYPE_CHECKING: from dvc_data.hashfile.db import HashFileDB from dvc_data.hashfile.hash_info import HashInfo - from dvc_data.status import CompareStatusResult + from dvc_data.hashfile.status import CompareStatusResult logger = logging.getLogger(__name__) @@ -57,7 +57,7 @@ class DataCloud: def _init_odb(self, name): from dvc.fs import get_cloud_fs - from dvc_data.db import get_odb + from dvc_data.hashfile.db import get_odb cls, config, fs_path = get_cloud_fs(self.repo, name=name) config["tmp_dir"] = self.repo.index_db_dir @@ -82,7 +82,7 @@ class DataCloud: **kwargs, ): from dvc.exceptions import FileTransferError - from dvc_data.transfer import TransferError, transfer + from dvc_data.hashfile.transfer import TransferError, transfer try: return transfer(src_odb, dest_odb, objs, **kwargs) @@ -164,7 +164,7 @@ class DataCloud: log_missing: log warning messages if file doesn't exist neither in cache, neither in cloud. """ - from dvc_data.status import compare_status + from dvc_data.hashfile.status import compare_status if not odb: odb = self.get_remote_odb(remote, "status") diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index b2e85be58..5a95da192 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -63,7 +63,7 @@ class RepoDependency(Dependency): return {self.PARAM_PATH: self.def_path, self.PARAM_REPO: self.def_repo} def download(self, to, jobs=None): - from dvc_data.checkout import checkout + from dvc_data.hashfile.checkout import checkout for odb, objs in self.get_used_objs().items(): self.repo.cloud.pull(objs, jobs=jobs, odb=odb) @@ -103,8 +103,8 @@ class RepoDependency(Dependency): from dvc.config import NoRemoteError from dvc.exceptions import NoOutputOrStageError, PathMissingError from dvc.utils import as_posix - from dvc_data.build import build - from dvc_data.objects.tree import Tree, TreeError + from dvc_data.hashfile.build import build + from dvc_data.hashfile.tree import Tree, TreeError local_odb = self.repo.odb.local locked = kwargs.pop("locked", True) @@ -153,8 +153,8 @@ class RepoDependency(Dependency): def _check_circular_import(self, odb, obj_ids): from dvc.exceptions import CircularImportError from dvc.fs.dvc import DvcFileSystem - from dvc_data.db.reference import ReferenceHashFileDB - from dvc_data.objects.tree import Tree + from dvc_data.hashfile.db.reference import ReferenceHashFileDB + from dvc_data.hashfile.tree import Tree if not isinstance(odb, ReferenceHashFileDB): return diff --git a/dvc/odbmgr.py b/dvc/odbmgr.py index ee3d6d524..160cf4c54 100644 --- a/dvc/odbmgr.py +++ b/dvc/odbmgr.py @@ -1,5 +1,5 @@ from dvc.fs import GitFileSystem, Schemes -from dvc_data.db import get_odb +from dvc_data.hashfile.db import get_odb def _get_odb(repo, settings, fs=None): diff --git a/dvc/output.py b/dvc/output.py index 4a87119fa..f90e11b27 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -17,15 +17,15 @@ from dvc.exceptions import ( MergeError, RemoteCacheRequiredError, ) -from dvc_data import Tree -from dvc_data import check as ocheck -from dvc_data import load as oload -from dvc_data.build import build -from dvc_data.checkout import checkout +from dvc_data.hashfile import Tree +from dvc_data.hashfile import check as ocheck +from dvc_data.hashfile import load as oload +from dvc_data.hashfile.build import build +from dvc_data.hashfile.checkout import checkout from dvc_data.hashfile.hash_info import HashInfo from dvc_data.hashfile.istextfile import istextfile from dvc_data.hashfile.meta import Meta -from dvc_data.transfer import transfer as otransfer +from dvc_data.hashfile.transfer import transfer as otransfer from dvc_objects.errors import ObjectFormatError from .fs import LocalFileSystem, RemoteMissingDepsError, Schemes, get_cloud_fs @@ -606,8 +606,8 @@ class Output: self.odb.set_exec(self.fs_path) def _checkout(self, *args, **kwargs): - from dvc_data.checkout import CheckoutError as _CheckoutError - from dvc_data.checkout import LinkError, PromptError + from dvc_data.hashfile.checkout import CheckoutError as _CheckoutError + from dvc_data.hashfile.checkout import LinkError, PromptError kwargs.setdefault("ignore", self.dvcignore) try: @@ -1068,8 +1068,8 @@ class Output: ) def merge(self, ancestor, other): - from dvc_data.objects.tree import MergeError as TreeMergeError - from dvc_data.objects.tree import du, merge + from dvc_data.hashfile.tree import MergeError as TreeMergeError + from dvc_data.hashfile.tree import du, merge assert other diff --git a/dvc/repo/data.py b/dvc/repo/data.py index 6a4d25259..484ce0406 100644 --- a/dvc/repo/data.py +++ b/dvc/repo/data.py @@ -74,8 +74,8 @@ def _granular_diff( new_obj: Optional["HashFile"], cache: "HashFileDB", ) -> Dict[str, List[str]]: - from dvc_data.diff import ROOT - from dvc_data.diff import diff as odiff + from dvc_data.hashfile.diff import ROOT + from dvc_data.hashfile.diff import diff as odiff def path_join(root: str, *paths: str, isdir: bool = False) -> str: if not isdir and paths == ROOT: @@ -102,7 +102,7 @@ def _get_obj_items(root: str, obj: Optional["HashFile"]) -> List[str]: if not obj: return [] - from dvc_data.objects.tree import Tree + from dvc_data.hashfile.tree import Tree sep = os.path.sep if isinstance(obj, Tree): @@ -164,7 +164,7 @@ def _git_info(scm: "Base", untracked_files: str = "all") -> GitInfo: def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]: - from dvc_data.build import build + from dvc_data.hashfile.build import build unstaged_diff = defaultdict(list) for out in repo.index.outs: diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index 648a837e9..293e26f6d 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -115,7 +115,7 @@ def _paths_checksums(repo, targets): def _output_paths(repo, targets): from dvc.fs import LocalFileSystem - from dvc_data.build import build + from dvc_data.hashfile.build import build on_working_fs = isinstance(repo.fs, LocalFileSystem) diff --git a/dvc/repo/gc.py b/dvc/repo/gc.py index 964e68b99..35110e15f 100644 --- a/dvc/repo/gc.py +++ b/dvc/repo/gc.py @@ -60,8 +60,8 @@ def gc( from contextlib import ExitStack from dvc.repo import Repo - from dvc_data.db import get_index - from dvc_data.gc import gc as ogc + from dvc_data.hashfile.db import get_index + from dvc_data.hashfile.gc import gc as ogc if not repos: repos = [] diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index 0c4357e45..7cd6e46ed 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -9,7 +9,7 @@ from funcy import cached_property, first from dvc import fs from dvc.exceptions import DvcException from dvc.utils import dict_sha256, relpath -from dvc_data.transfer import _log_exceptions +from dvc_data.hashfile.transfer import _log_exceptions if TYPE_CHECKING: from dvc_objects.db import ObjectDB diff --git a/setup.cfg b/setup.cfg index b9b3cf15d..6d6f1248b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -66,7 +66,7 @@ install_requires = dvc-render==0.0.10 dvc-task==0.1.2 dvclive>=0.10.0 - dvc-data==0.5.3 + dvc-data==0.6.1 dvc-http==2.19.1 hydra-core>=1.1.0 diff --git a/tests/func/data/db/test_index.py b/tests/func/data/db/test_index.py index ac08d6c54..d1bfab69c 100644 --- a/tests/func/data/db/test_index.py +++ b/tests/func/data/db/test_index.py @@ -3,7 +3,7 @@ import pytest from dvc.exceptions import DownloadError, UploadError from dvc.fs import LocalFileSystem from dvc.utils.fs import remove -from dvc_data.db import get_index +from dvc_data.hashfile.db import get_index @pytest.fixture diff --git a/tests/func/test_add.py b/tests/func/test_add.py index b096bbf52..a072ab90d 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -103,7 +103,7 @@ def test_add_unsupported_file(dvc): def test_add_directory(tmp_dir, dvc): - from dvc_data import load + from dvc_data.hashfile import load (stage,) = tmp_dir.dvc_gen({"dir": {"file": "file"}}) @@ -457,7 +457,7 @@ class TestAddCommit(TestDvc): def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc): tmp_dir.gen({"data/data": "foo"}) - counter = mocker.spy(dvc_data.build, "_build_tree") + counter = mocker.spy(dvc_data.hashfile.build, "_build_tree") ret = main(["add", "data"]) assert ret == 0 assert counter.mock.call_count == 3 @@ -881,7 +881,7 @@ def test_add_with_cache_link_error(tmp_dir, dvc, mocker, capsys): tmp_dir.gen("foo", "foo") mocker.patch( - "dvc_data.checkout.test_links", + "dvc_data.hashfile.checkout.test_links", return_value=[], ) dvc.add("foo") diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index 44e636c14..da8eb12fc 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -90,7 +90,7 @@ class TestCheckoutCorruptedCacheFile(TestRepro): class TestCheckoutCorruptedCacheDir(TestDvc): def test(self): - from dvc_data import load + from dvc_data.hashfile import load # NOTE: using 'copy' so that cache and link don't have same inode ret = main(["config", "cache.type", "copy"]) diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index bc7394832..e4727f34e 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -13,8 +13,8 @@ from dvc.testing.test_remote import ( # noqa, pylint: disable=unused-import TestRemote, ) from dvc.utils.fs import remove -from dvc_data.db.local import LocalHashFileDB from dvc_data.hashfile.db import HashFileDB +from dvc_data.hashfile.db.local import LocalHashFileDB def test_cloud_cli(tmp_dir, dvc, remote, mocker): diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index 0f034f88f..e6a179b77 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -7,8 +7,8 @@ from dvc.external_repo import CLONES, external_repo from dvc.testing.tmp_dir import make_subrepo from dvc.utils import relpath from dvc.utils.fs import makedirs, remove -from dvc_data.build import build -from dvc_data.transfer import transfer +from dvc_data.hashfile.build import build +from dvc_data.hashfile.transfer import transfer def test_external_repo(erepo_dir, mocker): diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index 27a2e9f6a..3c8cabfcf 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -11,7 +11,7 @@ from dvc.exceptions import CollectCacheError from dvc.fs import LocalFileSystem from dvc.repo import Repo as DvcRepo from dvc.utils.fs import remove -from dvc_data.db.local import LocalHashFileDB +from dvc_data.hashfile.db.local import LocalHashFileDB from tests.basic_env import TestDir, TestDvcGit diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index 670d1b5df..fd374ca63 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -10,7 +10,7 @@ from dvc.pathspec_math import PatternInfo, merge_patterns from dvc.repo import Repo from dvc.testing.tmp_dir import TmpDir from dvc.types import List -from dvc_data.build import IgnoreInCollectedDirError +from dvc_data.hashfile.build import IgnoreInCollectedDirError from dvc_data.hashfile.utils import get_mtime_and_size diff --git a/tests/func/test_import.py b/tests/func/test_import.py index 181f05661..cb0ba08b4 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -527,7 +527,7 @@ def test_import_with_no_exec(tmp_dir, dvc, erepo_dir): def test_import_with_jobs(mocker, dvc, erepo_dir): - import dvc_data.transfer as otransfer + import dvc_data.hashfile.transfer as otransfer with erepo_dir.chdir(): erepo_dir.dvc_gen( diff --git a/tests/func/test_odb.py b/tests/func/test_odb.py index c2ded2551..4467bd755 100644 --- a/tests/func/test_odb.py +++ b/tests/func/test_odb.py @@ -43,7 +43,7 @@ def test_cache(tmp_dir, dvc): def test_cache_load_bad_dir_cache(tmp_dir, dvc): - from dvc_data import load + from dvc_data.hashfile import load dir_hash = "123.dir" fname = os.fspath(dvc.odb.local.oid_to_path(dir_hash)) diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index aac08ce79..ac596cc82 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -139,8 +139,8 @@ def test_upper_case_remote(tmp_dir, dvc, local_cloud): def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): - from dvc_data.build import build - from dvc_data.objects.tree import Tree + from dvc_data.hashfile.build import build + from dvc_data.hashfile.tree import Tree tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}}) @@ -150,7 +150,9 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): [{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}] ) tree.digest() - with patch("dvc_data.build._build_tree", return_value=(None, tree)): + with patch( + "dvc_data.hashfile.build._build_tree", return_value=(None, tree) + ): _, _, obj = build(dvc.odb.local, path, dvc.odb.local.fs, "md5") hash1 = obj.hash_info @@ -161,7 +163,9 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): [{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}] ) tree.digest() - with patch("dvc_data.build._build_tree", return_value=(None, tree)): + with patch( + "dvc_data.hashfile.build._build_tree", return_value=(None, tree) + ): _, _, obj = build(dvc.odb.local, path, dvc.odb.local.fs, "md5") hash2 = obj.hash_info diff --git a/tests/unit/cli/test_main.py b/tests/unit/cli/test_main.py index 73cf3bf72..fbe6ef7d2 100644 --- a/tests/unit/cli/test_main.py +++ b/tests/unit/cli/test_main.py @@ -4,7 +4,7 @@ import pytest from funcy import raiser from dvc.cli import main -from dvc_data.build import IgnoreInCollectedDirError +from dvc_data.hashfile.build import IgnoreInCollectedDirError from dvc_data.hashfile.cache import DiskError from dvc_objects.fs.base import FileSystem, RemoteMissingDepsError diff --git a/tests/unit/data/db/test_local.py b/tests/unit/data/db/test_local.py index b755cf64f..50063c2c9 100644 --- a/tests/unit/data/db/test_local.py +++ b/tests/unit/data/db/test_local.py @@ -4,7 +4,7 @@ import os import pytest from dvc.fs import LocalFileSystem -from dvc_data.db.local import LocalHashFileDB +from dvc_data.hashfile.db.local import LocalHashFileDB from dvc_data.hashfile.hash_info import HashInfo @@ -13,7 +13,7 @@ def test_status_download_optimization(mocker, dvc): And the desired files to fetch are already on the local cache, Don't check the existence of the desired files on the remote cache """ - from dvc_data.status import compare_status + from dvc_data.hashfile.status import compare_status odb = LocalHashFileDB(LocalFileSystem(), os.getcwd()) obj_ids = { @@ -85,9 +85,9 @@ def test_set_exec_ignore_errors(tmp_dir, dvc, mocker, err): def test_staging_file(tmp_dir, dvc): - from dvc_data import check - from dvc_data.build import build - from dvc_data.transfer import transfer + from dvc_data.hashfile import check + from dvc_data.hashfile.build import build + from dvc_data.hashfile.transfer import transfer tmp_dir.gen("foo", "foo") fs = LocalFileSystem() @@ -113,9 +113,9 @@ def test_staging_file(tmp_dir, dvc): def test_staging_dir(tmp_dir, dvc): - from dvc_data import check - from dvc_data.build import build - from dvc_data.transfer import transfer + from dvc_data.hashfile import check + from dvc_data.hashfile.build import build + from dvc_data.hashfile.transfer import transfer tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) fs = LocalFileSystem() diff --git a/tests/unit/fs/test_data.py b/tests/unit/fs/test_data.py index 0b838d1eb..6e3df4f09 100644 --- a/tests/unit/fs/test_data.py +++ b/tests/unit/fs/test_data.py @@ -6,7 +6,7 @@ import pytest import dvc_data from dvc.fs.data import DataFileSystem from dvc.utils.fs import remove -from dvc_data.build import build +from dvc_data.hashfile.build import build from dvc_data.hashfile.hash_info import HashInfo diff --git a/tests/unit/fs/test_dvc.py b/tests/unit/fs/test_dvc.py index 17c8c9d58..8d2e14d5e 100644 --- a/tests/unit/fs/test_dvc.py +++ b/tests/unit/fs/test_dvc.py @@ -7,7 +7,7 @@ import pytest from dvc.fs.dvc import DvcFileSystem from dvc.testing.tmp_dir import make_subrepo -from dvc_data.build import build +from dvc_data.hashfile.build import build from dvc_data.hashfile.hash_info import HashInfo @@ -566,7 +566,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc): def test_get_hash_dirty_file(tmp_dir, dvc): - from dvc_data import check + from dvc_data.hashfile import check from dvc_data.hashfile.hash import hash_file tmp_dir.dvc_gen("file", "file") diff --git a/tests/unit/remote/test_index.py b/tests/unit/remote/test_index.py index 9c6a541f4..80e2db099 100644 --- a/tests/unit/remote/test_index.py +++ b/tests/unit/remote/test_index.py @@ -3,7 +3,7 @@ import os import pytest from funcy import first -from dvc_data.db.index import ObjectDBIndex +from dvc_data.hashfile.db.index import ObjectDBIndex @pytest.fixture diff --git a/tests/unit/remote/test_remote.py b/tests/unit/remote/test_remote.py index ac08c75dd..2820e1ba4 100644 --- a/tests/unit/remote/test_remote.py +++ b/tests/unit/remote/test_remote.py @@ -5,7 +5,7 @@ from dvc_gs import GSFileSystem from dvc_s3 import S3FileSystem from dvc.fs import get_cloud_fs -from dvc_data.db import get_index +from dvc_data.hashfile.db import get_index def test_remote_with_hash_jobs(dvc):