diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 096f0948c..1377302b3 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -180,5 +180,5 @@ class DataCloud: def get_url_for(self, remote, checksum): odb = self.get_remote_odb(remote) - path = odb.hash_to_path(checksum) + path = odb.oid_to_path(checksum) return odb.fs.unstrip_protocol(path) diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index c6ee61883..a3e1fd52e 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -155,19 +155,19 @@ class RepoDependency(Dependency): def _check_circular_import(self, odb, obj_ids): from dvc.exceptions import CircularImportError from dvc.fs.dvc import DvcFileSystem - from dvc_data.db.reference import ReferenceObjectDB + from dvc_data.db.reference import ReferenceHashFileDB from dvc_data.objects.tree import Tree - if not isinstance(odb, ReferenceObjectDB): + if not isinstance(odb, ReferenceHashFileDB): return def iter_objs(): for hash_info in obj_ids: if hash_info.isdir: tree = Tree.load(odb, hash_info) - yield from (odb.get(oid) for _, _, oid in tree) + yield from (odb.get(hi.value) for _, _, hi in tree) else: - yield odb.get(hash_info) + yield odb.get(hash_info.value) checked_urls = set() for obj in iter_objs(): diff --git a/dvc/fs/data.py b/dvc/fs/data.py index ecb3314c4..72c07333a 100644 --- a/dvc/fs/data.py +++ b/dvc/fs/data.py @@ -70,7 +70,7 @@ class _DataFileSystem(AbstractFileSystem): # pylint:disable=abstract-method if not value: raise FileNotFoundError - cache_path = self.repo.odb.local.hash_to_path(value) + cache_path = self.repo.odb.local.oid_to_path(value) if self.repo.odb.local.fs.exists(cache_path): return self.repo.odb.local.fs, cache_path @@ -79,7 +79,7 @@ class _DataFileSystem(AbstractFileSystem): # pylint:disable=abstract-method remote_odb = self.repo.cloud.get_remote_odb(remote) except NoRemoteError as exc: raise FileNotFoundError from exc - remote_fs_path = remote_odb.hash_to_path(value) + remote_fs_path = remote_odb.oid_to_path(value) return remote_odb.fs, remote_fs_path def open( # type: ignore diff --git a/dvc/info.py b/dvc/info.py index a3dd212fc..2797915a3 100644 --- a/dvc/info.py +++ b/dvc/info.py @@ -80,7 +80,7 @@ def _get_linktype_support_info(repo): links = generic.test_links( ["reflink", "hardlink", "symlink"], odb.fs, - odb.fs_path, + odb.path, repo.fs, repo.root_dir, ) diff --git a/dvc/output.py b/dvc/output.py index c039cf794..60c55e6f6 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -417,7 +417,7 @@ class Output: @property def cache_path(self): return self.odb.fs.unstrip_protocol( - self.odb.hash_to_path(self.hash_info.value) + self.odb.oid_to_path(self.hash_info.value) ) def get_hash(self): @@ -885,7 +885,7 @@ class Output: if not self.is_dir_checksum: raise DvcException("cannot get dir cache for file checksum") - obj = self.odb.get(self.hash_info) + obj = self.odb.get(self.hash_info.value) try: ocheck(self.odb, obj) except FileNotFoundError: @@ -916,7 +916,7 @@ class Output: logger.debug(f"failed to pull cache for '{self}'") try: - ocheck(self.odb, self.odb.get(self.hash_info)) + ocheck(self.odb, self.odb.get(self.hash_info.value)) except FileNotFoundError: msg = ( "Missing cache for directory '{}'. " @@ -973,7 +973,7 @@ class Output: else: obj = self.get_obj(filter_info=kwargs.get("filter_info")) if not obj: - obj = self.odb.get(self.hash_info) + obj = self.odb.get(self.hash_info.value) if not obj: return {} @@ -1065,7 +1065,7 @@ class Output: except TreeMergeError as exc: raise MergeError(str(exc)) from exc - self.odb.add(merged.fs_path, merged.fs, merged.hash_info) + self.odb.add(merged.path, merged.fs, merged.oid) self.hash_info = merged.hash_info self.meta = Meta( diff --git a/dvc/repo/index.py b/dvc/repo/index.py index 1a2eda587..1090651e0 100644 --- a/dvc/repo/index.py +++ b/dvc/repo/index.py @@ -172,7 +172,7 @@ class Index: from dvc.config import NoRemoteError from dvc_data import Tree - tree = Tree(None, None, None) + tree = Tree() for out in self.outs: if not out.use_cache: diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index a18d5d9fe..e5ac33d33 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -223,7 +223,7 @@ class StageCache: from_fs = from_odb.fs to_fs = to_odb.fs func = _log_exceptions(fs.generic.copy) - runs = from_fs.path.join(from_odb.fs_path, "runs") + runs = from_fs.path.join(from_odb.path, "runs") http_odb = next( ( @@ -234,7 +234,7 @@ class StageCache: None, ) if http_odb: - path = http_odb.fs_path + path = http_odb.path message = f"run-cache is not supported for http filesystem: {path}" raise RunCacheNotSupported(message) @@ -243,11 +243,11 @@ class StageCache: return ret for src in from_fs.find(runs): - rel = from_fs.path.relpath(src, from_odb.fs_path) + rel = from_fs.path.relpath(src, from_odb.path) if not isinstance(to_fs, LocalFileSystem): rel = from_fs.path.as_posix(rel) - dst = to_fs.path.join(to_odb.fs_path, rel) + dst = to_fs.path.join(to_odb.path, rel) key = to_fs.path.parent(dst) # check if any build cache already exists for this key # TODO: check if MaxKeys=1 or something like that applies diff --git a/dvc/testing/tmp_dir.py b/dvc/testing/tmp_dir.py index e79bec160..2517166eb 100644 --- a/dvc/testing/tmp_dir.py +++ b/dvc/testing/tmp_dir.py @@ -239,7 +239,7 @@ class TmpDir(pathlib.Path): } return super().read_text(*args, encoding="utf-8", **kwargs) - def hash_to_path(self, hash_): + def oid_to_path(self, hash_): return str(self / hash_[0:2] / hash_[2:]) def dump(self, *args, **kwargs): diff --git a/setup.cfg b/setup.cfg index 8ab2ccbb5..9d6c55eb0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -69,7 +69,7 @@ install_requires = scmrepo==0.0.24 dvc-render==0.0.5 dvclive>=0.7.3 - dvc-data==0.0.3 + dvc-data==0.0.4 [options.extras_require] all = diff --git a/tests/func/data/db/test_index.py b/tests/func/data/db/test_index.py index d45db5973..c803e5ee0 100644 --- a/tests/func/data/db/test_index.py +++ b/tests/func/data/db/test_index.py @@ -60,8 +60,8 @@ def test_clear_on_download_err(tmp_dir, dvc, index, mocker): out = tmp_dir.dvc_gen({"dir": {"foo": "foo content"}})[0].outs[0] dvc.push() - for _, _, oid in out.obj: - remove(dvc.odb.local.get(oid).fs_path) + for _, _, hi in out.obj: + remove(dvc.odb.local.get(hi.value).path) remove(out.fs_path) assert list(index.hashes()) diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 91ae21c6c..b17d42d81 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -957,9 +957,7 @@ def test_add_to_remote(tmp_dir, dvc, remote, workspace): hash_info = stage.outs[0].hash_info meta = stage.outs[0].meta - with open( - remote.hash_to_path(hash_info.value), encoding="utf-8" - ) as stream: + with open(remote.oid_to_path(hash_info.value), encoding="utf-8") as stream: assert stream.read() == "foo" assert meta.size == len("foo") diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index 8a7401536..0711dfbf1 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -107,7 +107,7 @@ class TestCheckoutCorruptedCacheDir(TestDvc): # to check if dvc will detect that the cache is corrupted. obj = load(self.dvc.odb.local, out.hash_info) _, _, entry_oid = list(obj)[0] - cache = self.dvc.odb.local.hash_to_path(entry_oid.value) + cache = self.dvc.odb.local.oid_to_path(entry_oid.value) os.chmod(cache, 0o644) with open(cache, "w+", encoding="utf-8") as fobj: diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 62cd7d689..730628a69 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -13,8 +13,8 @@ from dvc.testing.test_remote import ( # noqa, pylint: disable=unused-import TestRemote, ) from dvc.utils.fs import remove -from dvc_data.db.local import LocalObjectDB -from dvc_objects.hashfile.db import ObjectDB +from dvc_data.db.local import LocalHashFileDB +from dvc_objects.hashfile.db import HashFileDB def test_cloud_cli(tmp_dir, dvc, remote, mocker): @@ -37,32 +37,32 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): cache_dir = stage_dir.outs[0].cache_path # FIXME check status output - hashes_exist = mocker.spy(LocalObjectDB, "hashes_exist") + oids_exist = mocker.spy(LocalHashFileDB, "oids_exist") assert main(["push"] + args) == 0 assert os.path.exists(cache) assert os.path.isfile(cache) assert os.path.isfile(cache_dir) - assert hashes_exist.called + assert oids_exist.called assert all( _kwargs["jobs"] == jobs - for (_args, _kwargs) in hashes_exist.call_args_list + for (_args, _kwargs) in oids_exist.call_args_list ) remove(dvc.odb.local.cache_dir) - hashes_exist.reset_mock() + oids_exist.reset_mock() assert main(["fetch"] + args) == 0 assert os.path.exists(cache) assert os.path.isfile(cache) assert os.path.isfile(cache_dir) - assert hashes_exist.called + assert oids_exist.called assert all( _kwargs["jobs"] == jobs - for (_args, _kwargs) in hashes_exist.call_args_list + for (_args, _kwargs) in oids_exist.call_args_list ) - hashes_exist.reset_mock() + oids_exist.reset_mock() assert main(["pull"] + args) == 0 assert os.path.exists(cache) @@ -70,10 +70,10 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): assert os.path.isfile(cache_dir) assert os.path.isfile("foo") assert os.path.isdir("data_dir") - assert hashes_exist.called + assert oids_exist.called assert all( _kwargs["jobs"] == jobs - for (_args, _kwargs) in hashes_exist.call_args_list + for (_args, _kwargs) in oids_exist.call_args_list ) with open(cache, encoding="utf-8") as fd: @@ -84,37 +84,36 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): if remote.url.startswith("http"): return - hashes_exist.reset_mock() + oids_exist.reset_mock() - _list_hashes_traverse = mocker.spy(ObjectDB, "_list_hashes_traverse") + _list_oids_traverse = mocker.spy(HashFileDB, "_list_oids_traverse") # NOTE: check if remote gc works correctly on directories assert main(["gc", "-cw", "-f"] + args) == 0 - assert _list_hashes_traverse.called + assert _list_oids_traverse.called assert all( - _kwargs["jobs"] == 2 - for (_args, _kwargs) in hashes_exist.call_args_list + _kwargs["jobs"] == 2 for (_args, _kwargs) in oids_exist.call_args_list ) shutil.move(dvc.odb.local.cache_dir, dvc.odb.local.cache_dir + ".back") assert main(["fetch"] + args) == 0 - assert hashes_exist.called + assert oids_exist.called assert all( _kwargs["jobs"] == jobs - for (_args, _kwargs) in hashes_exist.call_args_list + for (_args, _kwargs) in oids_exist.call_args_list ) - hashes_exist.reset_mock() + oids_exist.reset_mock() assert main(["pull", "-f"] + args) == 0 assert os.path.exists(cache) assert os.path.isfile(cache) assert os.path.isfile(cache_dir) assert os.path.isfile("foo") assert os.path.isdir("data_dir") - assert hashes_exist.called + assert oids_exist.called assert all( _kwargs["jobs"] == jobs - for (_args, _kwargs) in hashes_exist.call_args_list + for (_args, _kwargs) in oids_exist.call_args_list ) diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index ce5e40592..b52836576 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -11,7 +11,7 @@ from dvc.exceptions import CollectCacheError from dvc.fs import LocalFileSystem from dvc.repo import Repo as DvcRepo from dvc.utils.fs import remove -from dvc_data.db.local import LocalObjectDB +from dvc_data.db.local import LocalHashFileDB from tests.basic_env import TestDir, TestDvcGit @@ -24,12 +24,12 @@ class TestGC(TestDvcGit): raw_dir_hash = stages[0].outs[0].hash_info.as_raw().value self.good_cache = [ - self.dvc.odb.local.hash_to_path(md5) + self.dvc.odb.local.oid_to_path(md5) for md5 in self.dvc.odb.local.all() if md5 != raw_dir_hash ] - self.bad_cache = [self.dvc.odb.local.hash_to_path(raw_dir_hash)] + self.bad_cache = [self.dvc.odb.local.oid_to_path(raw_dir_hash)] for i in ["123", "234", "345"]: path = os.path.join(self.dvc.odb.local.cache_dir, i[0:2], i[2:]) self.create(path, i) @@ -221,7 +221,7 @@ def test_gc_no_unpacked_dir(tmp_dir, dvc): os.remove("dir.dvc") unpackeddir = ( - dir_stages[0].outs[0].cache_path + LocalObjectDB.UNPACKED_DIR_SUFFIX + dir_stages[0].outs[0].cache_path + LocalHashFileDB.UNPACKED_DIR_SUFFIX ) # older (pre 1.0) versions of dvc used to generate this dir diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py index ab810dc2f..33e79e408 100644 --- a/tests/func/test_import_url.py +++ b/tests/func/test_import_url.py @@ -187,7 +187,7 @@ def test_import_url_to_remote_single_file( hash_info = stage.outs[0].hash_info with open( - local_remote.hash_to_path(hash_info.value), encoding="utf-8" + local_remote.oid_to_path(hash_info.value), encoding="utf-8" ) as fobj: assert fobj.read() == "foo" assert stage.outs[0].meta.size == len("foo") @@ -216,7 +216,7 @@ def test_import_url_to_remote_directory(tmp_dir, dvc, workspace, local_remote): hash_info = stage.outs[0].hash_info with open( - local_remote.hash_to_path(hash_info.value), encoding="utf-8" + local_remote.oid_to_path(hash_info.value), encoding="utf-8" ) as stream: file_parts = json.load(stream) @@ -229,7 +229,7 @@ def test_import_url_to_remote_directory(tmp_dir, dvc, workspace, local_remote): for file_part in file_parts: with open( - local_remote.hash_to_path(file_part["md5"]), encoding="utf-8" + local_remote.oid_to_path(file_part["md5"]), encoding="utf-8" ) as fobj: assert fobj.read() == file_part["relpath"] diff --git a/tests/func/test_odb.py b/tests/func/test_odb.py index 6bee78147..56473ebfb 100644 --- a/tests/func/test_odb.py +++ b/tests/func/test_odb.py @@ -36,8 +36,8 @@ def test_cache(tmp_dir, dvc): assert cache1_md5 in md5_list assert cache2_md5 in md5_list - odb_cache1 = odb.local.hash_to_path(cache1_md5) - odb_cache2 = odb.local.hash_to_path(cache2_md5) + odb_cache1 = odb.local.oid_to_path(cache1_md5) + odb_cache2 = odb.local.oid_to_path(cache2_md5) assert os.fspath(odb_cache1) == cache1 assert os.fspath(odb_cache2) == cache2 @@ -46,13 +46,13 @@ def test_cache_load_bad_dir_cache(tmp_dir, dvc): from dvc_data import load dir_hash = "123.dir" - fname = os.fspath(dvc.odb.local.hash_to_path(dir_hash)) + fname = os.fspath(dvc.odb.local.oid_to_path(dir_hash)) tmp_dir.gen({fname: "not,json"}) with pytest.raises(ObjectFormatError): load(dvc.odb.local, HashInfo("md5", dir_hash)) dir_hash = "234.dir" - fname = os.fspath(dvc.odb.local.hash_to_path(dir_hash)) + fname = os.fspath(dvc.odb.local.oid_to_path(dir_hash)) tmp_dir.gen({fname: '{"a": "b"}'}) with pytest.raises(ObjectFormatError): load(dvc.odb.local, HashInfo("md5", dir_hash)) @@ -89,7 +89,7 @@ def test_remote_cache_references(tmp_dir, dvc): dvc.odb = ODBManager(dvc) - assert dvc.odb.ssh.fs_path == "/tmp" + assert dvc.odb.ssh.path == "/tmp" def test_shared_cache_dir(tmp_dir): diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index 8e8e0fc52..e2e2c8334 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -155,7 +155,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): hash1 = obj.hash_info # remove the raw dir obj to force building the tree on the next stage call - dvc.odb.local.fs.remove(dvc.odb.local.hash_to_path(hash1.as_raw().value)) + dvc.odb.local.fs.remove(dvc.odb.local.oid_to_path(hash1.as_raw().value)) tree = Tree.from_list( [{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}] @@ -181,7 +181,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote): def unreliable_upload(from_fs, from_info, to_fs, to_info, **kwargs): if os.path.abspath(to_info) == os.path.abspath( - odb.get(foo.hash_info).fs_path + odb.get(foo.hash_info.value).path ): raise Exception("stop foo") return original(from_fs, from_info, to_fs, to_info, **kwargs) @@ -191,9 +191,9 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote): dvc.push() assert upload_error_info.value.amount == 2 - assert not odb.exists(foo.hash_info) - assert odb.exists(bar.hash_info) - assert not odb.exists(baz.hash_info) + assert not odb.exists(foo.hash_info.value) + assert odb.exists(bar.hash_info.value) + assert not odb.exists(baz.hash_info.value) # Push everything and delete local cache dvc.push() @@ -273,8 +273,8 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote): # foo .dir file should be uploaded after bar odb = dvc.cloud.get_remote_odb("upstream") - foo_path = odb.hash_to_path(foo.hash_info.value) - bar_path = odb.hash_to_path(foo.obj._trie[("bar",)][1].value) + foo_path = odb.oid_to_path(foo.hash_info.value) + bar_path = odb.oid_to_path(foo.obj._trie[("bar",)][1].value) paths = [args[3] for args, _ in mocked_upload.call_args_list] assert paths.index(foo_path) > paths.index(bar_path) @@ -412,7 +412,7 @@ def test_protect_local_remote(tmp_dir, dvc, local_remote): dvc.push() odb = dvc.cloud.get_remote_odb("upstream") - remote_cache_file = odb.hash_to_path(stage.outs[0].hash_info.value) + remote_cache_file = odb.oid_to_path(stage.outs[0].hash_info.value) assert os.path.exists(remote_cache_file) assert stat.S_IMODE(os.stat(remote_cache_file).st_mode) == 0o444 @@ -427,9 +427,9 @@ def test_push_incomplete_dir(tmp_dir, dvc, mocker, local_remote): file_objs = [entry_obj for _, _, entry_obj in out.obj] # remove one of the cache files for directory - remove(odb.hash_to_path(file_objs[0].value)) + remove(odb.oid_to_path(file_objs[0].value)) dvc.push() - assert not remote_odb.exists(out.hash_info) - assert not remote_odb.exists(file_objs[0]) - assert remote_odb.exists(file_objs[1]) + assert not remote_odb.exists(out.hash_info.value) + assert not remote_odb.exists(file_objs[0].value) + assert remote_odb.exists(file_objs[1].value) diff --git a/tests/unit/data/db/test_local.py b/tests/unit/data/db/test_local.py index 6e60294f8..b68256f34 100644 --- a/tests/unit/data/db/test_local.py +++ b/tests/unit/data/db/test_local.py @@ -4,7 +4,7 @@ import os import pytest from dvc.fs import LocalFileSystem -from dvc_data.db.local import LocalObjectDB +from dvc_data.db.local import LocalHashFileDB from dvc_objects.hashfile.hash_info import HashInfo @@ -15,19 +15,19 @@ def test_status_download_optimization(mocker, dvc): """ from dvc_data.status import compare_status - odb = LocalObjectDB(LocalFileSystem(), os.getcwd()) + odb = LocalHashFileDB(LocalFileSystem(), os.getcwd()) obj_ids = { HashInfo("md5", "acbd18db4cc2f85cedef654fccc4a4d8"), HashInfo("md5", "37b51d194a7513e45b56f6524f2d51f2"), } local_exists = [hash_info.value for hash_info in obj_ids] - mocker.patch.object(odb, "hashes_exist", return_value=local_exists) + mocker.patch.object(odb, "oids_exist", return_value=local_exists) src_odb = mocker.Mock() compare_status(src_odb, odb, obj_ids, check_deleted=False) - assert src_odb.hashes_exist.call_count == 0 + assert src_odb.oids_exist.call_count == 0 @pytest.mark.parametrize("link_name", ["hardlink", "symlink"]) @@ -97,8 +97,8 @@ def test_staging_file(tmp_dir, dvc): local_odb, (tmp_dir / "foo").fs_path, fs, "md5" ) - assert not local_odb.exists(obj.hash_info) - assert staging_odb.exists(obj.hash_info) + assert not local_odb.exists(obj.hash_info.value) + assert staging_odb.exists(obj.hash_info.value) with pytest.raises(FileNotFoundError): check(local_odb, obj) @@ -108,7 +108,7 @@ def test_staging_file(tmp_dir, dvc): check(local_odb, obj) check(staging_odb, obj) - path = local_odb.hash_to_path(obj.hash_info.value) + path = local_odb.oid_to_path(obj.hash_info.value) assert fs.exists(path) @@ -125,8 +125,8 @@ def test_staging_dir(tmp_dir, dvc): local_odb, (tmp_dir / "dir").fs_path, fs, "md5" ) - assert not local_odb.exists(obj.hash_info) - assert staging_odb.exists(obj.hash_info) + assert not local_odb.exists(obj.hash_info.value) + assert staging_odb.exists(obj.hash_info.value) with pytest.raises(FileNotFoundError): check(local_odb, obj) @@ -138,5 +138,5 @@ def test_staging_dir(tmp_dir, dvc): check(local_odb, obj) check(staging_odb, obj) - path = local_odb.hash_to_path(obj.hash_info.value) + path = local_odb.oid_to_path(obj.hash_info.value) assert fs.exists(path) diff --git a/tests/unit/data/test_tree.py b/tests/unit/data/test_tree.py deleted file mode 100644 index 49f972feb..000000000 --- a/tests/unit/data/test_tree.py +++ /dev/null @@ -1,200 +0,0 @@ -from operator import itemgetter - -import pytest - -from dvc_data.objects.tree import Tree, _merge -from dvc_objects.hashfile.hash_info import HashInfo -from dvc_objects.hashfile.meta import Meta - - -@pytest.mark.parametrize( - "lst, trie_dict", - [ - ([], {}), - ( - [ - {"md5": "def", "relpath": "zzz"}, - {"md5": "123", "relpath": "foo"}, - {"md5": "abc", "relpath": "aaa"}, - {"md5": "456", "relpath": "bar"}, - ], - { - ("zzz",): (None, HashInfo("md5", "def")), - ("foo",): (None, HashInfo("md5", "123")), - ("bar",): (None, HashInfo("md5", "456")), - ("aaa",): (None, HashInfo("md5", "abc")), - }, - ), - ( - [ - {"md5": "123", "relpath": "dir/b"}, - {"md5": "456", "relpath": "dir/z"}, - {"md5": "789", "relpath": "dir/a"}, - {"md5": "abc", "relpath": "b"}, - {"md5": "def", "relpath": "a"}, - {"md5": "ghi", "relpath": "z"}, - {"md5": "jkl", "relpath": "dir/subdir/b"}, - {"md5": "mno", "relpath": "dir/subdir/z"}, - {"md5": "pqr", "relpath": "dir/subdir/a"}, - ], - { - ("dir", "b"): ( - None, - HashInfo("md5", "123"), - ), - ("dir", "z"): ( - None, - HashInfo("md5", "456"), - ), - ("dir", "a"): ( - None, - HashInfo("md5", "789"), - ), - ("b",): (None, HashInfo("md5", "abc")), - ("a",): (None, HashInfo("md5", "def")), - ("z",): (None, HashInfo("md5", "ghi")), - ("dir", "subdir", "b"): ( - None, - HashInfo("md5", "jkl"), - ), - ("dir", "subdir", "z"): ( - None, - HashInfo("md5", "mno"), - ), - ("dir", "subdir", "a"): ( - None, - HashInfo("md5", "pqr"), - ), - }, - ), - ], -) -def test_list(lst, trie_dict): - tree = Tree.from_list(lst) - assert tree._dict == trie_dict - assert tree.as_list() == sorted(lst, key=itemgetter("relpath")) - - -@pytest.mark.parametrize( - "trie_dict, nfiles", - [ - ({}, 0), - ( - { - ("a",): (Meta(size=1), HashInfo("md5", "abc")), - ("b",): (Meta(size=2), HashInfo("md5", "def")), - ("c",): (Meta(size=3), HashInfo("md5", "ghi")), - ("dir", "foo"): (Meta(size=4), HashInfo("md5", "jkl")), - ("dir", "bar"): (Meta(size=5), HashInfo("md5", "mno")), - ("dir", "baz"): (Meta(size=6), HashInfo("md5", "pqr")), - }, - 6, - ), - ( - { - ("a",): (Meta(size=1), HashInfo("md5", "abc")), - ("b",): (Meta(), HashInfo("md5", "def")), - }, - 2, - ), - ], -) -def test_nfiles(trie_dict, nfiles): - tree = Tree(None, None, None) - tree._dict = trie_dict - assert len(tree) == nfiles - - -@pytest.mark.parametrize( - "trie_dict", - [ - {}, - { - ("a",): (None, HashInfo("md5", "abc")), - ("b",): (None, HashInfo("md5", "def")), - ("c",): (None, HashInfo("md5", "ghi")), - ("dir", "foo"): (None, HashInfo("md5", "jkl")), - ("dir", "bar"): (None, HashInfo("md5", "mno")), - ("dir", "baz"): (None, HashInfo("md5", "pqr")), - ("dir", "subdir", "1"): (None, HashInfo("md5", "stu")), - ("dir", "subdir", "2"): (None, HashInfo("md5", "vwx")), - ("dir", "subdir", "3"): (None, HashInfo("md5", "yz")), - }, - ], -) -def test_items(trie_dict): - tree = Tree(None, None, None) - tree._dict = trie_dict - assert list(tree) == [ - (key, value[0], value[1]) for key, value in trie_dict.items() - ] - - -@pytest.mark.parametrize( - "ancestor_dict, our_dict, their_dict, merged_dict", - [ - ({}, {}, {}, {}), - ( - {("foo",): HashInfo("md5", "123")}, - { - ("foo",): HashInfo("md5", "123"), - ("bar",): HashInfo("md5", "345"), - }, - { - ("foo",): HashInfo("md5", "123"), - ("baz",): HashInfo("md5", "678"), - }, - { - ("foo",): HashInfo("md5", "123"), - ("bar",): HashInfo("md5", "345"), - ("baz",): HashInfo("md5", "678"), - }, - ), - ( - { - ("common",): HashInfo("md5", "123"), - ("subdir", "foo"): HashInfo("md5", "345"), - }, - { - ("common",): HashInfo("md5", "123"), - ("subdir", "foo"): HashInfo("md5", "345"), - ("subdir", "bar"): HashInfo("md5", "678"), - }, - { - ("common",): HashInfo("md5", "123"), - ("subdir", "foo"): HashInfo("md5", "345"), - ("subdir", "baz"): HashInfo("md5", "91011"), - }, - { - ("common",): HashInfo("md5", "123"), - ("subdir", "foo"): HashInfo("md5", "345"), - ("subdir", "bar"): HashInfo("md5", "678"), - ("subdir", "baz"): HashInfo("md5", "91011"), - }, - ), - ( - {}, - {("foo",): HashInfo("md5", "123")}, - {("bar",): HashInfo("md5", "456")}, - { - ("foo",): HashInfo("md5", "123"), - ("bar",): HashInfo("md5", "456"), - }, - ), - ( - {}, - {}, - {("bar",): HashInfo("md5", "123")}, - {("bar",): HashInfo("md5", "123")}, - ), - ( - {}, - {("bar",): HashInfo("md5", "123")}, - {}, - {("bar",): HashInfo("md5", "123")}, - ), - ], -) -def test_merge(ancestor_dict, our_dict, their_dict, merged_dict): - actual = _merge(ancestor_dict, our_dict, their_dict) - assert actual == merged_dict diff --git a/tests/unit/remote/test_base.py b/tests/unit/remote/test_base.py index 2f9f560bd..7d729c6c7 100644 --- a/tests/unit/remote/test_base.py +++ b/tests/unit/remote/test_base.py @@ -16,17 +16,17 @@ class _CallableOrNone: CallableOrNone = _CallableOrNone() -@mock.patch.object(ObjectDB, "_list_hashes_traverse") -@mock.patch.object(ObjectDB, "list_hashes_exists") -def test_hashes_exist(object_exists, traverse, dvc): +@mock.patch.object(ObjectDB, "_list_oids_traverse") +@mock.patch.object(ObjectDB, "list_oids_exists") +def test_oids_exist(object_exists, traverse, dvc): odb = ObjectDB(FileSystem(), None) # remote does not support traverse odb.fs.CAN_TRAVERSE = False - with mock.patch.object(odb, "_list_hashes", return_value=list(range(256))): - hashes = set(range(1000)) - odb.hashes_exist(hashes) - object_exists.assert_called_with(hashes, None) + with mock.patch.object(odb, "_list_oids", return_value=list(range(256))): + oids = set(range(1000)) + odb.oids_exist(oids) + object_exists.assert_called_with(oids, None) traverse.assert_not_called() odb.fs.CAN_TRAVERSE = True @@ -34,20 +34,18 @@ def test_hashes_exist(object_exists, traverse, dvc): # large remote, small local object_exists.reset_mock() traverse.reset_mock() - with mock.patch.object( - odb, "_list_hashes", return_value=list(range(2048)) - ): - hashes = list(range(1000)) - odb.hashes_exist(hashes) + with mock.patch.object(odb, "_list_oids", return_value=list(range(2048))): + oids = list(range(1000)) + odb.oids_exist(oids) # verify that _odb_paths_with_max() short circuits - # before returning all 2048 remote hashes - max_hashes = math.ceil( - odb._max_estimation_size(hashes) + # before returning all 2048 remote oids + max_oids = math.ceil( + odb._max_estimation_size(oids) / pow(16, odb.fs.TRAVERSE_PREFIX_LEN) ) - assert max_hashes < 2048 + assert max_oids < 2048 object_exists.assert_called_with( - frozenset(range(max_hashes, 1000)), None + frozenset(range(max_oids, 1000)), None ) traverse.assert_not_called() @@ -55,9 +53,9 @@ def test_hashes_exist(object_exists, traverse, dvc): object_exists.reset_mock() traverse.reset_mock() odb.fs._JOBS = 16 - with mock.patch.object(odb, "_list_hashes", return_value=list(range(256))): - hashes = list(range(1000000)) - odb.hashes_exist(hashes) + with mock.patch.object(odb, "_list_oids", return_value=list(range(256))): + oids = list(range(1000000)) + odb.oids_exist(oids) object_exists.assert_not_called() traverse.assert_called_with( 256 * pow(16, odb.fs.TRAVERSE_PREFIX_LEN), @@ -66,36 +64,38 @@ def test_hashes_exist(object_exists, traverse, dvc): ) -@mock.patch.object(ObjectDB, "_list_hashes", return_value=[]) -@mock.patch.object(ObjectDB, "_path_to_hash", side_effect=lambda x: x) -def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc): +@mock.patch.object(ObjectDB, "_list_oids", return_value=[]) +@mock.patch.object(ObjectDB, "path_to_oid", side_effect=lambda x: x) +def test_list_oids_traverse( + path_to_oid, list_oids, dvc # pylint: disable=unused-argument +): odb = ObjectDB(FileSystem(), None) odb.fs_path = "foo" # parallel traverse size = 256 / odb.fs._JOBS * odb.fs.LIST_OBJECT_PAGE_SIZE - list(odb._list_hashes_traverse(size, {0})) + list(odb._list_oids_traverse(size, {0})) for i in range(1, 16): - list_hashes.assert_any_call(f"{i:0{odb.fs.TRAVERSE_PREFIX_LEN}x}") + list_oids.assert_any_call(f"{i:0{odb.fs.TRAVERSE_PREFIX_LEN}x}") for i in range(1, 256): - list_hashes.assert_any_call(f"{i:02x}") + list_oids.assert_any_call(f"{i:02x}") # default traverse (small remote) size -= 1 - list_hashes.reset_mock() - list(odb._list_hashes_traverse(size - 1, {0})) - list_hashes.assert_called_with(None) + list_oids.reset_mock() + list(odb._list_oids_traverse(size - 1, {0})) + list_oids.assert_called_with(None) -def test_list_hashes(dvc): +def test_list_oids(dvc): odb = ObjectDB(FileSystem(), None) odb.fs_path = "foo" with mock.patch.object( odb, "_list_paths", return_value=["12/3456", "bar"] ): - hashes = list(odb._list_hashes()) - assert hashes == ["123456"] + oids = list(odb._list_oids()) + assert oids == ["123456"] def test_list_paths(dvc):