Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

dvc: upgrade to dvc-objects 0.0.4

Browse Source
This commit is contained in:
Ruslan Kuprieiev
2022-06-13 18:03:53 +05:45
committed by skshetry
parent 7b16a377e5
commit ef689c5593
20 changed files with 110 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/data_cloud.py
+1 -1
View File
@@ -180,5 +180,5 @@ class DataCloud:
def get_url_for(self, remote, checksum):
odb = self.get_remote_odb(remote)
path = odb.hash_to_path(checksum)
path = odb.oid_to_path(checksum)
return odb.fs.unstrip_protocol(path)
dvc/dependency/repo.py
+4 -4
View File
@@ -155,19 +155,19 @@ class RepoDependency(Dependency):
def _check_circular_import(self, odb, obj_ids):
from dvc.exceptions import CircularImportError
from dvc.fs.dvc import DvcFileSystem
from dvc_data.db.reference import ReferenceObjectDB
from dvc_data.db.reference import ReferenceHashFileDB
from dvc_data.objects.tree import Tree
if not isinstance(odb, ReferenceObjectDB):
if not isinstance(odb, ReferenceHashFileDB):
return
def iter_objs():
for hash_info in obj_ids:
if hash_info.isdir:
tree = Tree.load(odb, hash_info)
yield from (odb.get(oid) for _, _, oid in tree)
yield from (odb.get(hi.value) for _, _, hi in tree)
else:
yield odb.get(hash_info)
yield odb.get(hash_info.value)
checked_urls = set()
for obj in iter_objs():
dvc/fs/data.py
+2 -2
View File
@@ -70,7 +70,7 @@ class _DataFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
if not value:
raise FileNotFoundError
cache_path = self.repo.odb.local.hash_to_path(value)
cache_path = self.repo.odb.local.oid_to_path(value)
if self.repo.odb.local.fs.exists(cache_path):
return self.repo.odb.local.fs, cache_path
@@ -79,7 +79,7 @@ class _DataFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
remote_odb = self.repo.cloud.get_remote_odb(remote)
except NoRemoteError as exc:
raise FileNotFoundError from exc
remote_fs_path = remote_odb.hash_to_path(value)
remote_fs_path = remote_odb.oid_to_path(value)
return remote_odb.fs, remote_fs_path
def open( # type: ignore
dvc/info.py
+1 -1
View File
@@ -80,7 +80,7 @@ def _get_linktype_support_info(repo):
links = generic.test_links(
["reflink", "hardlink", "symlink"],
odb.fs,
odb.fs_path,
odb.path,
repo.fs,
repo.root_dir,
)
dvc/output.py
+5 -5
View File
@@ -417,7 +417,7 @@ class Output:
@property
def cache_path(self):
return self.odb.fs.unstrip_protocol(
self.odb.hash_to_path(self.hash_info.value)
self.odb.oid_to_path(self.hash_info.value)
)
def get_hash(self):
@@ -885,7 +885,7 @@ class Output:
if not self.is_dir_checksum:
raise DvcException("cannot get dir cache for file checksum")
obj = self.odb.get(self.hash_info)
obj = self.odb.get(self.hash_info.value)
try:
ocheck(self.odb, obj)
except FileNotFoundError:
@@ -916,7 +916,7 @@ class Output:
logger.debug(f"failed to pull cache for '{self}'")
try:
ocheck(self.odb, self.odb.get(self.hash_info))
ocheck(self.odb, self.odb.get(self.hash_info.value))
except FileNotFoundError:
msg = (
"Missing cache for directory '{}'. "
@@ -973,7 +973,7 @@ class Output:
else:
obj = self.get_obj(filter_info=kwargs.get("filter_info"))
if not obj:
obj = self.odb.get(self.hash_info)
obj = self.odb.get(self.hash_info.value)
if not obj:
return {}
@@ -1065,7 +1065,7 @@ class Output:
except TreeMergeError as exc:
raise MergeError(str(exc)) from exc
self.odb.add(merged.fs_path, merged.fs, merged.hash_info)
self.odb.add(merged.path, merged.fs, merged.oid)
self.hash_info = merged.hash_info
self.meta = Meta(
dvc/repo/index.py
+1 -1
View File
@@ -172,7 +172,7 @@ class Index:
from dvc.config import NoRemoteError
from dvc_data import Tree
tree = Tree(None, None, None)
tree = Tree()
for out in self.outs:
if not out.use_cache:
dvc/stage/cache.py
+4 -4
View File
@@ -223,7 +223,7 @@ class StageCache:
from_fs = from_odb.fs
to_fs = to_odb.fs
func = _log_exceptions(fs.generic.copy)
runs = from_fs.path.join(from_odb.fs_path, "runs")
runs = from_fs.path.join(from_odb.path, "runs")
http_odb = next(
(
@@ -234,7 +234,7 @@ class StageCache:
None,
)
if http_odb:
path = http_odb.fs_path
path = http_odb.path
message = f"run-cache is not supported for http filesystem: {path}"
raise RunCacheNotSupported(message)
@@ -243,11 +243,11 @@ class StageCache:
return ret
for src in from_fs.find(runs):
rel = from_fs.path.relpath(src, from_odb.fs_path)
rel = from_fs.path.relpath(src, from_odb.path)
if not isinstance(to_fs, LocalFileSystem):
rel = from_fs.path.as_posix(rel)
dst = to_fs.path.join(to_odb.fs_path, rel)
dst = to_fs.path.join(to_odb.path, rel)
key = to_fs.path.parent(dst)
# check if any build cache already exists for this key
# TODO: check if MaxKeys=1 or something like that applies
dvc/testing/tmp_dir.py
+1 -1
View File
@@ -239,7 +239,7 @@ class TmpDir(pathlib.Path):
}
return super().read_text(*args, encoding="utf-8", **kwargs)
def hash_to_path(self, hash_):
def oid_to_path(self, hash_):
return str(self / hash_[0:2] / hash_[2:])
def dump(self, *args, **kwargs):
setup.cfg
+1 -1
View File
@@ -69,7 +69,7 @@ install_requires =
scmrepo==0.0.24
dvc-render==0.0.5
dvclive>=0.7.3
dvc-data==0.0.3
dvc-data==0.0.4
[options.extras_require]
all =
tests/func/data/db/test_index.py
+2 -2
View File
@@ -60,8 +60,8 @@ def test_clear_on_download_err(tmp_dir, dvc, index, mocker):
out = tmp_dir.dvc_gen({"dir": {"foo": "foo content"}})[0].outs[0]
dvc.push()
for _, _, oid in out.obj:
remove(dvc.odb.local.get(oid).fs_path)
for _, _, hi in out.obj:
remove(dvc.odb.local.get(hi.value).path)
remove(out.fs_path)
assert list(index.hashes())
tests/func/test_add.py
+1 -3
View File
@@ -957,9 +957,7 @@ def test_add_to_remote(tmp_dir, dvc, remote, workspace):
hash_info = stage.outs[0].hash_info
meta = stage.outs[0].meta
with open(
remote.hash_to_path(hash_info.value), encoding="utf-8"
) as stream:
with open(remote.oid_to_path(hash_info.value), encoding="utf-8") as stream:
assert stream.read() == "foo"
assert meta.size == len("foo")
tests/func/test_checkout.py
+1 -1
View File
@@ -107,7 +107,7 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
# to check if dvc will detect that the cache is corrupted.
obj = load(self.dvc.odb.local, out.hash_info)
_, _, entry_oid = list(obj)[0]
cache = self.dvc.odb.local.hash_to_path(entry_oid.value)
cache = self.dvc.odb.local.oid_to_path(entry_oid.value)
os.chmod(cache, 0o644)
with open(cache, "w+", encoding="utf-8") as fobj:
tests/func/test_data_cloud.py
+20 -21
View File
@@ -13,8 +13,8 @@ from dvc.testing.test_remote import ( # noqa, pylint: disable=unused-import
TestRemote,
)
from dvc.utils.fs import remove
from dvc_data.db.local import LocalObjectDB
from dvc_objects.hashfile.db import ObjectDB
from dvc_data.db.local import LocalHashFileDB
from dvc_objects.hashfile.db import HashFileDB
def test_cloud_cli(tmp_dir, dvc, remote, mocker):
@@ -37,32 +37,32 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker):
cache_dir = stage_dir.outs[0].cache_path
# FIXME check status output
hashes_exist = mocker.spy(LocalObjectDB, "hashes_exist")
oids_exist = mocker.spy(LocalHashFileDB, "oids_exist")
assert main(["push"] + args) == 0
assert os.path.exists(cache)
assert os.path.isfile(cache)
assert os.path.isfile(cache_dir)
assert hashes_exist.called
assert oids_exist.called
assert all(
_kwargs["jobs"] == jobs
for (_args, _kwargs) in hashes_exist.call_args_list
for (_args, _kwargs) in oids_exist.call_args_list
)
remove(dvc.odb.local.cache_dir)
hashes_exist.reset_mock()
oids_exist.reset_mock()
assert main(["fetch"] + args) == 0
assert os.path.exists(cache)
assert os.path.isfile(cache)
assert os.path.isfile(cache_dir)
assert hashes_exist.called
assert oids_exist.called
assert all(
_kwargs["jobs"] == jobs
for (_args, _kwargs) in hashes_exist.call_args_list
for (_args, _kwargs) in oids_exist.call_args_list
)
hashes_exist.reset_mock()
oids_exist.reset_mock()
assert main(["pull"] + args) == 0
assert os.path.exists(cache)
@@ -70,10 +70,10 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker):
assert os.path.isfile(cache_dir)
assert os.path.isfile("foo")
assert os.path.isdir("data_dir")
assert hashes_exist.called
assert oids_exist.called
assert all(
_kwargs["jobs"] == jobs
for (_args, _kwargs) in hashes_exist.call_args_list
for (_args, _kwargs) in oids_exist.call_args_list
)
with open(cache, encoding="utf-8") as fd:
@@ -84,37 +84,36 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker):
if remote.url.startswith("http"):
return
hashes_exist.reset_mock()
oids_exist.reset_mock()
_list_hashes_traverse = mocker.spy(ObjectDB, "_list_hashes_traverse")
_list_oids_traverse = mocker.spy(HashFileDB, "_list_oids_traverse")
# NOTE: check if remote gc works correctly on directories
assert main(["gc", "-cw", "-f"] + args) == 0
assert _list_hashes_traverse.called
assert _list_oids_traverse.called
assert all(
_kwargs["jobs"] == 2
for (_args, _kwargs) in hashes_exist.call_args_list
_kwargs["jobs"] == 2 for (_args, _kwargs) in oids_exist.call_args_list
)
shutil.move(dvc.odb.local.cache_dir, dvc.odb.local.cache_dir + ".back")
assert main(["fetch"] + args) == 0
assert hashes_exist.called
assert oids_exist.called
assert all(
_kwargs["jobs"] == jobs
for (_args, _kwargs) in hashes_exist.call_args_list
for (_args, _kwargs) in oids_exist.call_args_list
)
hashes_exist.reset_mock()
oids_exist.reset_mock()
assert main(["pull", "-f"] + args) == 0
assert os.path.exists(cache)
assert os.path.isfile(cache)
assert os.path.isfile(cache_dir)
assert os.path.isfile("foo")
assert os.path.isdir("data_dir")
assert hashes_exist.called
assert oids_exist.called
assert all(
_kwargs["jobs"] == jobs
for (_args, _kwargs) in hashes_exist.call_args_list
for (_args, _kwargs) in oids_exist.call_args_list
)
tests/func/test_gc.py
+4 -4
View File
@@ -11,7 +11,7 @@ from dvc.exceptions import CollectCacheError
from dvc.fs import LocalFileSystem
from dvc.repo import Repo as DvcRepo
from dvc.utils.fs import remove
from dvc_data.db.local import LocalObjectDB
from dvc_data.db.local import LocalHashFileDB
from tests.basic_env import TestDir, TestDvcGit
@@ -24,12 +24,12 @@ class TestGC(TestDvcGit):
raw_dir_hash = stages[0].outs[0].hash_info.as_raw().value
self.good_cache = [
self.dvc.odb.local.hash_to_path(md5)
self.dvc.odb.local.oid_to_path(md5)
for md5 in self.dvc.odb.local.all()
if md5 != raw_dir_hash
]
self.bad_cache = [self.dvc.odb.local.hash_to_path(raw_dir_hash)]
self.bad_cache = [self.dvc.odb.local.oid_to_path(raw_dir_hash)]
for i in ["123", "234", "345"]:
path = os.path.join(self.dvc.odb.local.cache_dir, i[0:2], i[2:])
self.create(path, i)
@@ -221,7 +221,7 @@ def test_gc_no_unpacked_dir(tmp_dir, dvc):
os.remove("dir.dvc")
unpackeddir = (
dir_stages[0].outs[0].cache_path + LocalObjectDB.UNPACKED_DIR_SUFFIX
dir_stages[0].outs[0].cache_path + LocalHashFileDB.UNPACKED_DIR_SUFFIX
)
# older (pre 1.0) versions of dvc used to generate this dir
tests/func/test_import_url.py
+3 -3
View File
@@ -187,7 +187,7 @@ def test_import_url_to_remote_single_file(
hash_info = stage.outs[0].hash_info
with open(
local_remote.hash_to_path(hash_info.value), encoding="utf-8"
local_remote.oid_to_path(hash_info.value), encoding="utf-8"
) as fobj:
assert fobj.read() == "foo"
assert stage.outs[0].meta.size == len("foo")
@@ -216,7 +216,7 @@ def test_import_url_to_remote_directory(tmp_dir, dvc, workspace, local_remote):
hash_info = stage.outs[0].hash_info
with open(
local_remote.hash_to_path(hash_info.value), encoding="utf-8"
local_remote.oid_to_path(hash_info.value), encoding="utf-8"
) as stream:
file_parts = json.load(stream)
@@ -229,7 +229,7 @@ def test_import_url_to_remote_directory(tmp_dir, dvc, workspace, local_remote):
for file_part in file_parts:
with open(
local_remote.hash_to_path(file_part["md5"]), encoding="utf-8"
local_remote.oid_to_path(file_part["md5"]), encoding="utf-8"
) as fobj:
assert fobj.read() == file_part["relpath"]
tests/func/test_odb.py
+5 -5
View File
@@ -36,8 +36,8 @@ def test_cache(tmp_dir, dvc):
assert cache1_md5 in md5_list
assert cache2_md5 in md5_list
odb_cache1 = odb.local.hash_to_path(cache1_md5)
odb_cache2 = odb.local.hash_to_path(cache2_md5)
odb_cache1 = odb.local.oid_to_path(cache1_md5)
odb_cache2 = odb.local.oid_to_path(cache2_md5)
assert os.fspath(odb_cache1) == cache1
assert os.fspath(odb_cache2) == cache2
@@ -46,13 +46,13 @@ def test_cache_load_bad_dir_cache(tmp_dir, dvc):
from dvc_data import load
dir_hash = "123.dir"
fname = os.fspath(dvc.odb.local.hash_to_path(dir_hash))
fname = os.fspath(dvc.odb.local.oid_to_path(dir_hash))
tmp_dir.gen({fname: "<clearly>not,json"})
with pytest.raises(ObjectFormatError):
load(dvc.odb.local, HashInfo("md5", dir_hash))
dir_hash = "234.dir"
fname = os.fspath(dvc.odb.local.hash_to_path(dir_hash))
fname = os.fspath(dvc.odb.local.oid_to_path(dir_hash))
tmp_dir.gen({fname: '{"a": "b"}'})
with pytest.raises(ObjectFormatError):
load(dvc.odb.local, HashInfo("md5", dir_hash))
@@ -89,7 +89,7 @@ def test_remote_cache_references(tmp_dir, dvc):
dvc.odb = ODBManager(dvc)
assert dvc.odb.ssh.fs_path == "/tmp"
assert dvc.odb.ssh.path == "/tmp"
def test_shared_cache_dir(tmp_dir):
tests/func/test_remote.py
+12 -12
View File
@@ -155,7 +155,7 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
hash1 = obj.hash_info
# remove the raw dir obj to force building the tree on the next stage call
dvc.odb.local.fs.remove(dvc.odb.local.hash_to_path(hash1.as_raw().value))
dvc.odb.local.fs.remove(dvc.odb.local.oid_to_path(hash1.as_raw().value))
tree = Tree.from_list(
[{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}]
@@ -181,7 +181,7 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
def unreliable_upload(from_fs, from_info, to_fs, to_info, **kwargs):
if os.path.abspath(to_info) == os.path.abspath(
odb.get(foo.hash_info).fs_path
odb.get(foo.hash_info.value).path
):
raise Exception("stop foo")
return original(from_fs, from_info, to_fs, to_info, **kwargs)
@@ -191,9 +191,9 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory, local_remote):
dvc.push()
assert upload_error_info.value.amount == 2
assert not odb.exists(foo.hash_info)
assert odb.exists(bar.hash_info)
assert not odb.exists(baz.hash_info)
assert not odb.exists(foo.hash_info.value)
assert odb.exists(bar.hash_info.value)
assert not odb.exists(baz.hash_info.value)
# Push everything and delete local cache
dvc.push()
@@ -273,8 +273,8 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote):
# foo .dir file should be uploaded after bar
odb = dvc.cloud.get_remote_odb("upstream")
foo_path = odb.hash_to_path(foo.hash_info.value)
bar_path = odb.hash_to_path(foo.obj._trie[("bar",)][1].value)
foo_path = odb.oid_to_path(foo.hash_info.value)
bar_path = odb.oid_to_path(foo.obj._trie[("bar",)][1].value)
paths = [args[3] for args, _ in mocked_upload.call_args_list]
assert paths.index(foo_path) > paths.index(bar_path)
@@ -412,7 +412,7 @@ def test_protect_local_remote(tmp_dir, dvc, local_remote):
dvc.push()
odb = dvc.cloud.get_remote_odb("upstream")
remote_cache_file = odb.hash_to_path(stage.outs[0].hash_info.value)
remote_cache_file = odb.oid_to_path(stage.outs[0].hash_info.value)
assert os.path.exists(remote_cache_file)
assert stat.S_IMODE(os.stat(remote_cache_file).st_mode) == 0o444
@@ -427,9 +427,9 @@ def test_push_incomplete_dir(tmp_dir, dvc, mocker, local_remote):
file_objs = [entry_obj for _, _, entry_obj in out.obj]
# remove one of the cache files for directory
remove(odb.hash_to_path(file_objs[0].value))
remove(odb.oid_to_path(file_objs[0].value))
dvc.push()
assert not remote_odb.exists(out.hash_info)
assert not remote_odb.exists(file_objs[0])
assert remote_odb.exists(file_objs[1])
assert not remote_odb.exists(out.hash_info.value)
assert not remote_odb.exists(file_objs[0].value)
assert remote_odb.exists(file_objs[1].value)
tests/unit/data/db/test_local.py
+10 -10
View File
@@ -4,7 +4,7 @@ import os
import pytest
from dvc.fs import LocalFileSystem
from dvc_data.db.local import LocalObjectDB
from dvc_data.db.local import LocalHashFileDB
from dvc_objects.hashfile.hash_info import HashInfo
@@ -15,19 +15,19 @@ def test_status_download_optimization(mocker, dvc):
"""
from dvc_data.status import compare_status
odb = LocalObjectDB(LocalFileSystem(), os.getcwd())
odb = LocalHashFileDB(LocalFileSystem(), os.getcwd())
obj_ids = {
HashInfo("md5", "acbd18db4cc2f85cedef654fccc4a4d8"),
HashInfo("md5", "37b51d194a7513e45b56f6524f2d51f2"),
}
local_exists = [hash_info.value for hash_info in obj_ids]
mocker.patch.object(odb, "hashes_exist", return_value=local_exists)
mocker.patch.object(odb, "oids_exist", return_value=local_exists)
src_odb = mocker.Mock()
compare_status(src_odb, odb, obj_ids, check_deleted=False)
assert src_odb.hashes_exist.call_count == 0
assert src_odb.oids_exist.call_count == 0
@pytest.mark.parametrize("link_name", ["hardlink", "symlink"])
@@ -97,8 +97,8 @@ def test_staging_file(tmp_dir, dvc):
local_odb, (tmp_dir / "foo").fs_path, fs, "md5"
)
assert not local_odb.exists(obj.hash_info)
assert staging_odb.exists(obj.hash_info)
assert not local_odb.exists(obj.hash_info.value)
assert staging_odb.exists(obj.hash_info.value)
with pytest.raises(FileNotFoundError):
check(local_odb, obj)
@@ -108,7 +108,7 @@ def test_staging_file(tmp_dir, dvc):
check(local_odb, obj)
check(staging_odb, obj)
path = local_odb.hash_to_path(obj.hash_info.value)
path = local_odb.oid_to_path(obj.hash_info.value)
assert fs.exists(path)
@@ -125,8 +125,8 @@ def test_staging_dir(tmp_dir, dvc):
local_odb, (tmp_dir / "dir").fs_path, fs, "md5"
)
assert not local_odb.exists(obj.hash_info)
assert staging_odb.exists(obj.hash_info)
assert not local_odb.exists(obj.hash_info.value)
assert staging_odb.exists(obj.hash_info.value)
with pytest.raises(FileNotFoundError):
check(local_odb, obj)
@@ -138,5 +138,5 @@ def test_staging_dir(tmp_dir, dvc):
check(local_odb, obj)
check(staging_odb, obj)
path = local_odb.hash_to_path(obj.hash_info.value)
path = local_odb.oid_to_path(obj.hash_info.value)
assert fs.exists(path)
tests/unit/data/test_tree.py
-200
View File
@@ -1,200 +0,0 @@
from operator import itemgetter
import pytest
from dvc_data.objects.tree import Tree, _merge
from dvc_objects.hashfile.hash_info import HashInfo
from dvc_objects.hashfile.meta import Meta
@pytest.mark.parametrize(
"lst, trie_dict",
[
([], {}),
(
[
{"md5": "def", "relpath": "zzz"},
{"md5": "123", "relpath": "foo"},
{"md5": "abc", "relpath": "aaa"},
{"md5": "456", "relpath": "bar"},
],
{
("zzz",): (None, HashInfo("md5", "def")),
("foo",): (None, HashInfo("md5", "123")),
("bar",): (None, HashInfo("md5", "456")),
("aaa",): (None, HashInfo("md5", "abc")),
},
),
(
[
{"md5": "123", "relpath": "dir/b"},
{"md5": "456", "relpath": "dir/z"},
{"md5": "789", "relpath": "dir/a"},
{"md5": "abc", "relpath": "b"},
{"md5": "def", "relpath": "a"},
{"md5": "ghi", "relpath": "z"},
{"md5": "jkl", "relpath": "dir/subdir/b"},
{"md5": "mno", "relpath": "dir/subdir/z"},
{"md5": "pqr", "relpath": "dir/subdir/a"},
],
{
("dir", "b"): (
None,
HashInfo("md5", "123"),
),
("dir", "z"): (
None,
HashInfo("md5", "456"),
),
("dir", "a"): (
None,
HashInfo("md5", "789"),
),
("b",): (None, HashInfo("md5", "abc")),
("a",): (None, HashInfo("md5", "def")),
("z",): (None, HashInfo("md5", "ghi")),
("dir", "subdir", "b"): (
None,
HashInfo("md5", "jkl"),
),
("dir", "subdir", "z"): (
None,
HashInfo("md5", "mno"),
),
("dir", "subdir", "a"): (
None,
HashInfo("md5", "pqr"),
),
},
),
],
)
def test_list(lst, trie_dict):
tree = Tree.from_list(lst)
assert tree._dict == trie_dict
assert tree.as_list() == sorted(lst, key=itemgetter("relpath"))
@pytest.mark.parametrize(
"trie_dict, nfiles",
[
({}, 0),
(
{
("a",): (Meta(size=1), HashInfo("md5", "abc")),
("b",): (Meta(size=2), HashInfo("md5", "def")),
("c",): (Meta(size=3), HashInfo("md5", "ghi")),
("dir", "foo"): (Meta(size=4), HashInfo("md5", "jkl")),
("dir", "bar"): (Meta(size=5), HashInfo("md5", "mno")),
("dir", "baz"): (Meta(size=6), HashInfo("md5", "pqr")),
},
6,
),
(
{
("a",): (Meta(size=1), HashInfo("md5", "abc")),
("b",): (Meta(), HashInfo("md5", "def")),
},
2,
),
],
)
def test_nfiles(trie_dict, nfiles):
tree = Tree(None, None, None)
tree._dict = trie_dict
assert len(tree) == nfiles
@pytest.mark.parametrize(
"trie_dict",
[
{},
{
("a",): (None, HashInfo("md5", "abc")),
("b",): (None, HashInfo("md5", "def")),
("c",): (None, HashInfo("md5", "ghi")),
("dir", "foo"): (None, HashInfo("md5", "jkl")),
("dir", "bar"): (None, HashInfo("md5", "mno")),
("dir", "baz"): (None, HashInfo("md5", "pqr")),
("dir", "subdir", "1"): (None, HashInfo("md5", "stu")),
("dir", "subdir", "2"): (None, HashInfo("md5", "vwx")),
("dir", "subdir", "3"): (None, HashInfo("md5", "yz")),
},
],
)
def test_items(trie_dict):
tree = Tree(None, None, None)
tree._dict = trie_dict
assert list(tree) == [
(key, value[0], value[1]) for key, value in trie_dict.items()
]
@pytest.mark.parametrize(
"ancestor_dict, our_dict, their_dict, merged_dict",
[
({}, {}, {}, {}),
(
{("foo",): HashInfo("md5", "123")},
{
("foo",): HashInfo("md5", "123"),
("bar",): HashInfo("md5", "345"),
},
{
("foo",): HashInfo("md5", "123"),
("baz",): HashInfo("md5", "678"),
},
{
("foo",): HashInfo("md5", "123"),
("bar",): HashInfo("md5", "345"),
("baz",): HashInfo("md5", "678"),
},
),
(
{
("common",): HashInfo("md5", "123"),
("subdir", "foo"): HashInfo("md5", "345"),
},
{
("common",): HashInfo("md5", "123"),
("subdir", "foo"): HashInfo("md5", "345"),
("subdir", "bar"): HashInfo("md5", "678"),
},
{
("common",): HashInfo("md5", "123"),
("subdir", "foo"): HashInfo("md5", "345"),
("subdir", "baz"): HashInfo("md5", "91011"),
},
{
("common",): HashInfo("md5", "123"),
("subdir", "foo"): HashInfo("md5", "345"),
("subdir", "bar"): HashInfo("md5", "678"),
("subdir", "baz"): HashInfo("md5", "91011"),
},
),
(
{},
{("foo",): HashInfo("md5", "123")},
{("bar",): HashInfo("md5", "456")},
{
("foo",): HashInfo("md5", "123"),
("bar",): HashInfo("md5", "456"),
},
),
(
{},
{},
{("bar",): HashInfo("md5", "123")},
{("bar",): HashInfo("md5", "123")},
),
(
{},
{("bar",): HashInfo("md5", "123")},
{},
{("bar",): HashInfo("md5", "123")},
),
],
)
def test_merge(ancestor_dict, our_dict, their_dict, merged_dict):
actual = _merge(ancestor_dict, our_dict, their_dict)
assert actual == merged_dict
tests/unit/remote/test_base.py
+32 -32
View File
@@ -16,17 +16,17 @@ class _CallableOrNone:
CallableOrNone = _CallableOrNone()
@mock.patch.object(ObjectDB, "_list_hashes_traverse")
@mock.patch.object(ObjectDB, "list_hashes_exists")
def test_hashes_exist(object_exists, traverse, dvc):
@mock.patch.object(ObjectDB, "_list_oids_traverse")
@mock.patch.object(ObjectDB, "list_oids_exists")
def test_oids_exist(object_exists, traverse, dvc):
odb = ObjectDB(FileSystem(), None)
# remote does not support traverse
odb.fs.CAN_TRAVERSE = False
with mock.patch.object(odb, "_list_hashes", return_value=list(range(256))):
hashes = set(range(1000))
odb.hashes_exist(hashes)
object_exists.assert_called_with(hashes, None)
with mock.patch.object(odb, "_list_oids", return_value=list(range(256))):
oids = set(range(1000))
odb.oids_exist(oids)
object_exists.assert_called_with(oids, None)
traverse.assert_not_called()
odb.fs.CAN_TRAVERSE = True
@@ -34,20 +34,18 @@ def test_hashes_exist(object_exists, traverse, dvc):
# large remote, small local
object_exists.reset_mock()
traverse.reset_mock()
with mock.patch.object(
odb, "_list_hashes", return_value=list(range(2048))
):
hashes = list(range(1000))
odb.hashes_exist(hashes)
with mock.patch.object(odb, "_list_oids", return_value=list(range(2048))):
oids = list(range(1000))
odb.oids_exist(oids)
# verify that _odb_paths_with_max() short circuits
# before returning all 2048 remote hashes
max_hashes = math.ceil(
odb._max_estimation_size(hashes)
# before returning all 2048 remote oids
max_oids = math.ceil(
odb._max_estimation_size(oids)
/ pow(16, odb.fs.TRAVERSE_PREFIX_LEN)
)
assert max_hashes < 2048
assert max_oids < 2048
object_exists.assert_called_with(
frozenset(range(max_hashes, 1000)), None
frozenset(range(max_oids, 1000)), None
)
traverse.assert_not_called()
@@ -55,9 +53,9 @@ def test_hashes_exist(object_exists, traverse, dvc):
object_exists.reset_mock()
traverse.reset_mock()
odb.fs._JOBS = 16
with mock.patch.object(odb, "_list_hashes", return_value=list(range(256))):
hashes = list(range(1000000))
odb.hashes_exist(hashes)
with mock.patch.object(odb, "_list_oids", return_value=list(range(256))):
oids = list(range(1000000))
odb.oids_exist(oids)
object_exists.assert_not_called()
traverse.assert_called_with(
256 * pow(16, odb.fs.TRAVERSE_PREFIX_LEN),
@@ -66,36 +64,38 @@ def test_hashes_exist(object_exists, traverse, dvc):
)
@mock.patch.object(ObjectDB, "_list_hashes", return_value=[])
@mock.patch.object(ObjectDB, "_path_to_hash", side_effect=lambda x: x)
def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc):
@mock.patch.object(ObjectDB, "_list_oids", return_value=[])
@mock.patch.object(ObjectDB, "path_to_oid", side_effect=lambda x: x)
def test_list_oids_traverse(
path_to_oid, list_oids, dvc # pylint: disable=unused-argument
):
odb = ObjectDB(FileSystem(), None)
odb.fs_path = "foo"
# parallel traverse
size = 256 / odb.fs._JOBS * odb.fs.LIST_OBJECT_PAGE_SIZE
list(odb._list_hashes_traverse(size, {0}))
list(odb._list_oids_traverse(size, {0}))
for i in range(1, 16):
list_hashes.assert_any_call(f"{i:0{odb.fs.TRAVERSE_PREFIX_LEN}x}")
list_oids.assert_any_call(f"{i:0{odb.fs.TRAVERSE_PREFIX_LEN}x}")
for i in range(1, 256):
list_hashes.assert_any_call(f"{i:02x}")
list_oids.assert_any_call(f"{i:02x}")
# default traverse (small remote)
size -= 1
list_hashes.reset_mock()
list(odb._list_hashes_traverse(size - 1, {0}))
list_hashes.assert_called_with(None)
list_oids.reset_mock()
list(odb._list_oids_traverse(size - 1, {0}))
list_oids.assert_called_with(None)
def test_list_hashes(dvc):
def test_list_oids(dvc):
odb = ObjectDB(FileSystem(), None)
odb.fs_path = "foo"
with mock.patch.object(
odb, "_list_paths", return_value=["12/3456", "bar"]
):
hashes = list(odb._list_hashes())
assert hashes == ["123456"]
oids = list(odb._list_oids())
assert oids == ["123456"]
def test_list_paths(dvc):