Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-09 23:10:24 +01:00
Code Issues Packages Projects Releases Wiki Activity

odb: make some listing methods private

Browse Source 
These are not used anywhere outside of odb, so we can make them private
to reduce the scope for the upcoming refactoring.
This commit is contained in:
Ruslan Kuprieiev
2021-12-06 14:10:50 +02:00
parent 954ac6ab03
commit ec73300782
2 changed files with 18 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/objects/db/base.py
+10 -10
View File
@@ -206,7 +206,7 @@ class ObjectDB:
return "".join(parts)
def list_hashes(self, prefix=None, progress_callback=None):
def _list_hashes(self, prefix=None, progress_callback=None):
"""Iterate over hashes in this fs.
If `prefix` is specified, only hashes which begin with `prefix`
@@ -222,12 +222,12 @@ class ObjectDB:
def _hashes_with_limit(self, limit, prefix=None, progress_callback=None):
count = 0
for hash_ in self.list_hashes(prefix, progress_callback):
for hash_ in self._list_hashes(prefix, progress_callback):
yield hash_
count += 1
if count > limit:
logger.debug(
"`list_hashes()` returned max '{}' hashes, "
"`_list_hashes()` returned max '{}' hashes, "
"skipping remaining results".format(limit)
)
return
@@ -266,7 +266,7 @@ class ObjectDB:
max_hashes / total_prefixes, prefix, update
)
else:
hashes = self.list_hashes(prefix, update)
hashes = self._list_hashes(prefix, update)
remote_hashes = set(hashes)
if remote_hashes:
@@ -276,7 +276,7 @@ class ObjectDB:
logger.debug(f"Estimated remote size: {remote_size} files")
return remote_size, remote_hashes
def list_hashes_traverse(
def _list_hashes_traverse(
self, remote_size, remote_hashes, jobs=None, name=None
):
"""Iterate over all hashes found in this fs.
@@ -322,7 +322,7 @@ class ObjectDB:
def list_with_update(prefix):
return list(
self.list_hashes(
self._list_hashes(
prefix=prefix, progress_callback=pbar.update
)
)
@@ -346,10 +346,10 @@ class ObjectDB:
)
if not self.fs.CAN_TRAVERSE:
return self.list_hashes()
return self._list_hashes()
remote_size, remote_hashes = self._estimate_remote_size(name=name)
return self.list_hashes_traverse(
return self._list_hashes_traverse(
remote_size, remote_hashes, jobs, name
)
@@ -455,7 +455,7 @@ class ObjectDB:
# During the tests, for ensuring that the traverse behavior
# is working we turn on this option. It will ensure the
# list_hashes_traverse() is called.
# _list_hashes_traverse() is called.
always_traverse = getattr(self.fs, "_ALWAYS_TRAVERSE", False)
hashes = set(hashes)
@@ -492,6 +492,6 @@ class ObjectDB:
logger.debug(f"Querying '{len(hashes)}' hashes via traverse")
remote_hashes = set(
self.list_hashes_traverse(remote_size, remote_hashes, jobs, name)
self._list_hashes_traverse(remote_size, remote_hashes, jobs, name)
)
return list(hashes & set(remote_hashes))
tests/unit/remote/test_base.py
+8 -8
View File
@@ -34,14 +34,14 @@ def test_cmd_error(dvc):
FileSystem(**config).remove("file")
@mock.patch.object(ObjectDB, "list_hashes_traverse")
@mock.patch.object(ObjectDB, "_list_hashes_traverse")
@mock.patch.object(ObjectDB, "list_hashes_exists")
def test_hashes_exist(object_exists, traverse, dvc):
odb = ObjectDB(FileSystem(), None)
# remote does not support traverse
odb.fs.CAN_TRAVERSE = False
with mock.patch.object(odb, "list_hashes", return_value=list(range(256))):
with mock.patch.object(odb, "_list_hashes", return_value=list(range(256))):
hashes = set(range(1000))
odb.hashes_exist(hashes)
object_exists.assert_called_with(hashes, None, None)
@@ -52,7 +52,7 @@ def test_hashes_exist(object_exists, traverse, dvc):
# large remote, small local
object_exists.reset_mock()
traverse.reset_mock()
with mock.patch.object(odb, "list_hashes", return_value=list(range(256))):
with mock.patch.object(odb, "_list_hashes", return_value=list(range(256))):
hashes = list(range(1000))
odb.hashes_exist(hashes)
# verify that _odb_paths_with_max() short circuits
@@ -71,7 +71,7 @@ def test_hashes_exist(object_exists, traverse, dvc):
object_exists.reset_mock()
traverse.reset_mock()
odb.fs._JOBS = 16
with mock.patch.object(odb, "list_hashes", return_value=list(range(256))):
with mock.patch.object(odb, "_list_hashes", return_value=list(range(256))):
hashes = list(range(1000000))
odb.hashes_exist(hashes)
object_exists.assert_not_called()
@@ -83,7 +83,7 @@ def test_hashes_exist(object_exists, traverse, dvc):
)
@mock.patch.object(ObjectDB, "list_hashes", return_value=[])
@mock.patch.object(ObjectDB, "_list_hashes", return_value=[])
@mock.patch.object(ObjectDB, "_path_to_hash", side_effect=lambda x: x)
def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc):
odb = ObjectDB(FileSystem(), None)
@@ -91,7 +91,7 @@ def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc):
# parallel traverse
size = 256 / odb.fs._JOBS * odb.fs.LIST_OBJECT_PAGE_SIZE
list(odb.list_hashes_traverse(size, {0}))
list(odb._list_hashes_traverse(size, {0}))
for i in range(1, 16):
list_hashes.assert_any_call(
prefix=f"{i:03x}", progress_callback=CallableOrNone
@@ -104,7 +104,7 @@ def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc):
# default traverse (small remote)
size -= 1
list_hashes.reset_mock()
list(odb.list_hashes_traverse(size - 1, {0}))
list(odb._list_hashes_traverse(size - 1, {0}))
list_hashes.assert_called_with(
prefix=None, progress_callback=CallableOrNone
)
@@ -117,7 +117,7 @@ def test_list_hashes(dvc):
with mock.patch.object(
odb, "_list_paths", return_value=["12/3456", "bar"]
):
hashes = list(odb.list_hashes())
hashes = list(odb._list_hashes())
assert hashes == ["123456"]