objects: migrate remote push/pull to objects.transfer (#6308)

* objects: migrate remote status to objects.status

* data_cloud: use objects.status

* objects: migrate remote._process to objects.transfer

* data_cloud: use objects.transfer

* objects: handle memfs staging objects in transfer

* fetch/import: use unified transfer()

* update get_remote and transfer exception usage

* update tests

* objects.transfer: use src ODB verify rules after xfer

* odb: move index from remote into odb

* objects.transfer: skip src status check when possible

* update tests

* remove dvc.remote

* odb.index: migrate to diskcache

* objects: fix status/transfer optimization

* update index usage

* state: remove dead sqlite related code

* update cloud tests

* drop unnecessary remove in tests

* use abstract base class in odb index

* status: load trees from local cache instead of remote odb when possible

* handle status when dir cache is explicitly removed
This commit is contained in:
Peter Rowlands (변기호)
2021-07-20 13:30:01 +03:00
committed by GitHub
parent 66d0df2d12
commit 3d84df930b
42 changed files with 890 additions and 1285 deletions
+5 -9
View File
@@ -8,7 +8,6 @@ from dvc.hash_info import HashInfo
from dvc.objects.db.local import LocalObjectDB
from dvc.objects.file import HashFile
from dvc.path_info import PathInfo
from dvc.remote.index import RemoteIndexNoop
def test_status_download_optimization(mocker, dvc):
@@ -16,8 +15,9 @@ def test_status_download_optimization(mocker, dvc):
And the desired files to fetch are already on the local cache,
Don't check the existence of the desired files on the remote cache
"""
odb = LocalObjectDB(LocalFileSystem(), PathInfo("."))
from dvc.objects.status import compare_status
odb = LocalObjectDB(LocalFileSystem(), PathInfo("."))
objs = {
HashFile(
None, odb.fs, HashInfo("md5", "acbd18db4cc2f85cedef654fccc4a4d8")
@@ -30,14 +30,10 @@ def test_status_download_optimization(mocker, dvc):
local_exists = [obj.hash_info.value for obj in objs]
mocker.patch.object(odb, "hashes_exist", return_value=local_exists)
other_remote = mocker.Mock()
other_remote.url = "other_remote"
other_remote.hashes_exist.return_value = []
other_remote.index = RemoteIndexNoop()
src_odb = mocker.Mock()
other_remote.status(odb, objs, download=True)
assert other_remote.hashes_exist.call_count == 0
compare_status(src_odb, odb, objs, check_deleted=False)
assert src_odb.hashes_exist.call_count == 0
@pytest.mark.parametrize("link_name", ["hardlink", "symlink"])