mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-15 23:09:47 +01:00
objects: migrate remote push/pull to objects.transfer (#6308)
* objects: migrate remote status to objects.status * data_cloud: use objects.status * objects: migrate remote._process to objects.transfer * data_cloud: use objects.transfer * objects: handle memfs staging objects in transfer * fetch/import: use unified transfer() * update get_remote and transfer exception usage * update tests * objects.transfer: use src ODB verify rules after xfer * odb: move index from remote into odb * objects.transfer: skip src status check when possible * update tests * remove dvc.remote * odb.index: migrate to diskcache * objects: fix status/transfer optimization * update index usage * state: remove dead sqlite related code * update cloud tests * drop unnecessary remove in tests * use abstract base class in odb index * status: load trees from local cache instead of remote odb when possible * handle status when dir cache is explicitly removed
This commit is contained in:
@@ -0,0 +1,88 @@
|
||||
import pytest
|
||||
|
||||
from dvc.exceptions import DownloadError, UploadError
|
||||
from dvc.fs.local import LocalFileSystem
|
||||
from dvc.objects.db import get_index
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def index(dvc, local_remote, mocker):
|
||||
odb = dvc.cloud.get_remote_odb("upstream")
|
||||
return get_index(odb)
|
||||
|
||||
|
||||
def test_indexed_on_status(tmp_dir, dvc, index):
|
||||
foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0]
|
||||
bar = tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})[0].outs[0]
|
||||
baz_hash = bar.obj.trie.get(("baz",)).hash_info
|
||||
dvc.push()
|
||||
index.clear()
|
||||
|
||||
dvc.status(cloud=True)
|
||||
assert {bar.hash_info.value, baz_hash.value} == set(index.hashes())
|
||||
assert [bar.hash_info.value] == list(index.dir_hashes())
|
||||
assert foo.hash_info.value not in index.hashes()
|
||||
|
||||
|
||||
def test_indexed_on_push(tmp_dir, dvc, index):
|
||||
foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0]
|
||||
bar = tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})[0].outs[0]
|
||||
baz_hash = bar.obj.trie.get(("baz",)).hash_info
|
||||
|
||||
dvc.push()
|
||||
assert {bar.hash_info.value, baz_hash.value} == set(index.hashes())
|
||||
assert [bar.hash_info.value] == list(index.dir_hashes())
|
||||
assert foo.hash_info.value not in index.hashes()
|
||||
|
||||
|
||||
def test_indexed_dir_missing(tmp_dir, dvc, index):
|
||||
bar = tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})[0].outs[0]
|
||||
index.update([bar.hash_info.value], [])
|
||||
dvc.status(cloud=True)
|
||||
assert not list(index.hashes())
|
||||
|
||||
|
||||
def test_clear_on_gc(tmp_dir, dvc, index):
|
||||
(foo,) = tmp_dir.dvc_gen({"dir": {"foo": "foo content"}})
|
||||
dvc.push()
|
||||
dvc.remove(foo.relpath)
|
||||
|
||||
assert list(index.hashes())
|
||||
dvc.gc(workspace=True, cloud=True)
|
||||
assert not list(index.hashes())
|
||||
|
||||
|
||||
def test_clear_on_download_err(tmp_dir, dvc, index, mocker):
|
||||
out = tmp_dir.dvc_gen({"dir": {"foo": "foo content"}})[0].outs[0]
|
||||
dvc.push()
|
||||
|
||||
for _, entry in out.obj:
|
||||
remove(dvc.odb.local.get(entry.hash_info).path_info)
|
||||
remove(out.path_info)
|
||||
|
||||
assert list(index.hashes())
|
||||
|
||||
mocker.patch(
|
||||
"dvc.fs.local.LocalFileSystem.upload_fobj", side_effect=Exception
|
||||
)
|
||||
with pytest.raises(DownloadError):
|
||||
dvc.pull()
|
||||
assert not list(index.hashes())
|
||||
|
||||
|
||||
def test_partial_upload(tmp_dir, dvc, index, mocker):
|
||||
tmp_dir.dvc_gen({"foo": "foo content"})
|
||||
tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})
|
||||
|
||||
original = LocalFileSystem._upload
|
||||
|
||||
def unreliable_upload(self, from_file, to_info, name=None, **kwargs):
|
||||
if "baz" in name:
|
||||
raise Exception("stop baz")
|
||||
return original(self, from_file, to_info, name, **kwargs)
|
||||
|
||||
mocker.patch.object(LocalFileSystem, "upload_fobj", unreliable_upload)
|
||||
with pytest.raises(UploadError):
|
||||
dvc.push()
|
||||
assert not list(index.hashes())
|
||||
Reference in New Issue
Block a user