mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
objects: use object IDs and references instead of naive objs in status/transfer (#6360)
* objects.status: use object IDs instead of naive objects * objects.transfer: use object IDs instead of naive objects * objects: add ReferenceHashFile * odb: add ReferenceObjectDB * repo: collect used object IDs instead of naive objs * support basic/naive fs serialization * objects.stage: stage file objects in addition to trees * objects: remove save(), update transfer() * output/dep: update for osave removal * path_info: support both url and path attributes * status/gc: use object IDs instead of naive objs * diff: update staging usage * commit: update staging usage * tests: clean staging when needed * tests: update for save/transfer/stage changes * dvcfs: don't allow config serialization (repofs should be used instead) * utils: return mtime as str, size as int * tests: update unit tests for save/stage/transfer changes * objects: use Repo.open when needed in ReferenceHashFile * fix ls test staging * objects.stage: don't skip hash check when loading from state - individual hashes will still be loaded from state if they are still valid * ReferenceHashFile: serialize repo fs paths relative to repo root * preserve state when staging * objects.tree: skip hash computation in digest() when we have already loaded hash from state * always check file existence in referencehashfile * update repofs test since stage no longer allows nonexistent files * odb: only protect on check() if hash was verified * staging: use unique memfs namespace for staging per dest ODB * tests: clean full staging namespace * refodb: preserve repofs cache dir * repodependency: stage imports to local ODB and strict check circular imports * fetch: fetch staging last * staging: make dest odb mandatory and preserve state * refodb: cache nonlocal fs's * fix rebase errors * repofs: support full instantiation from fs config * repofs: preserve subrepo factory for erepos * tests: clarify repofs dirty file behavior
This commit is contained in:
@@ -50,7 +50,7 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
|
||||
out = stage.outs[0]
|
||||
cache = out.cache_path
|
||||
foo_hash = out.hash_info
|
||||
foo_objs = out.get_used_objs().get(None, set())
|
||||
foo_hashes = out.get_used_objs().get(None, set())
|
||||
|
||||
(stage_dir,) = tmp_dir.dvc_gen(
|
||||
{
|
||||
@@ -63,7 +63,6 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
|
||||
)
|
||||
out_dir = stage_dir.outs[0]
|
||||
cache_dir = out_dir.cache_path
|
||||
dir_objs = {out_dir.obj}
|
||||
dir_hash = out_dir.hash_info
|
||||
dir_hashes = {dir_hash} | {
|
||||
entry_obj.hash_info for _, entry_obj in out_dir.obj
|
||||
@@ -72,23 +71,23 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
|
||||
def _check_status(status, **kwargs):
|
||||
for key in ("ok", "missing", "new", "deleted"):
|
||||
expected = kwargs.get(key, set())
|
||||
assert expected == {obj.hash_info for obj in getattr(status, key)}
|
||||
assert expected == set(getattr(status, key))
|
||||
|
||||
# Check status
|
||||
status = dvc.cloud.status(foo_objs)
|
||||
status = dvc.cloud.status(foo_hashes)
|
||||
_check_status(status, new={foo_hash})
|
||||
|
||||
status_dir = dvc.cloud.status(dir_objs)
|
||||
status_dir = dvc.cloud.status(dir_hashes)
|
||||
_check_status(status_dir, new=dir_hashes)
|
||||
|
||||
# Move cache and check status
|
||||
# See issue https://github.com/iterative/dvc/issues/4383 for details
|
||||
backup_dir = dvc.odb.local.cache_dir + ".backup"
|
||||
move(dvc.odb.local.cache_dir, backup_dir)
|
||||
status = dvc.cloud.status(foo_objs)
|
||||
status = dvc.cloud.status(foo_hashes)
|
||||
_check_status(status, missing={foo_hash})
|
||||
|
||||
status_dir = dvc.cloud.status(dir_objs)
|
||||
status_dir = dvc.cloud.status(dir_hashes)
|
||||
_check_status(status_dir, missing=dir_hashes)
|
||||
|
||||
# Restore original cache:
|
||||
@@ -96,42 +95,42 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
|
||||
move(backup_dir, dvc.odb.local.cache_dir)
|
||||
|
||||
# Push and check status
|
||||
dvc.cloud.push(foo_objs)
|
||||
dvc.cloud.push(foo_hashes)
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
|
||||
dvc.cloud.push(dir_objs)
|
||||
dvc.cloud.push(dir_hashes)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
status = dvc.cloud.status(foo_objs)
|
||||
status = dvc.cloud.status(foo_hashes)
|
||||
_check_status(status, ok={foo_hash})
|
||||
|
||||
status_dir = dvc.cloud.status(dir_objs)
|
||||
status_dir = dvc.cloud.status(dir_hashes)
|
||||
_check_status(status_dir, ok=dir_hashes)
|
||||
|
||||
# Remove and check status
|
||||
remove(dvc.odb.local.cache_dir)
|
||||
|
||||
status = dvc.cloud.status(foo_objs)
|
||||
status = dvc.cloud.status(foo_hashes)
|
||||
_check_status(status, deleted={foo_hash})
|
||||
|
||||
status_dir = dvc.cloud.status(dir_objs)
|
||||
status_dir = dvc.cloud.status(dir_hashes)
|
||||
_check_status(status_dir, deleted=dir_hashes)
|
||||
|
||||
# Pull and check status
|
||||
dvc.cloud.pull(foo_objs)
|
||||
dvc.cloud.pull(foo_hashes)
|
||||
assert os.path.exists(cache)
|
||||
assert os.path.isfile(cache)
|
||||
with open(cache) as fd:
|
||||
assert fd.read() == "foo"
|
||||
|
||||
dvc.cloud.pull(dir_objs)
|
||||
dvc.cloud.pull(dir_hashes)
|
||||
assert os.path.isfile(cache_dir)
|
||||
|
||||
status = dvc.cloud.status(foo_objs)
|
||||
status = dvc.cloud.status(foo_hashes)
|
||||
_check_status(status, ok={foo_hash})
|
||||
|
||||
status_dir = dvc.cloud.status(dir_objs)
|
||||
status_dir = dvc.cloud.status(dir_hashes)
|
||||
_check_status(status_dir, ok=dir_hashes)
|
||||
|
||||
|
||||
@@ -242,10 +241,13 @@ def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote):
|
||||
|
||||
|
||||
def test_missing_cache(tmp_dir, dvc, local_remote, caplog):
|
||||
from tests.utils import clean_staging
|
||||
|
||||
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
|
||||
|
||||
# purge cache
|
||||
remove(dvc.odb.local.cache_dir)
|
||||
clean_staging()
|
||||
|
||||
header = (
|
||||
"Some of the cache files do not exist "
|
||||
@@ -355,6 +357,8 @@ def test_pull_external_dvc_imports(tmp_dir, dvc, scm, erepo_dir):
|
||||
|
||||
|
||||
def clean(outs, dvc=None):
|
||||
from tests.utils import clean_staging
|
||||
|
||||
if dvc:
|
||||
outs = outs + [dvc.odb.local.cache_dir]
|
||||
for path in outs:
|
||||
@@ -363,6 +367,7 @@ def clean(outs, dvc=None):
|
||||
if dvc:
|
||||
os.makedirs(dvc.odb.local.cache_dir, exist_ok=True)
|
||||
clean_repos()
|
||||
clean_staging()
|
||||
|
||||
|
||||
def recurse_list_dir(d):
|
||||
|
||||
Reference in New Issue
Block a user