objects: use object IDs and references instead of naive objs in status/transfer (#6360)

* objects.status: use object IDs instead of naive objects

* objects.transfer: use object IDs instead of naive objects

* objects: add ReferenceHashFile

* odb: add ReferenceObjectDB

* repo: collect used object IDs instead of naive objs

* support basic/naive fs serialization

* objects.stage: stage file objects in addition to trees

* objects: remove save(), update transfer()

* output/dep: update for osave removal

* path_info: support both url and path attributes

* status/gc: use object IDs instead of naive objs

* diff: update staging usage

* commit: update staging usage

* tests: clean staging when needed

* tests: update for save/transfer/stage changes

* dvcfs: don't allow config serialization (repofs should be used instead)

* utils: return mtime as str, size as int

* tests: update unit tests for save/stage/transfer changes

* objects: use Repo.open when needed in ReferenceHashFile

* fix ls test staging

* objects.stage: don't skip hash check when loading from state

- individual hashes will still be loaded from state if they are still
  valid

* ReferenceHashFile: serialize repo fs paths relative to repo root

* preserve state when staging

* objects.tree: skip hash computation in digest() when we have already
loaded hash from state

* always check file existence in referencehashfile

* update repofs test since stage no longer allows nonexistent files

* odb: only protect on check() if hash was verified

* staging: use unique memfs namespace for staging per dest ODB

* tests: clean full staging namespace

* refodb: preserve repofs cache dir

* repodependency: stage imports to local ODB and strict check circular
imports

* fetch: fetch staging last

* staging: make dest odb mandatory and preserve state

* refodb: cache nonlocal fs's

* fix rebase errors

* repofs: support full instantiation from fs config

* repofs: preserve subrepo factory for erepos

* tests: clarify repofs dirty file behavior
This commit is contained in:
Peter Rowlands (변기호)
2021-08-04 13:55:52 +03:00
committed by GitHub
parent 34697ef298
commit 114a07e7a4
42 changed files with 945 additions and 523 deletions
+22 -17
View File
@@ -50,7 +50,7 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
out = stage.outs[0]
cache = out.cache_path
foo_hash = out.hash_info
foo_objs = out.get_used_objs().get(None, set())
foo_hashes = out.get_used_objs().get(None, set())
(stage_dir,) = tmp_dir.dvc_gen(
{
@@ -63,7 +63,6 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
)
out_dir = stage_dir.outs[0]
cache_dir = out_dir.cache_path
dir_objs = {out_dir.obj}
dir_hash = out_dir.hash_info
dir_hashes = {dir_hash} | {
entry_obj.hash_info for _, entry_obj in out_dir.obj
@@ -72,23 +71,23 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
def _check_status(status, **kwargs):
for key in ("ok", "missing", "new", "deleted"):
expected = kwargs.get(key, set())
assert expected == {obj.hash_info for obj in getattr(status, key)}
assert expected == set(getattr(status, key))
# Check status
status = dvc.cloud.status(foo_objs)
status = dvc.cloud.status(foo_hashes)
_check_status(status, new={foo_hash})
status_dir = dvc.cloud.status(dir_objs)
status_dir = dvc.cloud.status(dir_hashes)
_check_status(status_dir, new=dir_hashes)
# Move cache and check status
# See issue https://github.com/iterative/dvc/issues/4383 for details
backup_dir = dvc.odb.local.cache_dir + ".backup"
move(dvc.odb.local.cache_dir, backup_dir)
status = dvc.cloud.status(foo_objs)
status = dvc.cloud.status(foo_hashes)
_check_status(status, missing={foo_hash})
status_dir = dvc.cloud.status(dir_objs)
status_dir = dvc.cloud.status(dir_hashes)
_check_status(status_dir, missing=dir_hashes)
# Restore original cache:
@@ -96,42 +95,42 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
move(backup_dir, dvc.odb.local.cache_dir)
# Push and check status
dvc.cloud.push(foo_objs)
dvc.cloud.push(foo_hashes)
assert os.path.exists(cache)
assert os.path.isfile(cache)
dvc.cloud.push(dir_objs)
dvc.cloud.push(dir_hashes)
assert os.path.isfile(cache_dir)
status = dvc.cloud.status(foo_objs)
status = dvc.cloud.status(foo_hashes)
_check_status(status, ok={foo_hash})
status_dir = dvc.cloud.status(dir_objs)
status_dir = dvc.cloud.status(dir_hashes)
_check_status(status_dir, ok=dir_hashes)
# Remove and check status
remove(dvc.odb.local.cache_dir)
status = dvc.cloud.status(foo_objs)
status = dvc.cloud.status(foo_hashes)
_check_status(status, deleted={foo_hash})
status_dir = dvc.cloud.status(dir_objs)
status_dir = dvc.cloud.status(dir_hashes)
_check_status(status_dir, deleted=dir_hashes)
# Pull and check status
dvc.cloud.pull(foo_objs)
dvc.cloud.pull(foo_hashes)
assert os.path.exists(cache)
assert os.path.isfile(cache)
with open(cache) as fd:
assert fd.read() == "foo"
dvc.cloud.pull(dir_objs)
dvc.cloud.pull(dir_hashes)
assert os.path.isfile(cache_dir)
status = dvc.cloud.status(foo_objs)
status = dvc.cloud.status(foo_hashes)
_check_status(status, ok={foo_hash})
status_dir = dvc.cloud.status(dir_objs)
status_dir = dvc.cloud.status(dir_hashes)
_check_status(status_dir, ok=dir_hashes)
@@ -242,10 +241,13 @@ def test_hash_recalculation(mocker, dvc, tmp_dir, local_remote):
def test_missing_cache(tmp_dir, dvc, local_remote, caplog):
from tests.utils import clean_staging
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
# purge cache
remove(dvc.odb.local.cache_dir)
clean_staging()
header = (
"Some of the cache files do not exist "
@@ -355,6 +357,8 @@ def test_pull_external_dvc_imports(tmp_dir, dvc, scm, erepo_dir):
def clean(outs, dvc=None):
from tests.utils import clean_staging
if dvc:
outs = outs + [dvc.odb.local.cache_dir]
for path in outs:
@@ -363,6 +367,7 @@ def clean(outs, dvc=None):
if dvc:
os.makedirs(dvc.odb.local.cache_dir, exist_ok=True)
clean_repos()
clean_staging()
def recurse_list_dir(d):