output/remote: get rid of NamedCache (#6008)

* objects: make HashInfo & HashFile hashable

* stage/output: collect objects (HashFiles) instead of NamedCache

* remote/data_cloud: use objs instead of NamedCache

* repo: use objects instead of NamedCache

* rerun black after rebase

* remove NamedCache usage from run-cache

* drop NamedCache

* update HashInfo.__hash__

* update unit tests

* revert remote_name change

* start cleaning up test failures

* fix tree obj expansion

* update missing cache logging

* fix external dict handling

* update naive odb.gc

* update state save behavior for pull

* update test_data_cloud

* update test_remote

* fix gc with multiple repos

* objects.tree: support filtering w/preserved hash_info

* fix stage cache restore

* drop fs.path_info from HashFile.__hash__

* objects: add optional name attribute

* revert missing cache logging name/suffix changes

* revert name changes in remote tests

* review fixes

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update dvc/remote/base.py

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Co-authored-by: Ruslan Kuprieiev <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Peter Rowlands (변기호)
2021-06-02 22:19:58 +03:00
committed by GitHub
co-authored by Ruslan Kuprieiev pre-commit-ci[bot]
parent 5fb181fcf8
commit eb46c03c6c
21 changed files with 402 additions and 344 deletions
+17 -17
View File
@@ -4,6 +4,7 @@ import shutil
import pytest
from dvc.exceptions import OutputDuplicationError
from dvc.hash_info import HashInfo
from dvc.repo import NotDvcRepoError, Repo, locked
@@ -44,26 +45,25 @@ def test_find_outs_by_path_does_graph_checks(tmp_dir, dvc):
[os.path.join("dir", "subdir", "file"), os.path.join("dir", "subdir")],
)
def test_used_cache(tmp_dir, dvc, path):
from dvc.objects.db import NamedCache
from dvc.objects.tree import Tree
tmp_dir.dvc_gen({"dir": {"subdir": {"file": "file"}, "other": "other"}})
expected = NamedCache.make(
"local", "70922d6bf66eb073053a82f77d58c536.dir", "dir"
)
expected.add_child_cache(
"70922d6bf66eb073053a82f77d58c536.dir",
NamedCache.make(
"local",
"8c7dd922ad47494fc02c388e12c00eac",
os.path.join("dir", "subdir", "file"),
),
)
used_cache = dvc.used_cache([path])
assert (
used_cache._items == expected._items
and used_cache.external == expected.external
)
expected_objs = {
HashInfo("md5", "70922d6bf66eb073053a82f77d58c536.dir"),
HashInfo("md5", "8c7dd922ad47494fc02c388e12c00eac"),
}
expected_external = {}
objs, external = dvc.used_cache([path])
used = set()
for obj in objs:
used.add(obj.hash_info)
if isinstance(obj, Tree):
used.update(entry_obj.hash_info for _, entry_obj in obj)
assert used == expected_objs
assert external == expected_external
def test_locked(mocker):