index: load meta from outs.files

This commit is contained in:
Ruslan Kuprieiev
2023-12-22 05:26:18 +02:00
parent 98a286d88d
commit 6a44783d72
4 changed files with 49 additions and 11 deletions
+15
View File
@@ -129,10 +129,25 @@ def _load_data_from_outs(index, prefix, outs):
for key_len in range(1, len(key)):
parents.add((ws, key[:key_len]))
loaded = None
if out.files:
loaded = True
for okey, ometa, ohi in out.get_obj():
for key_len in range(1, len(okey)):
parents.add((ws, (*key, *okey[:key_len])))
fkey = (*key, *okey)
index[(*prefix, ws, *fkey)] = DataIndexEntry(
key=fkey,
meta=ometa,
hash_info=ohi,
)
entry = DataIndexEntry(
key=key,
meta=out.meta,
hash_info=out.hash_info,
loaded=loaded,
)
if (
+8
View File
@@ -32,6 +32,14 @@ class Cloud(ABC):
def write_bytes(self, contents):
raise NotImplementedError
@abstractmethod
def unlink(self, missing_ok: bool = False) -> None:
pass
@abstractmethod
def rmdir(self, recursive: bool = True) -> None:
pass
def read_text(self, encoding=None, errors=None):
if not encoding:
encoding = locale.getpreferredencoding(False)
+25 -10
View File
@@ -171,7 +171,7 @@ class TestRemoteVersionAware:
(stage,) = tmp_dir.dvc_gen("foo", "foo")
run_copy("foo", "foo_copy", name="copy")
dvc.push()
assert dvc.push()
assert (remote_version_aware / "foo").read_text() == "foo"
assert (remote_version_aware / "foo_copy").read_text() == "foo"
foo_dvc = (tmp_dir / "foo.dvc").read_text()
@@ -185,26 +185,26 @@ class TestRemoteVersionAware:
remove(tmp_dir / "foo")
remove(tmp_dir / "foo_copy")
dvc.pull()
assert dvc.pull()
assert (tmp_dir / "foo").read_text() == "foo"
assert (tmp_dir / "foo_copy").read_text() == "foo"
assert (tmp_dir / "foo.dvc").read_text() == foo_dvc
assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
dvc.push()
assert not dvc.push()
assert (remote_version_aware / "foo").read_text() == "foo"
assert (remote_version_aware / "foo_copy").read_text() == "foo"
assert (tmp_dir / "foo.dvc").read_text() == foo_dvc
assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
dvc.reproduce()
dvc.push()
assert not dvc.push()
assert (remote_version_aware / "foo").read_text() == "foo"
assert (remote_version_aware / "foo_copy").read_text() == "foo"
assert (tmp_dir / "foo.dvc").read_text() == foo_dvc
assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
def test_dir(self, tmp_dir, dvc, run_copy, remote_version_aware):
def test_dir(self, tmp_dir, dvc, run_copy, remote_version_aware): # noqa: PLR0915
(stage,) = tmp_dir.dvc_gen(
{
"data_dir": {
@@ -215,7 +215,7 @@ class TestRemoteVersionAware:
}
)
dvc.push()
assert dvc.push()
data_dir_dvc = (tmp_dir / "data_dir.dvc").read_text()
assert "files" in data_dir_dvc
@@ -230,7 +230,7 @@ class TestRemoteVersionAware:
remove(dvc.cache.local.path)
remove(tmp_dir / "data_dir")
dvc.pull()
assert dvc.pull()
assert (tmp_dir / "data_dir" / "data").read_text() == "data"
assert (
tmp_dir / "data_dir" / "data_sub_dir" / "data_sub"
@@ -240,7 +240,7 @@ class TestRemoteVersionAware:
run_copy("data_dir", "data_dir_copy", name="copy")
dvc_lock = (tmp_dir / "dvc.lock").read_text()
dvc.push()
assert dvc.push()
assert (remote_version_aware / "data_dir").exists()
assert (remote_version_aware / "data_dir" / "data").exists()
assert (remote_version_aware / "data_dir_copy").exists()
@@ -249,7 +249,7 @@ class TestRemoteVersionAware:
assert (tmp_dir / "dvc.lock").read_text() != dvc_lock
dvc_lock = (tmp_dir / "dvc.lock").read_text()
dvc.push()
assert not dvc.push()
assert (remote_version_aware / "data_dir").exists()
assert (remote_version_aware / "data_dir" / "data").exists()
assert (remote_version_aware / "data_dir_copy").exists()
@@ -259,7 +259,8 @@ class TestRemoteVersionAware:
dvc.cache.local.clear()
remove(tmp_dir / "data_dir")
dvc.push()
remove(tmp_dir / "data_dir_copy")
assert not dvc.push()
assert (remote_version_aware / "data_dir").exists()
assert (remote_version_aware / "data_dir" / "data").exists()
assert (remote_version_aware / "data_dir_copy").exists()
@@ -267,6 +268,20 @@ class TestRemoteVersionAware:
assert (tmp_dir / "data_dir.dvc").read_text() == data_dir_dvc
assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
(remote_version_aware / "data_dir").rmdir()
(remote_version_aware / "data_dir_copy").rmdir()
assert not (remote_version_aware / "data_dir").exists()
assert not (remote_version_aware / "data_dir_copy").exists()
assert dvc.pull()
assert (tmp_dir / "data_dir" / "data").read_text() == "data"
assert (
tmp_dir / "data_dir" / "data_sub_dir" / "data_sub"
).read_text() == "data_sub"
assert (tmp_dir / "data_dir_copy" / "data").read_text() == "data"
assert (
tmp_dir / "data_dir_copy" / "data_sub_dir" / "data_sub"
).read_text() == "data_sub"
class TestRemoteWorktree:
def test_file(self, tmp_dir, dvc, remote_worktree):
+1 -1
View File
@@ -35,7 +35,7 @@ dependencies = [
"configobj>=5.0.6",
"distro>=1.3",
"dpath<3,>=2.1.0",
"dvc-data>=3.1.1,<3.2",
"dvc-data>=3.2,<3.3",
"dvc-http>=2.29.0",
"dvc-render>=1.0.0,<2",
"dvc-studio-client>=0.17.1,<1",