Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

repo: data_index: fallback to in-memory

Browse Source 
Makes for a much simpler resource management elsewhere.
This commit is contained in:
Ruslan Kuprieiev
2023-03-11 01:23:12 +02:00
parent 8ed9b911db
commit a0bf2d997a
2 changed files with 22 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/repo/__init__.py
+7 -7
View File
@@ -397,17 +397,17 @@ class Repo:
self._reset()
@property
def data_index(self) -> Optional["DataIndex"]:
def data_index(self) -> "DataIndex":
from dvc_data.index import DataIndex
if not self.index_db_dir:
return None
if self._data_index is None:
index_dir = os.path.join(self.index_db_dir, "index", "data")
os.makedirs(index_dir, exist_ok=True)
if self.index_db_dir:
index_dir = os.path.join(self.index_db_dir, "index", "data")
os.makedirs(index_dir, exist_ok=True)
self._data_index = DataIndex.open(os.path.join(index_dir, "db.db"))
self._data_index = DataIndex.open(os.path.join(index_dir, "db.db"))
else:
self._data_index = DataIndex()
return self._data_index
dvc/repo/index.py
+15 -23
View File
@@ -354,42 +354,34 @@ class Index:
@cached_property
def data(self) -> "Dict[str, DataIndex]":
from dvc_data.index import DataIndex
prefix: "DataIndexKey"
loaded = False
index = self.repo.data_index
if index is None:
index = DataIndex()
prefix = ("tree", self.data_tree.hash_info.value)
if index.has_node(prefix):
loaded = True
try:
if not loaded:
_load_data_from_outs(index, prefix, self.outs)
index.commit()
if not loaded:
_load_data_from_outs(index, prefix, self.outs)
index.commit()
by_workspace = {}
by_workspace["repo"] = index.view((*prefix, "repo"))
by_workspace["local"] = index.view((*prefix, "local"))
by_workspace = {}
by_workspace["repo"] = index.view((*prefix, "repo"))
by_workspace["local"] = index.view((*prefix, "local"))
for out in self.outs:
if not out.use_cache:
continue
for out in self.outs:
if not out.use_cache:
continue
ws, key = out.index_key
if ws not in by_workspace:
by_workspace[ws] = index.view((*prefix, ws))
ws, key = out.index_key
if ws not in by_workspace:
by_workspace[ws] = index.view((*prefix, ws))
data_index = by_workspace[ws]
_load_storage_from_out(data_index.storage_map, key, out)
data_index = by_workspace[ws]
_load_storage_from_out(data_index.storage_map, key, out)
return by_workspace
finally:
index.close()
return by_workspace
@staticmethod
def _hash_targets(