mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-09 23:10:24 +01:00
Peter Rowlands (변기호)
71bae7e4af
* fs: use fsspecwrapper in memfs * odb: add memfs staging ODB in ODBManager * odb: add read_only attribute for odb instances * objects: handle memfs paths when staging trees * odb: remove git ODB * output: use staging ODB * RepoDependency: use staging odb for imports * dvcfs: handle case where dir cache is unavailable * fix circular import check * odb: use temp local ODB for staging * objects: use staging ODB in stage() and tree load * objects: add typing for add/check/load/stage * output/dep: use new staging * add unit tests for local odb staging * localfs: account for string paths in move() * include local in odb.by_scheme * tests: count dir cache like any other object in test_pull_git_imports * odb: clear staging on gc() * objects: add tree.get/filter/insert * output: use obj.filter/get for granular dir commit * squash filter logic output * dependency: don't filter imports at the dep level * don't init staging ODB for external cache * use staging ODB when computing tree hash for external outs * remove unneeded force in test granular commit * use unique-per-manager memfs URLs for staging * odb: make staging per-ODBManager * objects: move state dependent staging into ODBManager * objects: support checking multiple odbs - staging dependent checks can be done via ODBManager.check * checkout: do state lookup before explicit stage() in _changed() * output: use odbmanager based staging * RepoDependency: use odbmanager based staging * update obj staging unit tests * checkout: remove check(), handle individual file exceptions on link() * diff: use odbmanager based staging * objects: add ObjectPermissionError for read/write perm errors * odb: move staging out of odbmanager * objects: handle staging ODB in objects.stage() * objects.tree: don't stage anything in tree methods * odb: clear staged objects on gc() * remote: don't filter objs by scheme, objs are already grouped by ODB * diff: update objects.stage usage * output/dep: update stage() usage * update ODB tests * update func tests * remove memfs from FS_MAP * revert unneeded objects.check() changes * remove unused tree.insert() * revert unused odbmanager changes * add separate odb error classes * stage external outs directly in ODB * check for obj existence in main ODB before staging * use .dvc/tmp/staging instead of per-odb staging * gc: only gc loal staging once
68 lines
1.8 KiB
Python
68 lines
1.8 KiB
Python
import logging
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from typing import TYPE_CHECKING, Optional
|
|
|
|
from dvc.progress import Tqdm
|
|
|
|
from .tree import Tree
|
|
|
|
if TYPE_CHECKING:
|
|
from dvc.hash_info import HashInfo
|
|
|
|
from .db.base import ObjectDB
|
|
from .file import HashFile
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def save(
|
|
odb: "ObjectDB",
|
|
obj: "HashFile",
|
|
jobs: Optional[int] = None,
|
|
**kwargs,
|
|
):
|
|
if isinstance(obj, Tree):
|
|
from .stage import get_staging
|
|
|
|
with ThreadPoolExecutor(max_workers=jobs) as executor:
|
|
for future in Tqdm(
|
|
as_completed(
|
|
executor.submit(
|
|
odb.add,
|
|
entry.path_info,
|
|
entry.fs,
|
|
entry.hash_info,
|
|
**kwargs,
|
|
)
|
|
for _, entry in obj
|
|
),
|
|
desc="Saving files",
|
|
total=len(obj),
|
|
unit="file",
|
|
):
|
|
future.result()
|
|
|
|
# if dir cache for this tree has already been staged, move the
|
|
# staged object when possible
|
|
staging = get_staging(odb)
|
|
if staging.exists(obj.hash_info):
|
|
obj = staging.get(obj.hash_info)
|
|
kwargs["move"] = True
|
|
|
|
assert obj.path_info and obj.fs and obj.hash_info
|
|
odb.add(obj.path_info, obj.fs, obj.hash_info, **kwargs)
|
|
|
|
|
|
def check(odb: "ObjectDB", obj: "HashFile"):
|
|
if isinstance(obj, Tree):
|
|
for _, entry in obj:
|
|
odb.check(entry.hash_info)
|
|
|
|
odb.check(obj.hash_info)
|
|
|
|
|
|
def load(odb: "ObjectDB", hash_info: "HashInfo") -> "HashFile":
|
|
if hash_info.isdir:
|
|
return Tree.load(odb, hash_info)
|
|
return odb.get(hash_info)
|