mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-10 15:28:34 +01:00
Batuhan TaskayaandRuslan Kuprieiev
68897aa0de
* objects: merge transfer() with stage() * objects: drop streamfile * objects: preserve the signature of get_file_hash * objects: override hash_jobs if jobs= passed * objects: support no progress bar directive * tests: spy on _get_file_hash * objects: update=... directive * update: only transfer missing caches with --to-remote * tests: remove local_remote from the primary test (due to it moves) * objects: add move=True/False option * objects: implement _get_file_obj on top of get_file_hash * objects: use threading on save() * objects: pass jobs= on to-cache as well * objects: use in place tqdm * objects: check for FileExistsError * Update dvc/objects/__init__.py Co-authored-by: Ruslan Kuprieiev <[email protected]> Co-authored-by: Ruslan Kuprieiev <[email protected]>
46 lines
1.1 KiB
Python
46 lines
1.1 KiB
Python
import logging
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
from dvc.progress import Tqdm
|
|
|
|
from .tree import Tree
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def save(odb, obj, jobs=None, **kwargs):
|
|
if isinstance(obj, Tree):
|
|
with ThreadPoolExecutor(max_workers=jobs) as executor:
|
|
for future in Tqdm(
|
|
as_completed(
|
|
executor.submit(
|
|
odb.add,
|
|
entry.path_info,
|
|
entry.fs,
|
|
entry.hash_info,
|
|
**kwargs
|
|
)
|
|
for _, entry in obj
|
|
),
|
|
desc="Saving files",
|
|
total=len(obj),
|
|
unit="file",
|
|
):
|
|
future.result()
|
|
|
|
odb.add(obj.path_info, obj.fs, obj.hash_info, **kwargs)
|
|
|
|
|
|
def check(odb, obj):
|
|
odb.check(obj.hash_info)
|
|
|
|
if isinstance(obj, Tree):
|
|
for _, entry in obj:
|
|
odb.check(entry.hash_info)
|
|
|
|
|
|
def load(odb, hash_info):
|
|
if hash_info.isdir:
|
|
return Tree.load(odb, hash_info)
|
|
return odb.get(hash_info)
|