exp: run: don't relink during commit

Part of #8809
This commit is contained in:
Ruslan Kuprieiev
2023-02-24 08:09:35 +02:00
parent ad5b91e51a
commit 81da91a973
4 changed files with 28 additions and 18 deletions
+19 -15
View File
@@ -709,7 +709,7 @@ class Output:
except _CheckoutError as exc:
raise CheckoutError(exc.paths, {}) # noqa: B904
def commit(self, filter_info=None) -> None:
def commit(self, filter_info=None, relink=True) -> None:
if not self.exists:
raise self.DoesNotExistError(self)
@@ -719,8 +719,11 @@ class Output:
granular = (
self.is_dir_checksum and filter_info and filter_info != self.fs_path
)
# NOTE: trying to use hardlink during transfer only if we will be
# relinking later
hardlink = relink
if granular:
obj = self._commit_granular_dir(filter_info)
obj = self._commit_granular_dir(filter_info, hardlink)
else:
staging, _, obj = build(
self.cache,
@@ -734,20 +737,21 @@ class Output:
self.cache,
{obj.hash_info},
shallow=False,
hardlink=True,
hardlink=hardlink,
)
self._checkout(
filter_info or self.fs_path,
self.fs,
obj,
self.cache,
relink=True,
state=self.repo.state,
prompt=prompt.confirm,
)
self.set_exec()
if relink:
self._checkout(
filter_info or self.fs_path,
self.fs,
obj,
self.cache,
relink=True,
state=self.repo.state,
prompt=prompt.confirm,
)
self.set_exec()
def _commit_granular_dir(self, filter_info) -> Optional["HashFile"]:
def _commit_granular_dir(self, filter_info, hardlink) -> Optional["HashFile"]:
prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path))
staging, _, save_obj = build(
self.cache,
@@ -764,7 +768,7 @@ class Output:
self.cache,
{save_obj.hash_info} | {oid for _, _, oid in save_obj},
shallow=True,
hardlink=True,
hardlink=hardlink,
)
return checkout_obj
+6 -1
View File
@@ -43,6 +43,7 @@ def commit(
force=False,
allow_missing=False,
data_only=False,
relink=True,
):
stages_info = [
info
@@ -60,6 +61,10 @@ def commit(
if any(changes):
prompt_to_commit(stage, changes, force=force)
stage.save(allow_missing=allow_missing)
stage.commit(filter_info=stage_info.filter_info, allow_missing=allow_missing)
stage.commit(
filter_info=stage_info.filter_info,
allow_missing=allow_missing,
relink=relink,
)
stage.dump(update_pipeline=False)
return [s.stage for s in stages_info]
+1
View File
@@ -472,6 +472,7 @@ class BaseStashQueue(ABC):
force=True,
allow_missing=True,
data_only=True,
relink=False,
)
@staticmethod
+2 -2
View File
@@ -545,13 +545,13 @@ class Stage(params.StageParams):
)
@rwlocked(write=["outs"])
def commit(self, allow_missing=False, filter_info=None) -> None:
def commit(self, allow_missing=False, filter_info=None, **kwargs) -> None:
from dvc.output import OutputDoesNotExistError
link_failures = []
for out in self.filter_outs(filter_info):
try:
out.commit(filter_info=filter_info)
out.commit(filter_info=filter_info, **kwargs)
except OutputDoesNotExistError:
if not (allow_missing or out.checkpoint):
raise