gc: Add --not-in-remote arg.

Closes #2036
This commit is contained in:
David de la Iglesia Castro
2023-04-27 10:30:39 +02:00
parent a4cc198ac7
commit c7e2ec9707
4 changed files with 127 additions and 5 deletions
+9
View File
@@ -22,6 +22,8 @@ class CmdGC(CmdBase):
workspace=self.args.workspace,
rev=self.args.rev,
num=self.args.num,
cloud=self.args.cloud,
not_in_remote=self.args.not_in_remote,
)
if self.args.rev:
@@ -74,6 +76,7 @@ class CmdGC(CmdBase):
workspace=self.args.workspace,
rev=self.args.rev,
num=self.args.num,
not_in_remote=self.args.not_in_remote,
)
return 0
@@ -157,6 +160,12 @@ def add_parser(subparsers, parent_parser):
default=False,
help="Keep data files for all experiments.",
)
gc_parser.add_argument(
"--not-in-remote",
action="store_true",
default=False,
help="Keep data files that are not present in the remote.",
)
gc_parser.add_argument(
"-c",
"--cloud",
+18 -4
View File
@@ -12,6 +12,12 @@ logger = logging.getLogger(__name__)
def _validate_args(**kwargs):
not_in_remote = kwargs.pop("not_in_remote", None)
cloud = kwargs.pop("cloud", None)
if not_in_remote and cloud:
raise InvalidArgumentError(
"`--not-in-remote` and `--cloud` are mutually exclusive"
)
if not any(kwargs.values()):
raise InvalidArgumentError(
"Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags` "
@@ -23,7 +29,7 @@ def _validate_args(**kwargs):
@locked
def gc( # noqa: PLR0913
def gc( # noqa: PLR0913, C901
self: "Repo",
all_branches: bool = False,
cloud: bool = False,
@@ -39,6 +45,7 @@ def gc( # noqa: PLR0913
commit_date: Optional[str] = None,
rev: Optional[str] = None,
num: Optional[int] = None,
not_in_remote: bool = False,
):
# require `workspace` to be true to come into effect.
# assume `workspace` to be enabled if any of `all_tags`, `all_commits`,
@@ -52,6 +59,8 @@ def gc( # noqa: PLR0913
commit_date=commit_date,
rev=rev,
num=num,
cloud=cloud,
not_in_remote=not_in_remote,
)
from contextlib import ExitStack
@@ -85,6 +94,11 @@ def gc( # noqa: PLR0913
).values():
used_obj_ids.update(obj_ids)
if not_in_remote:
cloud_odb = self.cloud.get_remote_odb(remote, "gc --not-in-remote")
remote_hashes = list(cloud_odb.all(jobs=jobs))
used_obj_ids = {x for x in used_obj_ids if x.value not in remote_hashes}
for scheme, odb in self.cache.by_scheme():
if not odb:
continue
@@ -96,9 +110,9 @@ def gc( # noqa: PLR0913
if not cloud:
return
odb = self.cloud.get_remote_odb(remote, "gc -c")
removed = ogc(odb, used_obj_ids, jobs=jobs)
cloud_odb = self.cloud.get_remote_odb(remote, "gc -c")
removed = ogc(cloud_odb, used_obj_ids, jobs=jobs)
if removed:
get_index(odb).clear()
get_index(cloud_odb).clear()
else:
logger.info("No unused cache to remove from remote.")