From dae500d7c75ddf9c609bacd493238814328087e6 Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Fri, 18 Nov 2022 17:18:29 +0900 Subject: [PATCH] worktree: use view instead of full index in push/fetch --- dvc/repo/fetch.py | 2 +- dvc/repo/index.py | 2 +- dvc/repo/push.py | 2 +- dvc/repo/worktree.py | 51 ++++++++++++++++++++++++++++++++++++++------ 4 files changed, 48 insertions(+), 9 deletions(-) diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index 269d16bdd..0e3cf3397 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -45,7 +45,7 @@ def fetch( remote is configured """ from dvc.repo.imports import save_imports - from dvc.repo.worktree import fetch as fetch_worktree + from dvc.repo.worktree import fetch_worktree from dvc_data.hashfile.transfer import TransferResult if isinstance(targets, str): diff --git a/dvc/repo/index.py b/dvc/repo/index.py index 4c6be0cf5..221ae5bfd 100644 --- a/dvc/repo/index.py +++ b/dvc/repo/index.py @@ -152,7 +152,7 @@ class Index: def targets_view( self, - targets: "TargetType", + targets: Optional["TargetType"], stage_filter: Optional[Callable[["Stage"], bool]] = None, outs_filter: Optional[Callable[["Output"], bool]] = None, **kwargs: Any, diff --git a/dvc/repo/push.py b/dvc/repo/push.py index c0ef6d7d1..e8ce2f1c5 100644 --- a/dvc/repo/push.py +++ b/dvc/repo/push.py @@ -26,7 +26,7 @@ def push( odb: Optional["ObjectDB"] = None, include_imports=False, ): - from dvc.repo.worktree import push as push_worktree + from dvc.repo.worktree import push_worktree _remote = self.cloud.get_remote(name=remote) if _remote.worktree: diff --git a/dvc/repo/worktree.py b/dvc/repo/worktree.py index 10f93f5a7..6b80caac2 100644 --- a/dvc/repo/worktree.py +++ b/dvc/repo/worktree.py @@ -1,14 +1,52 @@ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from dvc.cloud import Remote + from dvc.index import Index, IndexView + from dvc.output import Output from dvc.repo import Repo + from dvc.stage import Stage + from dvc.types import TargetType -def fetch(repo: "Repo", remote: "Remote") -> int: +def worktree_view( + index: "Index", + targets: Optional["TargetType"] = None, + push: bool = False, + **kwargs, +) -> "IndexView": + """Return view of data that can be stored in worktree remotes. + + Args: + targets: Optional targets. + push: Whether the view should be restricted to pushable data only. + + Additional kwargs will be passed into target collection. + """ + + def stage_filter(stage: "Stage") -> bool: + if push and stage.is_repo_import: + return False + return True + + def outs_filter(out: "Output") -> bool: + if not out.is_in_repo or (push and not out.can_push): + return False + return True + + return index.targets_view( + targets, + stage_filter=stage_filter, + outs_filter=outs_filter, + **kwargs, + ) + + +def fetch_worktree(repo: "Repo", remote: "Remote") -> int: from dvc_data.index import save - index = repo.index.data["repo"] + view = worktree_view(repo.index) + index = view.data["repo"] for key, entry in index.iteritems(): entry.fs = remote.fs entry.path = remote.fs.path.join( @@ -19,14 +57,15 @@ def fetch(repo: "Repo", remote: "Remote") -> int: return len(index) -def push(repo: "Repo", remote: "Remote") -> int: +def push_worktree(repo: "Repo", remote: "Remote") -> int: from dvc_data.index import checkout from dvc_data.index.save import build_tree - index = repo.index.data["repo"] + view = worktree_view(repo.index, push=True) + index = view.data["repo"] checkout(index, remote.path, remote.fs) - for stage in repo.index.stages: + for stage in view.stages: for out in stage.outs: if not out.use_cache: continue