From 6cada6dc12cc03c4a538021808db9bf3c6921613 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 29 Jul 2022 20:12:40 +0300 Subject: [PATCH] dvc: cloud versioning POC --- dvc/config_schema.py | 1 + dvc/data_cloud.py | 46 ++++++++++++++++++++++++++++++++++---------- dvc/repo/fetch.py | 37 +++++++++++++++++++++++++++++++++++ dvc/repo/index.py | 1 + dvc/repo/push.py | 29 ++++++++++++++++++++++++++++ 5 files changed, 104 insertions(+), 10 deletions(-) diff --git a/dvc/config_schema.py b/dvc/config_schema.py index 0d6ff5ee9..13590cf97 100644 --- a/dvc/config_schema.py +++ b/dvc/config_schema.py @@ -78,6 +78,7 @@ REMOTE_COMMON = { "url": str, "checksum_jobs": All(Coerce(int), Range(1)), "jobs": All(Coerce(int), Range(1)), + Optional("worktree"): Bool, Optional("no_traverse"): Bool, # obsoleted Optional("version_aware"): Bool, } diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 8f76a55f9..ec495b18e 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -3,6 +3,8 @@ import logging from typing import TYPE_CHECKING, Iterable, Optional +from funcy import cached_property + from dvc_data.hashfile.db import get_index if TYPE_CHECKING: @@ -13,6 +15,25 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +class Remote: + def __init__(self, path, fs, **config): + self.path = path + self.fs = fs + + self.worktree = config.pop("worktree", False) + self.config = config + + @cached_property + def odb(self): + from dvc_data.hashfile.db import get_odb + + path = self.path + if self.worktree: + path = self.fs.path.join(path, ".dvc", "cache") + + return get_odb(self.fs, path, **self.config) + + class DataCloud: """Class that manages dvc remotes. @@ -27,18 +48,23 @@ class DataCloud: def __init__(self, repo): self.repo = repo - def get_remote_odb( + def get_remote( self, name: Optional[str] = None, command: str = "", - ) -> "HashFileDB": + ) -> "Remote": from dvc.config import NoRemoteError if not name: name = self.repo.config["core"].get("remote") if name: - return self._init_odb(name) + from dvc.fs import get_cloud_fs + + cls, config, fs_path = get_cloud_fs(self.repo, name=name) + fs = cls(**config) + config["tmp_dir"] = self.repo.index_db_dir + return Remote(fs_path, fs, **config) if bool(self.repo.config["remote"]): error_msg = ( @@ -55,13 +81,13 @@ class DataCloud: raise NoRemoteError(error_msg) - def _init_odb(self, name): - from dvc.fs import get_cloud_fs - from dvc_data.hashfile.db import get_odb - - cls, config, fs_path = get_cloud_fs(self.repo, name=name) - config["tmp_dir"] = self.repo.index_db_dir - return get_odb(cls(**config), fs_path, **config) + def get_remote_odb( + self, + name: Optional[str] = None, + command: str = "", + ) -> "HashFileDB": + remote = self.get_remote(name=name, command=command) + return remote.odb def _log_missing(self, status: "CompareStatusResult"): if status.missing: diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index 6a7b712df..3183bb683 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -1,6 +1,8 @@ import logging +from contextlib import suppress from typing import TYPE_CHECKING, Optional +from dvc.config import NoRemoteError from dvc.exceptions import DownloadError, FileTransferError from dvc.fs import Schemes @@ -12,6 +14,36 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +def _fetch_worktree(repo, remote): + from dvc_data.index import save + + index = repo.index.data["repo"] + for key, entry in index.iteritems(): + entry.fs = remote.fs + entry.path = remote.fs.path.join( + remote.path, + *key, + ) + save(index) + + for stage in repo.index.stages: + for out in stage.outs: + if not out.use_cache: + continue + + if not out.is_in_repo: + continue + + key = repo.fs.path.relparts(out.fs_path, repo.root_dir) + entry = repo.index.data["repo"][key] + out.hash_info = entry.hash_info + out.meta = entry.meta + + stage.dvcfile.dump(stage) + + return len(index) + + @locked def fetch( self, @@ -42,6 +74,11 @@ def fetch( if isinstance(targets, str): targets = [targets] + with suppress(NoRemoteError): + _remote = self.cloud.get_remote(name=remote) + if _remote.worktree: + return _fetch_worktree(self, _remote) + used = self.used_objs( targets, all_branches=all_branches, diff --git a/dvc/repo/index.py b/dvc/repo/index.py index acaa2e944..9b2b397d0 100644 --- a/dvc/repo/index.py +++ b/dvc/repo/index.py @@ -205,6 +205,7 @@ class Index: obj=out.obj, hash_info=out.hash_info, odb=out.odb, + cache=out.odb, remote=remote, ) diff --git a/dvc/repo/push.py b/dvc/repo/push.py index 04404c705..2b4ee07c2 100644 --- a/dvc/repo/push.py +++ b/dvc/repo/push.py @@ -9,6 +9,31 @@ if TYPE_CHECKING: from dvc_objects.db.base import ObjectDB +def _push_worktree(repo, remote): + from dvc_data.index import build, checkout + + index = repo.index.data["repo"] + checkout(index, remote.path, remote.fs) + build(index, remote.path, remote.fs) + + for stage in repo.index.stages: + for out in stage.outs: + if not out.use_cache: + continue + + if not out.is_in_repo: + continue + + key = repo.fs.path.relparts(out.fs_path, repo.root_dir) + entry = repo.index.data["repo"][key] + out.hash_info = entry.hash_info + out.meta = entry.meta + + stage.dvcfile.dump(stage) + + return len(index) + + @locked def push( self, @@ -26,6 +51,10 @@ def push( odb: Optional["ObjectDB"] = None, include_imports=False, ): + _remote = self.cloud.get_remote(name=remote) + if _remote.worktree: + return _push_worktree(self, _remote) + used_run_cache = ( self.stage_cache.push(remote, odb=odb) if run_cache else [] )