dvc: cloud versioning POC

This commit is contained in:
Ruslan Kuprieiev
2022-09-07 21:43:34 +03:00
parent 89d4461908
commit 6cada6dc12
5 changed files with 104 additions and 10 deletions
+1
View File
@@ -78,6 +78,7 @@ REMOTE_COMMON = {
"url": str,
"checksum_jobs": All(Coerce(int), Range(1)),
"jobs": All(Coerce(int), Range(1)),
Optional("worktree"): Bool,
Optional("no_traverse"): Bool, # obsoleted
Optional("version_aware"): Bool,
}
+36 -10
View File
@@ -3,6 +3,8 @@
import logging
from typing import TYPE_CHECKING, Iterable, Optional
from funcy import cached_property
from dvc_data.hashfile.db import get_index
if TYPE_CHECKING:
@@ -13,6 +15,25 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
class Remote:
def __init__(self, path, fs, **config):
self.path = path
self.fs = fs
self.worktree = config.pop("worktree", False)
self.config = config
@cached_property
def odb(self):
from dvc_data.hashfile.db import get_odb
path = self.path
if self.worktree:
path = self.fs.path.join(path, ".dvc", "cache")
return get_odb(self.fs, path, **self.config)
class DataCloud:
"""Class that manages dvc remotes.
@@ -27,18 +48,23 @@ class DataCloud:
def __init__(self, repo):
self.repo = repo
def get_remote_odb(
def get_remote(
self,
name: Optional[str] = None,
command: str = "<command>",
) -> "HashFileDB":
) -> "Remote":
from dvc.config import NoRemoteError
if not name:
name = self.repo.config["core"].get("remote")
if name:
return self._init_odb(name)
from dvc.fs import get_cloud_fs
cls, config, fs_path = get_cloud_fs(self.repo, name=name)
fs = cls(**config)
config["tmp_dir"] = self.repo.index_db_dir
return Remote(fs_path, fs, **config)
if bool(self.repo.config["remote"]):
error_msg = (
@@ -55,13 +81,13 @@ class DataCloud:
raise NoRemoteError(error_msg)
def _init_odb(self, name):
from dvc.fs import get_cloud_fs
from dvc_data.hashfile.db import get_odb
cls, config, fs_path = get_cloud_fs(self.repo, name=name)
config["tmp_dir"] = self.repo.index_db_dir
return get_odb(cls(**config), fs_path, **config)
def get_remote_odb(
self,
name: Optional[str] = None,
command: str = "<command>",
) -> "HashFileDB":
remote = self.get_remote(name=name, command=command)
return remote.odb
def _log_missing(self, status: "CompareStatusResult"):
if status.missing:
+37
View File
@@ -1,6 +1,8 @@
import logging
from contextlib import suppress
from typing import TYPE_CHECKING, Optional
from dvc.config import NoRemoteError
from dvc.exceptions import DownloadError, FileTransferError
from dvc.fs import Schemes
@@ -12,6 +14,36 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
def _fetch_worktree(repo, remote):
from dvc_data.index import save
index = repo.index.data["repo"]
for key, entry in index.iteritems():
entry.fs = remote.fs
entry.path = remote.fs.path.join(
remote.path,
*key,
)
save(index)
for stage in repo.index.stages:
for out in stage.outs:
if not out.use_cache:
continue
if not out.is_in_repo:
continue
key = repo.fs.path.relparts(out.fs_path, repo.root_dir)
entry = repo.index.data["repo"][key]
out.hash_info = entry.hash_info
out.meta = entry.meta
stage.dvcfile.dump(stage)
return len(index)
@locked
def fetch(
self,
@@ -42,6 +74,11 @@ def fetch(
if isinstance(targets, str):
targets = [targets]
with suppress(NoRemoteError):
_remote = self.cloud.get_remote(name=remote)
if _remote.worktree:
return _fetch_worktree(self, _remote)
used = self.used_objs(
targets,
all_branches=all_branches,
+1
View File
@@ -205,6 +205,7 @@ class Index:
obj=out.obj,
hash_info=out.hash_info,
odb=out.odb,
cache=out.odb,
remote=remote,
)
+29
View File
@@ -9,6 +9,31 @@ if TYPE_CHECKING:
from dvc_objects.db.base import ObjectDB
def _push_worktree(repo, remote):
from dvc_data.index import build, checkout
index = repo.index.data["repo"]
checkout(index, remote.path, remote.fs)
build(index, remote.path, remote.fs)
for stage in repo.index.stages:
for out in stage.outs:
if not out.use_cache:
continue
if not out.is_in_repo:
continue
key = repo.fs.path.relparts(out.fs_path, repo.root_dir)
entry = repo.index.data["repo"][key]
out.hash_info = entry.hash_info
out.meta = entry.meta
stage.dvcfile.dump(stage)
return len(index)
@locked
def push(
self,
@@ -26,6 +51,10 @@ def push(
odb: Optional["ObjectDB"] = None,
include_imports=False,
):
_remote = self.cloud.get_remote(name=remote)
if _remote.worktree:
return _push_worktree(self, _remote)
used_run_cache = (
self.stage_cache.push(remote, odb=odb) if run_cache else []
)