mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
dvc: cloud versioning POC
This commit is contained in:
@@ -78,6 +78,7 @@ REMOTE_COMMON = {
|
||||
"url": str,
|
||||
"checksum_jobs": All(Coerce(int), Range(1)),
|
||||
"jobs": All(Coerce(int), Range(1)),
|
||||
Optional("worktree"): Bool,
|
||||
Optional("no_traverse"): Bool, # obsoleted
|
||||
Optional("version_aware"): Bool,
|
||||
}
|
||||
|
||||
+36
-10
@@ -3,6 +3,8 @@
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Iterable, Optional
|
||||
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc_data.hashfile.db import get_index
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -13,6 +15,25 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Remote:
|
||||
def __init__(self, path, fs, **config):
|
||||
self.path = path
|
||||
self.fs = fs
|
||||
|
||||
self.worktree = config.pop("worktree", False)
|
||||
self.config = config
|
||||
|
||||
@cached_property
|
||||
def odb(self):
|
||||
from dvc_data.hashfile.db import get_odb
|
||||
|
||||
path = self.path
|
||||
if self.worktree:
|
||||
path = self.fs.path.join(path, ".dvc", "cache")
|
||||
|
||||
return get_odb(self.fs, path, **self.config)
|
||||
|
||||
|
||||
class DataCloud:
|
||||
"""Class that manages dvc remotes.
|
||||
|
||||
@@ -27,18 +48,23 @@ class DataCloud:
|
||||
def __init__(self, repo):
|
||||
self.repo = repo
|
||||
|
||||
def get_remote_odb(
|
||||
def get_remote(
|
||||
self,
|
||||
name: Optional[str] = None,
|
||||
command: str = "<command>",
|
||||
) -> "HashFileDB":
|
||||
) -> "Remote":
|
||||
from dvc.config import NoRemoteError
|
||||
|
||||
if not name:
|
||||
name = self.repo.config["core"].get("remote")
|
||||
|
||||
if name:
|
||||
return self._init_odb(name)
|
||||
from dvc.fs import get_cloud_fs
|
||||
|
||||
cls, config, fs_path = get_cloud_fs(self.repo, name=name)
|
||||
fs = cls(**config)
|
||||
config["tmp_dir"] = self.repo.index_db_dir
|
||||
return Remote(fs_path, fs, **config)
|
||||
|
||||
if bool(self.repo.config["remote"]):
|
||||
error_msg = (
|
||||
@@ -55,13 +81,13 @@ class DataCloud:
|
||||
|
||||
raise NoRemoteError(error_msg)
|
||||
|
||||
def _init_odb(self, name):
|
||||
from dvc.fs import get_cloud_fs
|
||||
from dvc_data.hashfile.db import get_odb
|
||||
|
||||
cls, config, fs_path = get_cloud_fs(self.repo, name=name)
|
||||
config["tmp_dir"] = self.repo.index_db_dir
|
||||
return get_odb(cls(**config), fs_path, **config)
|
||||
def get_remote_odb(
|
||||
self,
|
||||
name: Optional[str] = None,
|
||||
command: str = "<command>",
|
||||
) -> "HashFileDB":
|
||||
remote = self.get_remote(name=name, command=command)
|
||||
return remote.odb
|
||||
|
||||
def _log_missing(self, status: "CompareStatusResult"):
|
||||
if status.missing:
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import logging
|
||||
from contextlib import suppress
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.exceptions import DownloadError, FileTransferError
|
||||
from dvc.fs import Schemes
|
||||
|
||||
@@ -12,6 +14,36 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _fetch_worktree(repo, remote):
|
||||
from dvc_data.index import save
|
||||
|
||||
index = repo.index.data["repo"]
|
||||
for key, entry in index.iteritems():
|
||||
entry.fs = remote.fs
|
||||
entry.path = remote.fs.path.join(
|
||||
remote.path,
|
||||
*key,
|
||||
)
|
||||
save(index)
|
||||
|
||||
for stage in repo.index.stages:
|
||||
for out in stage.outs:
|
||||
if not out.use_cache:
|
||||
continue
|
||||
|
||||
if not out.is_in_repo:
|
||||
continue
|
||||
|
||||
key = repo.fs.path.relparts(out.fs_path, repo.root_dir)
|
||||
entry = repo.index.data["repo"][key]
|
||||
out.hash_info = entry.hash_info
|
||||
out.meta = entry.meta
|
||||
|
||||
stage.dvcfile.dump(stage)
|
||||
|
||||
return len(index)
|
||||
|
||||
|
||||
@locked
|
||||
def fetch(
|
||||
self,
|
||||
@@ -42,6 +74,11 @@ def fetch(
|
||||
if isinstance(targets, str):
|
||||
targets = [targets]
|
||||
|
||||
with suppress(NoRemoteError):
|
||||
_remote = self.cloud.get_remote(name=remote)
|
||||
if _remote.worktree:
|
||||
return _fetch_worktree(self, _remote)
|
||||
|
||||
used = self.used_objs(
|
||||
targets,
|
||||
all_branches=all_branches,
|
||||
|
||||
@@ -205,6 +205,7 @@ class Index:
|
||||
obj=out.obj,
|
||||
hash_info=out.hash_info,
|
||||
odb=out.odb,
|
||||
cache=out.odb,
|
||||
remote=remote,
|
||||
)
|
||||
|
||||
|
||||
@@ -9,6 +9,31 @@ if TYPE_CHECKING:
|
||||
from dvc_objects.db.base import ObjectDB
|
||||
|
||||
|
||||
def _push_worktree(repo, remote):
|
||||
from dvc_data.index import build, checkout
|
||||
|
||||
index = repo.index.data["repo"]
|
||||
checkout(index, remote.path, remote.fs)
|
||||
build(index, remote.path, remote.fs)
|
||||
|
||||
for stage in repo.index.stages:
|
||||
for out in stage.outs:
|
||||
if not out.use_cache:
|
||||
continue
|
||||
|
||||
if not out.is_in_repo:
|
||||
continue
|
||||
|
||||
key = repo.fs.path.relparts(out.fs_path, repo.root_dir)
|
||||
entry = repo.index.data["repo"][key]
|
||||
out.hash_info = entry.hash_info
|
||||
out.meta = entry.meta
|
||||
|
||||
stage.dvcfile.dump(stage)
|
||||
|
||||
return len(index)
|
||||
|
||||
|
||||
@locked
|
||||
def push(
|
||||
self,
|
||||
@@ -26,6 +51,10 @@ def push(
|
||||
odb: Optional["ObjectDB"] = None,
|
||||
include_imports=False,
|
||||
):
|
||||
_remote = self.cloud.get_remote(name=remote)
|
||||
if _remote.worktree:
|
||||
return _push_worktree(self, _remote)
|
||||
|
||||
used_run_cache = (
|
||||
self.stage_cache.push(remote, odb=odb) if run_cache else []
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user