mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-18 23:10:14 +01:00
* add support for tracking remote dataset This adds support for virtually tracking a dataset such as dvcx dataset, dataset from remote dvc/git registries, and cloud-versioned remotes. This PR introduces two different commands under `ds` namespace: `add` and `update`. `dvc ds add` command adds the dataset to `dvc.yaml` and tracks the sources in `dvc.lock` file. Similarly, `dvc ds update` updates the sources in `dvc.lock` file. Example Usage: ```console dvc ds add --name dogs --url dvcx://dogs # tracking dvcx dataset # tracking dvc/git registries dvc ds add --name example --url dvc://[email protected]/iterative/example.git # cloud-versioned-remotes dvc ds add --name versioning --url s3://cloud-versioning-demo ``` To update, specify the name. ```console dvc ds update <name> ``` `dvc ds add` freezes (in the traditional sense of `dvc import/import-url`). It keeps a "specification" of the dataset in `dvc.yaml` and also freezes information about the dataset in the `dvc.lock` file. They are kept inside `datasets` section in both `dvc.yaml` and `dvc.lock` files. This metadata is used in the pipelines. You can add a dependency to your stage using `ds://` scheme, followed by the name of the dataset in `dvc.yaml` file. As it is used in pipelines, the `name` of the dataset has to be unique in the repository. Different dataset of same names are not allowed. On `dvc repro`, `dvc` copies the frozen information about the particular dataset into the `deps` field for the stage in `dvc.lock`. On successive invocation, `dvc` will compare the information from the deps field of the lock with the frozen information in the `datasets` section and decides whether to rerun or not. As the dataset is frozen, `dvc repro` won't rerun until the dataset is updated via `dvc update`. Here are some examples for how `dvc.yaml` looks like: ```yaml datasets: - name: dogs url: dvcx://dogs type: dvcx - name: example-get-started url: [email protected]:iterative/example-get-started.git type: dvc path: path - name: dogs2 url: dvcx://dogs@v2 type: dvcx - name: cloud-versioning-demo url: s3://cloud-versioning-demo type: url ``` ```yaml schema: '2.0' datasets: - name: dogs url: dvcx://dogs type: dvcx version: 3 created_at: '2023-12-11T10:32:05.942708+00:00' - name: example-get-started url: [email protected]:iterative/example-get-started.git type: dvc path: path rev_lock: df75c16ef61f0772d6e4bb27ba4617b06b4b5398 - name: cloud-versioning-demo url: s3://cloud-versioning-demo type: url meta: isdir: true size: 323919 nfiles: 33 files: - relpath: myproject/model.pt meta: size: 106433 version_id: 5qrtnhnQ4fBzV73kqqK6pMGhTOzd_IPr etag: 3bc0028677ce6fb65bec8090c248b002 # truncated ``` The pipeline stages keep them in `dataset` section of individual deps. ```console dvc stage add -n train -d ds://dogs python train.py cat dvc.yaml ``` ```yaml # truncated stages: train: cmd: python train.py deps: - ds://dogs ``` When it is reproduced, the `dvc.lock` will look something like follows: ```yaml # truncated stages: train: cmd: python train.py deps: - path: ds://dogs dataset: name: dogs url: dvcx://dogs type: dvcx version: 3 created_at: '2023-12-11T10:32:05.942708+00:00' ``` * remodeling * remove non-cloud-versioned remotes * improve deserializing; handle invalidation
85 lines
2.6 KiB
Python
85 lines
2.6 KiB
Python
from typing import TYPE_CHECKING, Any, ClassVar
|
|
from urllib.parse import urlparse
|
|
|
|
from funcy import compact, merge
|
|
|
|
from dvc.exceptions import DvcException
|
|
from dvc_data.hashfile.hash_info import HashInfo
|
|
|
|
from .db import AbstractDependency
|
|
|
|
if TYPE_CHECKING:
|
|
from dvc.stage import Stage
|
|
|
|
|
|
class DatasetDependency(AbstractDependency):
|
|
PARAM_DATASET = "dataset"
|
|
DATASET_SCHEMA: ClassVar[dict] = {PARAM_DATASET: dict}
|
|
|
|
def __init__(self, stage: "Stage", p, info, *args, **kwargs):
|
|
super().__init__(stage, info, *args, **kwargs)
|
|
self.def_path = p
|
|
self.name = urlparse(p).netloc
|
|
dataset_info = info.get(self.PARAM_DATASET) or {}
|
|
self.hash_info = HashInfo(self.PARAM_DATASET, dataset_info) # type: ignore[arg-type]
|
|
self.hash_name = self.PARAM_DATASET
|
|
|
|
def __repr__(self):
|
|
return f"{self.__class__.__name__}({self.def_path!r})"
|
|
|
|
def __str__(self):
|
|
return self.def_path
|
|
|
|
@classmethod
|
|
def is_dataset(cls, p: str):
|
|
return urlparse(p).scheme == "ds"
|
|
|
|
@property
|
|
def protocol(self):
|
|
return None
|
|
|
|
def dumpd(self, **kwargs):
|
|
return compact({self.PARAM_PATH: self.def_path, **self.hash_info.to_dict()})
|
|
|
|
def fill_values(self, values=None):
|
|
"""Load params values dynamically."""
|
|
self.hash_info = HashInfo(
|
|
self.PARAM_DATASET, merge(self.hash_info.value, values or {})
|
|
)
|
|
|
|
def workspace_status(self):
|
|
ds = self.repo.datasets[self.name]
|
|
if not ds.lock:
|
|
return {str(self): "not in sync"}
|
|
|
|
info: dict[str, Any] = self.hash_info.value if self.hash_info else {} # type: ignore[assignment]
|
|
lock = self.repo.datasets._lock_from_info(info)
|
|
if not lock:
|
|
return {str(self): "new"}
|
|
if lock != ds.lock:
|
|
return {str(self): "modified"}
|
|
return {}
|
|
|
|
def status(self):
|
|
return self.workspace_status()
|
|
|
|
def get_hash(self):
|
|
ds = self.repo.datasets[self.name]
|
|
if not ds.lock:
|
|
if ds._invalidated:
|
|
raise DvcException(
|
|
"Dataset information is not in sync. "
|
|
f"Run 'dvc ds update {self.name}' to sync."
|
|
)
|
|
raise DvcException("Dataset information missing from dvc.lock file")
|
|
return HashInfo(self.PARAM_DATASET, ds.lock.to_dict()) # type: ignore[arg-type]
|
|
|
|
def save(self):
|
|
self.hash_info = self.get_hash()
|
|
|
|
def download(self, to, jobs=None):
|
|
raise NotImplementedError
|
|
|
|
def update(self, rev=None):
|
|
raise NotImplementedError
|