mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-17 07:18:26 +01:00
* add support for tracking remote dataset This adds support for virtually tracking a dataset such as dvcx dataset, dataset from remote dvc/git registries, and cloud-versioned remotes. This PR introduces two different commands under `ds` namespace: `add` and `update`. `dvc ds add` command adds the dataset to `dvc.yaml` and tracks the sources in `dvc.lock` file. Similarly, `dvc ds update` updates the sources in `dvc.lock` file. Example Usage: ```console dvc ds add --name dogs --url dvcx://dogs # tracking dvcx dataset # tracking dvc/git registries dvc ds add --name example --url dvc://[email protected]/iterative/example.git # cloud-versioned-remotes dvc ds add --name versioning --url s3://cloud-versioning-demo ``` To update, specify the name. ```console dvc ds update <name> ``` `dvc ds add` freezes (in the traditional sense of `dvc import/import-url`). It keeps a "specification" of the dataset in `dvc.yaml` and also freezes information about the dataset in the `dvc.lock` file. They are kept inside `datasets` section in both `dvc.yaml` and `dvc.lock` files. This metadata is used in the pipelines. You can add a dependency to your stage using `ds://` scheme, followed by the name of the dataset in `dvc.yaml` file. As it is used in pipelines, the `name` of the dataset has to be unique in the repository. Different dataset of same names are not allowed. On `dvc repro`, `dvc` copies the frozen information about the particular dataset into the `deps` field for the stage in `dvc.lock`. On successive invocation, `dvc` will compare the information from the deps field of the lock with the frozen information in the `datasets` section and decides whether to rerun or not. As the dataset is frozen, `dvc repro` won't rerun until the dataset is updated via `dvc update`. Here are some examples for how `dvc.yaml` looks like: ```yaml datasets: - name: dogs url: dvcx://dogs type: dvcx - name: example-get-started url: [email protected]:iterative/example-get-started.git type: dvc path: path - name: dogs2 url: dvcx://dogs@v2 type: dvcx - name: cloud-versioning-demo url: s3://cloud-versioning-demo type: url ``` ```yaml schema: '2.0' datasets: - name: dogs url: dvcx://dogs type: dvcx version: 3 created_at: '2023-12-11T10:32:05.942708+00:00' - name: example-get-started url: [email protected]:iterative/example-get-started.git type: dvc path: path rev_lock: df75c16ef61f0772d6e4bb27ba4617b06b4b5398 - name: cloud-versioning-demo url: s3://cloud-versioning-demo type: url meta: isdir: true size: 323919 nfiles: 33 files: - relpath: myproject/model.pt meta: size: 106433 version_id: 5qrtnhnQ4fBzV73kqqK6pMGhTOzd_IPr etag: 3bc0028677ce6fb65bec8090c248b002 # truncated ``` The pipeline stages keep them in `dataset` section of individual deps. ```console dvc stage add -n train -d ds://dogs python train.py cat dvc.yaml ``` ```yaml # truncated stages: train: cmd: python train.py deps: - ds://dogs ``` When it is reproduced, the `dvc.lock` will look something like follows: ```yaml # truncated stages: train: cmd: python train.py deps: - path: ds://dogs dataset: name: dogs url: dvcx://dogs type: dvcx version: 3 created_at: '2023-12-11T10:32:05.942708+00:00' ``` * remodeling * remove non-cloud-versioned remotes * improve deserializing; handle invalidation
122 lines
3.7 KiB
Python
122 lines
3.7 KiB
Python
from collections.abc import Iterator
|
|
from contextlib import contextmanager
|
|
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional
|
|
|
|
from funcy import compact, log_durations
|
|
|
|
from dvc.exceptions import DvcException
|
|
from dvc.log import logger
|
|
|
|
from .base import Dependency
|
|
|
|
if TYPE_CHECKING:
|
|
from dvc.output import Output
|
|
from dvc.repo import Repo
|
|
from dvc.stage import Stage
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
|
|
@contextmanager
|
|
def download_progress(to: "Output") -> Iterator[Callable[[int], Any]]:
|
|
from dvc.ui import ui
|
|
from dvc.ui._rich_progress import DbDownloadProgress
|
|
|
|
with log_durations(logger.debug, f"Saving to {to}"), DbDownloadProgress(
|
|
console=ui.error_console,
|
|
) as progress:
|
|
task = progress.add_task("Saving", total=None, output=to)
|
|
yield lambda n: progress.advance(task, advance=n)
|
|
progress.update(task, description="Saved", total=0)
|
|
|
|
|
|
class AbstractDependency(Dependency):
|
|
"""Dependency without workspace/fs/fs_path"""
|
|
|
|
def __init__(self, stage: "Stage", info: dict[str, Any], *args, **kwargs):
|
|
self.repo: "Repo" = stage.repo
|
|
self.stage = stage
|
|
self.fs = None
|
|
self.fs_path = None
|
|
self.def_path = None # type: ignore[assignment]
|
|
self.info = info or {}
|
|
|
|
|
|
class DbDependency(AbstractDependency):
|
|
PARAM_CONNECTION = "connection"
|
|
PARAM_DB = "db"
|
|
PARAM_QUERY = "query"
|
|
PARAM_TABLE = "table"
|
|
PARAM_FILE_FORMAT = "file_format"
|
|
DB_SCHEMA: ClassVar[dict] = {
|
|
PARAM_DB: {
|
|
PARAM_QUERY: str,
|
|
PARAM_CONNECTION: str,
|
|
PARAM_FILE_FORMAT: str,
|
|
PARAM_TABLE: str,
|
|
}
|
|
}
|
|
|
|
def __init__(self, stage: "Stage", info, *args, **kwargs):
|
|
super().__init__(stage, info, *args, **kwargs)
|
|
self.db_info: dict[str, str] = self.info.get(self.PARAM_DB, {})
|
|
self.connection = self.db_info.get(self.PARAM_CONNECTION)
|
|
|
|
@property
|
|
def sql(self) -> Optional[str]:
|
|
return self.db_info.get(self.PARAM_QUERY) or self.db_info.get(self.PARAM_TABLE)
|
|
|
|
def __repr__(self):
|
|
return "{}:{}".format(
|
|
self.__class__.__name__, "".join(f"{k}=={v}" for k, v in self.info.items())
|
|
)
|
|
|
|
def __str__(self):
|
|
from dvc.utils.humanize import truncate_text
|
|
|
|
return truncate_text(self.sql or "", 50)
|
|
|
|
def workspace_status(self):
|
|
return False # no workspace to check
|
|
|
|
def status(self):
|
|
return self.workspace_status()
|
|
|
|
def save(self):
|
|
"""nothing to save."""
|
|
|
|
def dumpd(self, **kwargs):
|
|
db_info = compact(self.db_info)
|
|
return {self.PARAM_DB: db_info} if db_info else {}
|
|
|
|
def update(self, rev=None):
|
|
"""nothing to update."""
|
|
|
|
def download(
|
|
self,
|
|
to: "Output",
|
|
jobs: Optional[int] = None, # noqa: ARG002
|
|
file_format: Optional[str] = None,
|
|
**kwargs: Any,
|
|
) -> None:
|
|
from dvc.database import client
|
|
from dvc.ui import ui
|
|
|
|
sql = self.sql
|
|
if not sql:
|
|
raise DvcException("Cannot download: no query or table specified")
|
|
|
|
db_config = self.repo.config.get(self.PARAM_DB, {})
|
|
config = db_config.get(self.connection)
|
|
if not config:
|
|
raise DvcException(f"connection {self.connection} not found in config")
|
|
|
|
file_format = file_format or self.db_info.get(self.PARAM_FILE_FORMAT, "csv")
|
|
assert file_format
|
|
with client(config) as db:
|
|
msg = "Testing connection"
|
|
with log_durations(logger.debug, msg), ui.status(msg) as status:
|
|
db.test_connection(onerror=status.stop)
|
|
with download_progress(to) as progress:
|
|
db.export(sql, to.fs_path, format=file_format, progress=progress)
|