Files
dvc/dvc/dependency/db.py
T
skshetry 34d80fbfce add support for tracking remote dataset (#10287)
* add support for tracking remote dataset

This adds support for virtually tracking a
dataset such as dvcx dataset, dataset from remote dvc/git
registries, and cloud-versioned remotes.

This PR introduces two different commands under `ds` namespace:
`add` and `update`.

`dvc ds add` command adds the dataset to `dvc.yaml` and tracks
the sources in `dvc.lock` file. Similarly, `dvc ds update`
updates the sources in `dvc.lock` file.

Example Usage:
```console
dvc ds add --name dogs --url dvcx://dogs # tracking dvcx dataset
# tracking dvc/git registries
dvc ds add --name example --url dvc://[email protected]/iterative/example.git
# cloud-versioned-remotes
dvc ds add --name versioning --url s3://cloud-versioning-demo
```

To update, specify the name.
```console
dvc ds update <name>
```

`dvc ds add` freezes (in the traditional sense of `dvc import/import-url`).
It keeps a "specification" of the dataset in `dvc.yaml` and also freezes
information about the dataset in the `dvc.lock` file. They are kept
inside `datasets` section in both `dvc.yaml` and `dvc.lock` files.

This metadata is used in the pipelines. You can add a dependency to your
stage using `ds://` scheme, followed by the name of the dataset in
`dvc.yaml` file. As it is used in pipelines, the `name` of the dataset
has to be unique in the repository. Different dataset of same names
are not allowed. On `dvc repro`, `dvc` copies the frozen information
about the particular dataset into the `deps` field for the stage in
`dvc.lock`. On successive invocation, `dvc` will compare the information
from the deps field of the lock with the frozen information in the
`datasets` section and decides whether to rerun or not.

As the dataset is frozen, `dvc repro` won't rerun until the dataset is
updated via `dvc update`.

Here are some examples for how `dvc.yaml` looks like:
```yaml
datasets:
- name: dogs
  url: dvcx://dogs
  type: dvcx
- name: example-get-started
  url: [email protected]:iterative/example-get-started.git
  type: dvc
  path: path
- name: dogs2
  url: dvcx://dogs@v2
  type: dvcx
- name: cloud-versioning-demo
  url: s3://cloud-versioning-demo
  type: url
```

```yaml
schema: '2.0'
datasets:
- name: dogs
  url: dvcx://dogs
  type: dvcx
  version: 3
  created_at: '2023-12-11T10:32:05.942708+00:00'
- name: example-get-started
  url: [email protected]:iterative/example-get-started.git
  type: dvc
  path: path
  rev_lock: df75c16ef61f0772d6e4bb27ba4617b06b4b5398
- name: cloud-versioning-demo
  url: s3://cloud-versioning-demo
  type: url
  meta:
    isdir: true
    size: 323919
    nfiles: 33
  files:
  - relpath: myproject/model.pt
    meta:
      size: 106433
      version_id: 5qrtnhnQ4fBzV73kqqK6pMGhTOzd_IPr
      etag: 3bc0028677ce6fb65bec8090c248b002
# truncated
```

The pipeline stages keep them in `dataset` section of individual deps.

```console
dvc stage add -n train -d ds://dogs python train.py
cat dvc.yaml
```

```yaml
# truncated
stages:
  train:
    cmd: python train.py
    deps:
    - ds://dogs
```

When it is reproduced, the `dvc.lock` will look something like follows:
```yaml
# truncated
stages:
  train:
    cmd: python train.py
    deps:
    - path: ds://dogs
      dataset:
        name: dogs
        url: dvcx://dogs
        type: dvcx
        version: 3
        created_at: '2023-12-11T10:32:05.942708+00:00'
```

* remodeling

* remove non-cloud-versioned remotes

* improve deserializing; handle invalidation
2024-02-26 21:28:39 +05:45

122 lines
3.7 KiB
Python

from collections.abc import Iterator
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional
from funcy import compact, log_durations
from dvc.exceptions import DvcException
from dvc.log import logger
from .base import Dependency
if TYPE_CHECKING:
from dvc.output import Output
from dvc.repo import Repo
from dvc.stage import Stage
logger = logger.getChild(__name__)
@contextmanager
def download_progress(to: "Output") -> Iterator[Callable[[int], Any]]:
from dvc.ui import ui
from dvc.ui._rich_progress import DbDownloadProgress
with log_durations(logger.debug, f"Saving to {to}"), DbDownloadProgress(
console=ui.error_console,
) as progress:
task = progress.add_task("Saving", total=None, output=to)
yield lambda n: progress.advance(task, advance=n)
progress.update(task, description="Saved", total=0)
class AbstractDependency(Dependency):
"""Dependency without workspace/fs/fs_path"""
def __init__(self, stage: "Stage", info: dict[str, Any], *args, **kwargs):
self.repo: "Repo" = stage.repo
self.stage = stage
self.fs = None
self.fs_path = None
self.def_path = None # type: ignore[assignment]
self.info = info or {}
class DbDependency(AbstractDependency):
PARAM_CONNECTION = "connection"
PARAM_DB = "db"
PARAM_QUERY = "query"
PARAM_TABLE = "table"
PARAM_FILE_FORMAT = "file_format"
DB_SCHEMA: ClassVar[dict] = {
PARAM_DB: {
PARAM_QUERY: str,
PARAM_CONNECTION: str,
PARAM_FILE_FORMAT: str,
PARAM_TABLE: str,
}
}
def __init__(self, stage: "Stage", info, *args, **kwargs):
super().__init__(stage, info, *args, **kwargs)
self.db_info: dict[str, str] = self.info.get(self.PARAM_DB, {})
self.connection = self.db_info.get(self.PARAM_CONNECTION)
@property
def sql(self) -> Optional[str]:
return self.db_info.get(self.PARAM_QUERY) or self.db_info.get(self.PARAM_TABLE)
def __repr__(self):
return "{}:{}".format(
self.__class__.__name__, "".join(f"{k}=={v}" for k, v in self.info.items())
)
def __str__(self):
from dvc.utils.humanize import truncate_text
return truncate_text(self.sql or "", 50)
def workspace_status(self):
return False # no workspace to check
def status(self):
return self.workspace_status()
def save(self):
"""nothing to save."""
def dumpd(self, **kwargs):
db_info = compact(self.db_info)
return {self.PARAM_DB: db_info} if db_info else {}
def update(self, rev=None):
"""nothing to update."""
def download(
self,
to: "Output",
jobs: Optional[int] = None, # noqa: ARG002
file_format: Optional[str] = None,
**kwargs: Any,
) -> None:
from dvc.database import client
from dvc.ui import ui
sql = self.sql
if not sql:
raise DvcException("Cannot download: no query or table specified")
db_config = self.repo.config.get(self.PARAM_DB, {})
config = db_config.get(self.connection)
if not config:
raise DvcException(f"connection {self.connection} not found in config")
file_format = file_format or self.db_info.get(self.PARAM_FILE_FORMAT, "csv")
assert file_format
with client(config) as db:
msg = "Testing connection"
with log_durations(logger.debug, msg), ui.status(msg) as status:
db.test_connection(onerror=status.stop)
with download_progress(to) as progress:
db.export(sql, to.fs_path, format=file_format, progress=progress)