mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-01 23:08:43 +01:00
skshetry
34d80fbfce
* add support for tracking remote dataset This adds support for virtually tracking a dataset such as dvcx dataset, dataset from remote dvc/git registries, and cloud-versioned remotes. This PR introduces two different commands under `ds` namespace: `add` and `update`. `dvc ds add` command adds the dataset to `dvc.yaml` and tracks the sources in `dvc.lock` file. Similarly, `dvc ds update` updates the sources in `dvc.lock` file. Example Usage: ```console dvc ds add --name dogs --url dvcx://dogs # tracking dvcx dataset # tracking dvc/git registries dvc ds add --name example --url dvc://[email protected]/iterative/example.git # cloud-versioned-remotes dvc ds add --name versioning --url s3://cloud-versioning-demo ``` To update, specify the name. ```console dvc ds update <name> ``` `dvc ds add` freezes (in the traditional sense of `dvc import/import-url`). It keeps a "specification" of the dataset in `dvc.yaml` and also freezes information about the dataset in the `dvc.lock` file. They are kept inside `datasets` section in both `dvc.yaml` and `dvc.lock` files. This metadata is used in the pipelines. You can add a dependency to your stage using `ds://` scheme, followed by the name of the dataset in `dvc.yaml` file. As it is used in pipelines, the `name` of the dataset has to be unique in the repository. Different dataset of same names are not allowed. On `dvc repro`, `dvc` copies the frozen information about the particular dataset into the `deps` field for the stage in `dvc.lock`. On successive invocation, `dvc` will compare the information from the deps field of the lock with the frozen information in the `datasets` section and decides whether to rerun or not. As the dataset is frozen, `dvc repro` won't rerun until the dataset is updated via `dvc update`. Here are some examples for how `dvc.yaml` looks like: ```yaml datasets: - name: dogs url: dvcx://dogs type: dvcx - name: example-get-started url: [email protected]:iterative/example-get-started.git type: dvc path: path - name: dogs2 url: dvcx://dogs@v2 type: dvcx - name: cloud-versioning-demo url: s3://cloud-versioning-demo type: url ``` ```yaml schema: '2.0' datasets: - name: dogs url: dvcx://dogs type: dvcx version: 3 created_at: '2023-12-11T10:32:05.942708+00:00' - name: example-get-started url: [email protected]:iterative/example-get-started.git type: dvc path: path rev_lock: df75c16ef61f0772d6e4bb27ba4617b06b4b5398 - name: cloud-versioning-demo url: s3://cloud-versioning-demo type: url meta: isdir: true size: 323919 nfiles: 33 files: - relpath: myproject/model.pt meta: size: 106433 version_id: 5qrtnhnQ4fBzV73kqqK6pMGhTOzd_IPr etag: 3bc0028677ce6fb65bec8090c248b002 # truncated ``` The pipeline stages keep them in `dataset` section of individual deps. ```console dvc stage add -n train -d ds://dogs python train.py cat dvc.yaml ``` ```yaml # truncated stages: train: cmd: python train.py deps: - ds://dogs ``` When it is reproduced, the `dvc.lock` will look something like follows: ```yaml # truncated stages: train: cmd: python train.py deps: - path: ds://dogs dataset: name: dogs url: dvcx://dogs type: dvcx version: 3 created_at: '2023-12-11T10:32:05.942708+00:00' ``` * remodeling * remove non-cloud-versioned remotes * improve deserializing; handle invalidation
64 lines
2.1 KiB
Python
64 lines
2.1 KiB
Python
import logging
|
|
import re
|
|
from argparse import SUPPRESS, ArgumentParser
|
|
from itertools import takewhile
|
|
|
|
import pytest
|
|
import shtab
|
|
|
|
from dvc.cli import main
|
|
from dvc.cli.parser import get_main_parser
|
|
|
|
|
|
def command_tuples():
|
|
root: tuple[str, ...] = ()
|
|
commands = [root]
|
|
|
|
def recurse_parser(parser: ArgumentParser, parents: tuple[str, ...] = root) -> None:
|
|
for positional in parser._get_positional_actions():
|
|
if positional.help != SUPPRESS and isinstance(positional.choices, dict):
|
|
public_cmds = shtab.get_public_subcommands(positional)
|
|
for subcmd, subparser in positional.choices.items():
|
|
cmd = (*parents, subcmd)
|
|
if subcmd in public_cmds:
|
|
commands.append(cmd)
|
|
recurse_parser(subparser, cmd)
|
|
|
|
main_parser = get_main_parser()
|
|
recurse_parser(main_parser)
|
|
|
|
# the no. of commands will usually go up,
|
|
# but if we ever remove commands and drop below, adjust the magic number accordingly
|
|
assert len(commands) >= 116
|
|
return sorted(commands)
|
|
|
|
|
|
def ids(values):
|
|
return "-".join(values) or "dvc"
|
|
|
|
|
|
@pytest.mark.parametrize("command_tuples", command_tuples(), ids=ids)
|
|
def test_help(caplog, capsys, command_tuples):
|
|
with caplog.at_level(logging.INFO), pytest.raises(SystemExit) as e:
|
|
main([*command_tuples, "--help"])
|
|
assert e.value.code == 0
|
|
assert not caplog.text
|
|
|
|
out, err = capsys.readouterr()
|
|
|
|
# validate metavars are all in lowercase
|
|
usage = "\n".join(takewhile(lambda o: bool(o), out.splitlines()))
|
|
|
|
message = (
|
|
"metavars not lowercased, you are likely missing formatter_class=XXX "
|
|
"in the command, where XXX should be any of the classes from "
|
|
"`dvc.cli.formatter`, which automatically lowercases metavars.\n"
|
|
"\nExample:\n"
|
|
"\nfrom dvc.cli import formatter\n"
|
|
"\nparser.add_parser(..., formatter_class=formatter.TextHelpFormatter)\n"
|
|
)
|
|
assert not re.findall(r"\b[A-Z]{2,}\b", usage, re.MULTILINE), message
|
|
|
|
assert not err
|
|
assert out
|