From 2431ec6189753e0c6a5ddffd03c10f4cdc4b1abb Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Mon, 4 Nov 2024 22:42:45 -0800 Subject: [PATCH] Rename dvcx to DataChain in DVC <> DataChain integration (#10611) --- .github/.test_durations | 8 ++-- dvc/api/dataset.py | 12 +++--- dvc/commands/dataset.py | 12 +++--- dvc/repo/datasets.py | 43 ++++++++++---------- tests/func/parsing/test_top_level.py | 6 +-- tests/func/test_dataset.py | 60 ++++++++++++++-------------- tests/unit/command/test_dataset.py | 14 +++---- 7 files changed, 79 insertions(+), 76 deletions(-) diff --git a/.github/.test_durations b/.github/.test_durations index 8fe76babd..ef0b5befc 100644 --- a/.github/.test_durations +++ b/.github/.test_durations @@ -835,9 +835,9 @@ "tests/func/test_dataset.py::test_dvc": 0.46001330000012786, "tests/func/test_dataset.py::test_dvc_dataset_pipeline": 0.7948393000000351, "tests/func/test_dataset.py::test_dvc_dump": 0.17996560000005957, - "tests/func/test_dataset.py::test_dvcx": 0.22482930000001033, - "tests/func/test_dataset.py::test_dvcx_dataset_pipeline": 0.43405649999999696, - "tests/func/test_dataset.py::test_dvcx_dump": 0.1858166000000665, + "tests/func/test_dataset.py::test_datachain": 0.22482930000001033, + "tests/func/test_dataset.py::test_datachain_dataset_pipeline": 0.43405649999999696, + "tests/func/test_dataset.py::test_datachain_dump": 0.1858166000000665, "tests/func/test_dataset.py::test_invalidation": 0.16291839999996682, "tests/func/test_dataset.py::test_parametrized": 0.19277699999997822, "tests/func/test_dataset.py::test_pipeline_when_not_in_sync": 0.2063928999999689, @@ -1627,7 +1627,7 @@ "tests/unit/command/test_data_sync.py::test_pull": 0.17549940000003517, "tests/unit/command/test_data_sync.py::test_push": 0.16576090000000931, "tests/unit/command/test_dataset.py::test_add[spec0-lock0-Adding ds (url:/path @ main)\\n]": 0.22054290000005494, - "tests/unit/command/test_dataset.py::test_add[spec1-lock1-Adding mydataset (dvcx://dataset @ v1)\\n]": 0.22979979999990974, + "tests/unit/command/test_dataset.py::test_add[spec1-lock1-Adding mydataset (dc://dataset @ v1)\\n]": 0.22979979999990974, "tests/unit/command/test_dataset.py::test_add[spec2-lock2-Adding mydataset (s3://bucket/path)\\n]": 0.20264819999988504, "tests/unit/command/test_dataset.py::test_add_already_exists": 0.2456899000001158, "tests/unit/command/test_dataset.py::test_update[spec0-old_lock0-new_lock0-expected_outputs0-missing]": 0.2354315999998562, diff --git a/dvc/api/dataset.py b/dvc/api/dataset.py index 600d7a4e8..2a2378c28 100644 --- a/dvc/api/dataset.py +++ b/dvc/api/dataset.py @@ -1,8 +1,8 @@ from typing import Literal, TypedDict, Union -class DVCXDataset(TypedDict): - type: Literal["dvcx"] +class DatachainDataset(TypedDict): + type: Literal["dc"] name: str version: int @@ -20,7 +20,7 @@ class URLDataset(TypedDict): path: str -def get(name: str) -> Union[DVCXDataset, DVCDataset, URLDataset]: +def get(name: str) -> Union[DatachainDataset, DVCDataset, URLDataset]: from difflib import get_close_matches from dvc.fs import get_cloud_fs @@ -46,9 +46,9 @@ def get(name: str) -> Union[DVCXDataset, DVCDataset, URLDataset]: path=dataset.lock.path, sha=dataset.lock.rev_lock, ) - if dataset.type == "dvcx": - return DVCXDataset( - type="dvcx", name=dataset.name_version[0], version=dataset.lock.version + if dataset.type == "dc": + return DatachainDataset( + type="dc", name=dataset.name_version[0], version=dataset.lock.version ) if dataset.type == "url": fs_cls, _, path = get_cloud_fs(repo.config, url=dataset.lock.url) diff --git a/dvc/commands/dataset.py b/dvc/commands/dataset.py index 4601f2452..66b8117a9 100644 --- a/dvc/commands/dataset.py +++ b/dvc/commands/dataset.py @@ -34,7 +34,7 @@ class CmdDatasetAdd(CmdBase): url = dataset.spec.url ver: str = "" - if dataset.type == "dvcx": + if dataset.type == "dc": ver = f"v{dataset.lock.version}" if dataset.type == "dvc": if dataset.lock.path: @@ -55,7 +55,7 @@ class CmdDatasetAdd(CmdBase): raise DvcException("--path can't be used without --dvc") d = vars(self.args) - for key in ["dvc", "dvcx", "url"]: + for key in ["dvc", "dc", "url"]: if url := d.pop(key, None): d.update({"type": key, "url": url}) break @@ -88,8 +88,8 @@ class CmdDatasetUpdate(CmdBase): assert new.lock v: Optional[tuple[str, str]] = None - if dataset.type == "dvcx": - assert new.type == "dvcx" + if dataset.type == "dc": + assert new.type == "dc" if new.lock.version < dataset.lock.version: action = "Downgrading" @@ -168,7 +168,7 @@ def add_parser(subparsers, parent_parser): url_exclusive_group = ds_add_parser.add_mutually_exclusive_group(required=True) url_exclusive_group.add_argument( - "--dvcx", metavar="name", help="Name of the dvcx dataset to track" + "--dc", metavar="name", help="Name of the DataChain dataset to track" ) url_exclusive_group.add_argument( "--dvc", @@ -219,7 +219,7 @@ remote://remote_name/path/to/file/or/dir (see `dvc remote`) "--rev", "--version", nargs="?", - help="DVCX dataset version or Git revision (e.g. SHA, branch, tag)", + help="DataChain dataset version or Git revision (e.g. SHA, branch, tag)", metavar="", ) ds_update_parser.set_defaults(func=CmdDatasetUpdate) diff --git a/dvc/repo/datasets.py b/dvc/repo/datasets.py index 3510b3948..e12cb6b17 100644 --- a/dvc/repo/datasets.py +++ b/dvc/repo/datasets.py @@ -16,7 +16,7 @@ from dvc.types import StrPath from dvc_data.hashfile.meta import Meta if TYPE_CHECKING: - from dvcx.dataset import DatasetRecord, DatasetVersion # type: ignore[import] + from datachain.dataset import DatasetRecord, DatasetVersion # type: ignore[import] from typing_extensions import Self from dvc.repo import Repo @@ -29,10 +29,10 @@ def _get_dataset_record(name: str) -> "DatasetRecord": from dvc.exceptions import DvcException try: - from dvcx.catalog import get_catalog # type: ignore[import] + from datachain.catalog import get_catalog # type: ignore[import] except ImportError as exc: - raise DvcException("dvcx is not installed") from exc + raise DvcException("datachain is not installed") from exc catalog = get_catalog() return catalog.get_remote_dataset(name) @@ -86,7 +86,7 @@ class SerDe: class DatasetSpec(SerDe): name: str url: str - type: Literal["dvc", "dvcx", "url"] + type: Literal["dvc", "dc", "url"] @frozen(kw_only=True) @@ -108,7 +108,7 @@ class DVCDatasetLock(DVCDatasetSpec): @frozen(kw_only=True) -class DVCXDatasetLock(DatasetSpec): +class DatachainDatasetLock(DatasetSpec): version: int created_at: datetime = field(converter=to_datetime) @@ -160,13 +160,13 @@ class DVCDataset: @frozen(kw_only=True) -class DVCXDataset: +class DatachainDataset: manifest_path: str spec: "DatasetSpec" - lock: "Optional[DVCXDatasetLock]" = field(default=None) + lock: "Optional[DatachainDatasetLock]" = field(default=None) _invalidated: bool = field(default=False, eq=False, repr=False) - type: ClassVar[Literal["dvcx"]] = "dvcx" + type: ClassVar[Literal["dc"]] = "dc" @property def pinned(self) -> bool: @@ -193,7 +193,7 @@ class DVCXDataset: name, _version = self.name_version version = version if version is not None else _version version_info = _get_dataset_info(name, record=record, version=version) - lock = DVCXDatasetLock( + lock = DatachainDatasetLock( **self.spec.to_dict(), version=version_info.version, created_at=version_info.created_at, @@ -226,9 +226,9 @@ class URLDataset: return evolve(self, lock=lock) -Lock = Union[DVCDatasetLock, DVCXDatasetLock, URLDatasetLock] +Lock = Union[DVCDatasetLock, DatachainDatasetLock, URLDatasetLock] Spec = Union[DatasetSpec, DVCDatasetSpec] -Dataset = Union[DVCDataset, DVCXDataset, URLDataset] +Dataset = Union[DVCDataset, DatachainDataset, URLDataset] class DatasetNotFoundError(DvcException, KeyError): @@ -307,13 +307,13 @@ class Datasets(Mapping[str, Dataset]): raise ValueError("type should be present in spec") if typ == "dvc": return DVCDatasetSpec.from_dict(spec) - if typ in {"dvcx", "url"}: + if typ in {"dc", "url"}: return DatasetSpec.from_dict(spec) raise ValueError(f"unknown dataset type: {spec.get('type', '')}") @staticmethod def _lock_from_info(lock: Optional[dict[str, Any]]) -> Optional[Lock]: - kl = {"dvc": DVCDatasetLock, "dvcx": DVCXDatasetLock, "url": URLDatasetLock} + kl = {"dvc": DVCDatasetLock, "dc": DatachainDatasetLock, "url": URLDatasetLock} if lock and (cls := kl.get(lock.get("type", ""))): # type: ignore[assignment] return cls.from_dict(lock) # type: ignore[attr-defined] return None @@ -356,9 +356,9 @@ class Datasets(Mapping[str, Dataset]): lock=lock, invalidated=_invalidated, ) - if spec.type == "dvcx": - assert lock is None or isinstance(lock, DVCXDatasetLock) - return DVCXDataset( + if spec.type == "dc": + assert lock is None or isinstance(lock, DatachainDatasetLock) + return DatachainDataset( manifest_path=manifest_path, spec=spec, lock=lock, @@ -374,7 +374,7 @@ class Datasets(Mapping[str, Dataset]): manifest_path: StrPath = "dvc.yaml", **kwargs: Any, ) -> Dataset: - assert type in {"dvc", "dvcx", "url"} + assert type in {"dvc", "dc", "url"} kwargs.update({"name": name, "url": url, "type": type}) dataset = self._build_dataset(os.path.abspath(manifest_path), kwargs) dataset = dataset.update(self.repo) @@ -389,13 +389,16 @@ class Datasets(Mapping[str, Dataset]): if dataset.type == "url" and (version or kwargs.get("rev")): raise ValueError("cannot update version/revision for a url") - if dataset.type == "dvcx" and version is not None: + if dataset.type == "dc" and version is not None: if not isinstance(version, int): raise TypeError( - f"dvcx version has to be an integer, got {type(version).__name__!r}" + "DataChain dataset version has to be an integer, " + f"got {type(version).__name__!r}" ) if version < 1: - raise ValueError(f"dvcx version should be >=1, got {version}") + raise ValueError( + f"DataChain dataset version should be >=1, got {version}" + ) new = dataset.update(self.repo, **kwargs) diff --git a/tests/func/parsing/test_top_level.py b/tests/func/parsing/test_top_level.py index 6f9419157..775401376 100644 --- a/tests/func/parsing/test_top_level.py +++ b/tests/func/parsing/test_top_level.py @@ -121,7 +121,7 @@ def test_artifacts(tmp_dir, dvc): def test_datasets(tmp_dir, dvc): template = { "datasets": [ - {"name": "${ds1.name}", "url": "${ds1.url}", "type": "dvcx"}, + {"name": "${ds1.name}", "url": "${ds1.url}", "type": "dc"}, { "name": "${ds2.name}", "url": "${ds2.url}", @@ -138,7 +138,7 @@ def test_datasets(tmp_dir, dvc): (tmp_dir / "params.yaml").dump( { - "ds1": {"name": "dogs", "url": "dvcx://dogs"}, + "ds1": {"name": "dogs", "url": "dc://dogs"}, "ds2": { "name": "example-get-started", "url": "git@github.com:iterative/example-get-started.git", @@ -153,7 +153,7 @@ def test_datasets(tmp_dir, dvc): resolver = DataResolver(dvc, tmp_dir, template) assert resolver.resolve_datasets() == [ - {"name": "dogs", "url": "dvcx://dogs", "type": "dvcx"}, + {"name": "dogs", "url": "dc://dogs", "type": "dc"}, { "name": "example-get-started", "url": "git@github.com:iterative/example-get-started.git", diff --git a/tests/func/test_dataset.py b/tests/func/test_dataset.py index b6a15aff6..098b38c7e 100644 --- a/tests/func/test_dataset.py +++ b/tests/func/test_dataset.py @@ -8,13 +8,13 @@ from attrs import define, evolve, field, has from dvc.dependency.base import Dependency from dvc.exceptions import ReproductionError from dvc.repo.datasets import ( + DatachainDataset, + DatachainDatasetLock, DatasetNotFoundError, DatasetSpec, DVCDataset, DVCDatasetLock, DVCDatasetSpec, - DVCXDataset, - DVCXDatasetLock, FileInfo, URLDataset, URLDatasetLock, @@ -27,7 +27,7 @@ if TYPE_CHECKING: @define -class MockedDVCXVersionInfo: +class MockedDatachainVersionInfo: version: int created_at: datetime = field(factory=lambda: datetime.now(timezone.utc)) @@ -77,21 +77,21 @@ def test_dvc(tmp_dir, scm, dvc: "Repo"): assert old == new -def test_dvcx(tmp_dir, dvc, mocker): +def test_datachain(tmp_dir, dvc, mocker): datasets = dvc.datasets - version_info = [MockedDVCXVersionInfo(1), MockedDVCXVersionInfo(2)] + version_info = [MockedDatachainVersionInfo(1), MockedDatachainVersionInfo(2)] version_info.append(version_info[1]) mocker.patch("dvc.repo.datasets._get_dataset_info", side_effect=version_info) - dataset = datasets.add("mydataset", "dataset", "dvcx") - expected = DVCXDataset( + dataset = datasets.add("mydataset", "dataset", "dc") + expected = DatachainDataset( manifest_path=(tmp_dir / "dvc.yaml").fs_path, - spec=DatasetSpec(name="mydataset", url="dataset", type="dvcx"), - lock=DVCXDatasetLock( + spec=DatasetSpec(name="mydataset", url="dataset", type="dc"), + lock=DatachainDatasetLock( name="mydataset", url="dataset", - type="dvcx", + type="dc", version=1, created_at=version_info[0].created_at, ), @@ -189,16 +189,16 @@ def test_dvc_dump(tmp_dir, dvc): assert dvc.datasets["mydataset"] == dataset -def test_dvcx_dump(tmp_dir, dvc): +def test_datachain_dump(tmp_dir, dvc): manifest_path = os.path.join(tmp_dir, "dvc.yaml") - spec = DatasetSpec(name="mydataset", url="dataset", type="dvcx") + spec = DatasetSpec(name="mydataset", url="dataset", type="dc") dt = datetime.now(tz=timezone.utc) - lock = DVCXDatasetLock(version=1, created_at=dt, **spec.to_dict()) - dataset = DVCXDataset(manifest_path=manifest_path, spec=spec, lock=lock) + lock = DatachainDatasetLock(version=1, created_at=dt, **spec.to_dict()) + dataset = DatachainDataset(manifest_path=manifest_path, spec=spec, lock=lock) dvc.datasets.dump(dataset) - spec_d = {"name": "mydataset", "type": "dvcx", "url": "dataset"} + spec_d = {"name": "mydataset", "type": "dc", "url": "dataset"} assert (tmp_dir / "dvc.yaml").parse() == {"datasets": [spec_d]} assert (tmp_dir / "dvc.lock").parse() == { "schema": "2.0", @@ -244,10 +244,10 @@ def test_url_dump(tmp_dir, dvc): def test_invalidation(tmp_dir, dvc): manifest_path = os.path.join(tmp_dir, "dvc.yaml") spec = DatasetSpec(name="mydataset", url="url1", type="url") - lock = DVCXDatasetLock( + lock = DatachainDatasetLock( name="mydataset", url="dataset", - type="dvcx", + type="dc", version=1, created_at=datetime.now(tz=timezone.utc), ) @@ -289,15 +289,15 @@ def test_dvc_dataset_pipeline(tmp_dir, dvc, scm): assert dvc.reproduce() == [stage] -def test_dvcx_dataset_pipeline(mocker, tmp_dir, dvc): - version_info = [MockedDVCXVersionInfo(1), MockedDVCXVersionInfo(2)] +def test_datachain_dataset_pipeline(mocker, tmp_dir, dvc): + version_info = [MockedDatachainVersionInfo(1), MockedDatachainVersionInfo(2)] mocker.patch("dvc.repo.datasets._get_dataset_info", side_effect=version_info) - dvc.datasets.add("mydataset", "dataset", "dvcx") + dvc.datasets.add("mydataset", "dataset", "dc") stage = dvc.stage.add(cmd="echo", name="train", deps=["ds://mydataset"]) assert (tmp_dir / "dvc.yaml").parse() == { - "datasets": [{"name": "mydataset", "url": "dataset", "type": "dvcx"}], + "datasets": [{"name": "mydataset", "url": "dataset", "type": "dc"}], "stages": {"train": {"cmd": "echo", "deps": ["ds://mydataset"]}}, } @@ -360,10 +360,10 @@ def test_url_dataset_pipeline(mocker, tmp_dir, dvc): def test_pipeline_when_not_in_sync(tmp_dir, dvc): manifest_path = os.path.join(tmp_dir, "dvc.yaml") spec = DatasetSpec(name="mydataset", url="url1", type="url") - lock = DVCXDatasetLock( + lock = DatachainDatasetLock( name="mydataset", url="dataset", - type="dvcx", + type="dc", version=1, created_at=datetime.now(tz=timezone.utc), ) @@ -382,9 +382,9 @@ def test_pipeline_when_not_in_sync(tmp_dir, dvc): def test_collect(tmp_dir, dvc): manifest_path1 = os.path.join(tmp_dir, "dvc.yaml") dt = datetime.now(tz=timezone.utc) - spec = DatasetSpec(name="mydataset1", url="url1", type="dvcx") - lock = DVCXDatasetLock(version=1, created_at=dt, **spec.to_dict()) - mydataset1 = DVCXDataset(manifest_path=manifest_path1, spec=spec, lock=lock) + spec = DatasetSpec(name="mydataset1", url="url1", type="dc") + lock = DatachainDatasetLock(version=1, created_at=dt, **spec.to_dict()) + mydataset1 = DatachainDataset(manifest_path=manifest_path1, spec=spec, lock=lock) dvc.datasets.dump(mydataset1) (tmp_dir / "sub").mkdir() @@ -418,7 +418,7 @@ def test_parametrized(tmp_dir, dvc): (tmp_dir / "dvc.yaml").dump( { "datasets": [ - {"name": "${ds1.name}", "url": "${ds1.url}", "type": "dvcx"}, + {"name": "${ds1.name}", "url": "${ds1.url}", "type": "dc"}, { "name": "${ds2.name}", "url": "${ds2.url}", @@ -435,7 +435,7 @@ def test_parametrized(tmp_dir, dvc): ) (tmp_dir / "params.yaml").dump( { - "ds1": {"name": "dogs", "url": "dvcx://dogs"}, + "ds1": {"name": "dogs", "url": "dc://dogs"}, "ds2": { "name": "example-get-started", "url": "git@github.com:iterative/example-get-started.git", @@ -450,9 +450,9 @@ def test_parametrized(tmp_dir, dvc): path = (tmp_dir / "dvc.yaml").fs_path assert dict(dvc.datasets.items()) == { - "dogs": DVCXDataset( + "dogs": DatachainDataset( manifest_path=path, - spec=DatasetSpec(name="dogs", url="dvcx://dogs", type="dvcx"), + spec=DatasetSpec(name="dogs", url="dc://dogs", type="dc"), ), "example-get-started": DVCDataset( manifest_path=path, diff --git a/tests/unit/command/test_dataset.py b/tests/unit/command/test_dataset.py index e32678c52..290075328 100644 --- a/tests/unit/command/test_dataset.py +++ b/tests/unit/command/test_dataset.py @@ -14,9 +14,9 @@ from dvc.cli import main "Adding ds (url:/path @ main)\n", ), ( - {"name": "mydataset", "url": "dvcx://dataset", "type": "dvcx"}, + {"name": "mydataset", "url": "dc://dataset", "type": "dc"}, {"version": 1, "created_at": datetime.now(tz=timezone.utc)}, - "Adding mydataset (dvcx://dataset @ v1)\n", + "Adding mydataset (dc://dataset @ v1)\n", ), ( {"name": "mydataset", "url": "s3://bucket/path", "type": "url"}, @@ -45,7 +45,7 @@ def test_add_already_exists(dvc, caplog, mocker): dataset = dvc.datasets._build_dataset("dvc.yaml", spec, None) mocker.patch("dvc.repo.datasets.Datasets.get", return_value=dataset) - assert main(["dataset", "add", "ds", "--dvcx", "dataset"]) == 255 + assert main(["dataset", "add", "ds", "--dc", "dataset"]) == 255 assert "ds already exists in dvc.yaml, use the --force to overwrite" in caplog.text @@ -64,21 +64,21 @@ def test_add_already_exists(dvc, caplog, mocker): }, ), ( - {"name": "mydataset", "url": "dvcx://dataset", "type": "dvcx"}, + {"name": "mydataset", "url": "dc://dataset", "type": "dc"}, {"version": 1, "created_at": datetime.now(tz=timezone.utc)}, {"version": 2}, { - "missing": "Updating mydataset (dvcx://dataset @ v2)\n", + "missing": "Updating mydataset (dc://dataset @ v2)\n", "unchanged": "Nothing to update\n", "updated": "Updating mydataset (v1 -> v2)\n", }, ), ( - {"name": "mydataset", "url": "dvcx://dataset", "type": "dvcx"}, + {"name": "mydataset", "url": "dc://dataset", "type": "dc"}, {"version": 2, "created_at": datetime.now(tz=timezone.utc)}, {"version": 1}, { - "missing": "Updating mydataset (dvcx://dataset @ v1)\n", + "missing": "Updating mydataset (dc://dataset @ v1)\n", "unchanged": "Nothing to update\n", "updated": "Downgrading mydataset (v2 -> v1)\n", },