mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
* [pre-commit.ci] pre-commit autoupdate updates: - [github.com/astral-sh/ruff-pre-commit: v0.12.12 → v0.13.0](https://github.com/astral-sh/ruff-pre-commit/compare/v0.12.12...v0.13.0) * fix ruff errors --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Saugat Pachhai (सौगात) <[email protected]>
68 lines
2.0 KiB
Python
68 lines
2.0 KiB
Python
from typing import Literal, TypedDict, Union
|
|
|
|
|
|
class DatachainDataset(TypedDict):
|
|
type: Literal["dc"]
|
|
name: str
|
|
version: int
|
|
|
|
|
|
class DVCDataset(TypedDict):
|
|
type: Literal["dvc"]
|
|
url: str
|
|
path: str
|
|
sha: str
|
|
|
|
|
|
class URLDataset(TypedDict):
|
|
type: Literal["url"]
|
|
files: list[str]
|
|
path: str
|
|
|
|
|
|
def get(name: str) -> Union[DatachainDataset, DVCDataset, URLDataset]:
|
|
from difflib import get_close_matches
|
|
|
|
from dvc.fs import get_cloud_fs
|
|
from dvc.repo import Repo, datasets
|
|
|
|
repo = Repo()
|
|
try:
|
|
dataset = repo.datasets[name]
|
|
except datasets.DatasetNotFoundError as e:
|
|
add_note = getattr(e, "add_note", lambda _: None)
|
|
if matches := get_close_matches(name, repo.datasets):
|
|
add_note(f"Did you mean: {matches[0]!r}?")
|
|
raise
|
|
|
|
if dataset._invalidated:
|
|
raise ValueError(f"dataset not in sync. Sync with 'dvc ds update {name}'.")
|
|
if not dataset.lock:
|
|
raise ValueError("missing lock information")
|
|
if dataset.type == "dvc":
|
|
return DVCDataset(
|
|
type="dvc",
|
|
url=dataset.lock.url,
|
|
path=dataset.lock.path,
|
|
sha=dataset.lock.rev_lock,
|
|
)
|
|
if dataset.type == "dc":
|
|
return DatachainDataset(
|
|
type="dc", name=dataset.name_version[0], version=dataset.lock.version
|
|
)
|
|
if dataset.type == "url":
|
|
fs_cls, _, path = get_cloud_fs(repo.config, url=dataset.lock.url)
|
|
assert fs_cls
|
|
join_version = getattr(fs_cls, "join_version", lambda path, _: path)
|
|
protocol = fs_cls.protocol
|
|
versioned_path = join_version(path, dataset.lock.meta.version_id)
|
|
versioned_path = f"{protocol}://{versioned_path}"
|
|
files = [
|
|
join_version(
|
|
fs_cls.join(versioned_path, file.relpath), file.meta.version_id
|
|
)
|
|
for file in dataset.lock.files
|
|
]
|
|
return URLDataset(type="url", files=files, path=versioned_path)
|
|
raise AssertionError("unreachable")
|