diff --git a/dvc/fs/dvc_path.py b/dvc/fs/dvc_path.py new file mode 100644 index 000000000..1a181f8b2 --- /dev/null +++ b/dvc/fs/dvc_path.py @@ -0,0 +1,54 @@ +"""UPath implementation for DVCFileSystem. + +This provides a `pathlib.Path` like interface to +work with DVCFileSystem. + +Examples +-------- + +>>> from upath import UPath + +>>> local = UPath("dvc://path/to/local/repo") +>>> https = UPath("dvc+https://github.com/iterative/example-get-started", rev="main") +>>> ssh = UPath("dvc+ssh://git@github.com:iterative/example-get-started.git") +""" + +from urllib.parse import urlsplit + +from upath import UPath + + +class DVCPath(UPath): + @classmethod + def _transform_init_args(cls, args, protocol, storage_options): + if not args: + args = ("/",) + elif ( + args + and "url" not in storage_options + and protocol in {"dvc+http", "dvc+https", "dvc+ssh"} + ): + url, *rest = args + url = urlsplit(str(url)) + proto = protocol.split("+")[1] + if proto == "ssh": + base_url = url.netloc + url.path + else: + base_url = url._replace(scheme=proto).geturl() + storage_options["url"] = base_url + # Assume the given path is a root url + args = ("/", *rest) + return super()._transform_init_args(args, "dvc", storage_options) + + def __str__(self): + s = super().__str__() + if url := self.storage_options.get("url"): + return s.replace("dvc://", f"dvc+{url}", 1) + return s + + def with_segments(self, *pathsegments): + obj = super().with_segments(*pathsegments) + # cache filesystem, as dvcfs does not cache filesystem + # caveat: any joinpath operation will instantiate filesystem + obj._fs_cached = self.fs + return obj diff --git a/pyproject.toml b/pyproject.toml index 1904067a5..ff7c37e5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -130,6 +130,14 @@ dvc = "dvc.cli:main" [project.entry-points."fsspec.specs"] dvc = "dvc.api:DVCFileSystem" +[project.entry-points."universal_pathlib.implementations"] +dvc = "dvc.fs.dvc_path:DVCPath" +# universal_pathlib does not support fsspec url chaining yet. +# see https://github.com/fsspec/universal_pathlib/issues/28. +"dvc+http" = "dvc.fs.dvc_path:DVCPath" +"dvc+https" = "dvc.fs.dvc_path:DVCPath" +"dvc+ssh" = "dvc.fs.dvc_path:DVCPath" + [project.entry-points."pyinstaller40"] hook-dirs = "dvc.__pyinstaller:get_hook_dirs" tests = "dvc.__pyinstaller:get_PyInstaller_tests" @@ -232,6 +240,7 @@ module = [ "ruamel.yaml.*", "shortuuid", "shtab", + "upath", "virtualenv", "viztracer", "voluptuous",