From 15e10a1d81a45018d61d5287c20c73c3aa49c266 Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Fri, 27 May 2022 12:49:45 +0200 Subject: [PATCH] api: open: Add docstring and type hints. --- dvc/api.py | 170 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 144 insertions(+), 26 deletions(-) diff --git a/dvc/api.py b/dvc/api.py index a7f9d663d..af719c119 100644 --- a/dvc/api.py +++ b/dvc/api.py @@ -1,5 +1,6 @@ import os from contextlib import _GeneratorContextManager as GCM +from typing import Optional from funcy import reraise @@ -33,32 +34,6 @@ def get_url(path, repo=None, rev=None, remote=None): return dvc_repo.cloud.get_url_for(remote, checksum=md5) -def open( # noqa, pylint: disable=redefined-builtin - path, repo=None, rev=None, remote=None, mode="r", encoding=None -): - """ - Open file in the supplied path tracked in a repo (both DVC projects and - plain Git repos are supported). For Git repos, HEAD is used unless a rev - argument is supplied. The default remote is tried unless a remote argument - is supplied. It may only be used as a context manager: - - with dvc.api.open( - 'path/to/file', - repo='https://example.com/url/to/repo' - ) as fd: - # ... Handle file object fd - """ - args = (path,) - kwargs = { - "repo": repo, - "remote": remote, - "rev": rev, - "mode": mode, - "encoding": encoding, - } - return _OpenContextManager(_open, args, kwargs) - - class _OpenContextManager(GCM): def __init__( self, func, args, kwds @@ -72,6 +47,149 @@ class _OpenContextManager(GCM): ) +def open( # noqa, pylint: disable=redefined-builtin + path: str, + repo: Optional[str] = None, + rev: Optional[str] = None, + remote: Optional[str] = None, + mode: str = "r", + encoding: Optional[str] = None, +): + """ + Opens a file tracked in a DVC project. + + This function may only be used as a context manager (using the `with` + keyword, as shown in the examples). + + This function makes a direct connection to the remote storage, so the file + contents can be streamed. Your code can process the data buffer as it's + streamed, which optimizes memory usage. + + Note: + Use dvc.api.read() to load the complete file contents + in a single function call, no context manager involved. + Neither function utilizes disc space. + + Args: + path (str): location and file name of the target to open, + relative to the root of `repo`. + repo (str, optional): location of the DVC project or Git Repo. + Defaults to the current DVC project (found by walking up from the + current working directory tree). + It can be a URL or a file system path. + Both HTTP and SSH protocols are supported for online Git repos + (e.g. [user@]server:project.git). + rev (str, optional): Any `Git revision`_ such as a branch or tag name, + a commit hash or a dvc experiment name. + Defaults to HEAD. + If `repo` is not a Git repo, this option is ignored. + remote (str, optional): Name of the `DVC remote`_ used to form the + returned URL string. + Defaults to the `default remote`_ of `repo`. + For local projects, the cache is tried before the default remote. + mode (str, optional): Specifies the mode in which the file is opened. + Defaults to "r" (read). + Mirrors the namesake parameter in builtin `open()`_. + encoding(str, optional): `Codec`_ used to decode the file contents. + Defaults to None. + This should only be used in text mode. + Mirrors the namesake parameter in builtin `open()`_. + + Returns: + _OpenContextManager: A context manager that generatse a corresponding + `file object`_. + The exact type of file object depends on the mode used. + For more details, please refer to Python's `open()`_ built-in, + which is used under the hood. + + Raises: + AttributeError: If this method is not used as a context manager. + + Examples: + + - Use data or models from a DVC repository. + + Any file tracked in a DVC project (and stored remotely) can be + processed directly in your Python code with this API. + For example, an XML file tracked in a public DVC repo on GitHub can be + processed like this: + + >>> from xml.sax import parse + >>> import dvc.api + >>> from mymodule import mySAXHandler + + >>> with dvc.api.open( + ... 'get-started/data.xml', + ... repo='https://github.com/iterative/dataset-registry' + ... ) as fd: + ... parse(fd, mySAXHandler) + + We use a SAX XML parser here because dvc.api.open() is able to stream + the data from remote storage. + The mySAXHandler object should handle the event-driven parsing of the + document in this case. + This increases the performance of the code (minimizing memory usage), + and is typically faster than loading the whole data into memory. + + - Accessing private repos + + This is just a matter of using the right repo argument, for example an + SSH URL (requires that the credentials are configured locally): + + >>> import dvc.api + + >>> with dvc.api.open( + ... 'features.dat', + ... repo='git@server.com:path/to/repo.git' + ... ) as fd: + ... # ... Process 'features' + ... pass + + - Use different versions of data + + Any git revision (see `rev`) can be accessed programmatically. + For example, if your DVC repo has tagged releases of a CSV dataset: + + >>> import csv + >>> import dvc.api + >>> with dvc.api.open( + ... 'clean.csv', + ... rev='v1.1.0' + ... ) as fd: + ... reader = csv.reader(fd) + ... # ... Process 'clean' data from version 1.1.0 + + .. _Git revision: + https://git-scm.com/docs/revisions + + .. _DVC remote: + https://dvc.org/doc/command-reference/remote + + .. _default remote: + https://dvc.org/doc/command-reference/remote/default + + .. _open(): + https://docs.python.org/3/library/functions.html#open + + .. _Codec: + https://docs.python.org/3/library/codecs.html#standard-encodings + + .. _file object: + https://docs.python.org/3/glossary.html#term-file-object + + """ + + args = (path,) + kwargs = { + "repo": repo, + "remote": remote, + "rev": rev, + "mode": mode, + "encoding": encoding, + } + return _OpenContextManager(_open, args, kwargs) + + def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None): with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo: with _repo.open_by_relpath(