Files
dvc/dvc/fs/base.py
T

385 lines
11 KiB
Python

import contextlib
import logging
import os
from concurrent.futures import ThreadPoolExecutor, as_completed
from functools import partialmethod
from multiprocessing import cpu_count
from typing import ClassVar, Dict, FrozenSet, Optional
from funcy import cached_property
from tqdm.utils import CallbackIOWrapper
from dvc.exceptions import DvcException
from dvc.progress import DEFAULT_CALLBACK, FsspecCallback
from dvc.ui import ui
from dvc.utils import tmp_fname
from dvc.utils.fs import makedirs, move
logger = logging.getLogger(__name__)
class RemoteCmdError(DvcException):
def __init__(self, remote, cmd, ret, err):
super().__init__(
"{remote} command '{cmd}' finished with non-zero return code"
" {ret}': {err}".format(remote=remote, cmd=cmd, ret=ret, err=err)
)
class RemoteActionNotImplemented(DvcException):
def __init__(self, action, scheme):
m = f"{action} is not supported for {scheme} remotes"
super().__init__(m)
class RemoteMissingDepsError(DvcException):
pass
class BaseFileSystem:
sep = "/"
scheme = "base"
REQUIRES: ClassVar[Dict[str, str]] = {}
_JOBS = 4 * cpu_count()
CHECKSUM_DIR_SUFFIX = ".dir"
HASH_JOBS = max(1, min(4, cpu_count() // 2))
LIST_OBJECT_PAGE_SIZE = 1000
TRAVERSE_WEIGHT_MULTIPLIER = 5
TRAVERSE_PREFIX_LEN = 3
TRAVERSE_THRESHOLD_SIZE = 500000
CAN_TRAVERSE = True
# Needed for some providers, and http open()
CHUNK_SIZE = 64 * 1024 * 1024 # 64 MiB
PARAM_CHECKSUM: ClassVar[Optional[str]] = None
DETAIL_FIELDS: FrozenSet[str] = frozenset()
def __init__(self, **kwargs):
self._check_requires(**kwargs)
self.jobs = kwargs.get("jobs") or self._JOBS
self.hash_jobs = kwargs.get("checksum_jobs") or self.HASH_JOBS
self._config = kwargs
@property
def config(self):
return self._config
@cached_property
def path(self):
from .path import Path
return Path(self.sep)
@classmethod
def _strip_protocol(cls, path: str):
return path
def unstrip_protocol(self, path):
return path
@staticmethod
def _get_kwargs_from_urls(urlpath): # pylint:disable=unused-argument
return {}
@classmethod
def get_missing_deps(cls):
import importlib
missing = []
for package, module in cls.REQUIRES.items():
try:
importlib.import_module(module)
except ImportError:
missing.append(package)
return missing
def _check_requires(self, **kwargs):
from ..scheme import Schemes
from ..utils import format_link
from ..utils.pkg import PKG
missing = self.get_missing_deps()
if not missing:
return
url = kwargs.get("url", f"{self.scheme}://")
scheme = self.scheme
if scheme == Schemes.WEBDAVS:
scheme = Schemes.WEBDAV
by_pkg = {
"pip": f"pip install 'dvc[{scheme}]'",
"conda": f"conda install -c conda-forge dvc-{scheme}",
}
cmd = by_pkg.get(PKG)
if cmd:
link = format_link("https://dvc.org/doc/install")
hint = (
f"To install dvc with those dependencies, run:\n"
"\n"
f"\t{cmd}\n"
"\n"
f"See {link} for more info."
)
else:
link = format_link("https://github.com/iterative/dvc/issues")
hint = f"Please report this bug to {link}. Thank you!"
raise RemoteMissingDepsError(
f"URL '{url}' is supported but requires these missing "
f"dependencies: {missing}. {hint}"
)
def checksum(self, path) -> str:
raise NotImplementedError
def open(self, path, mode: str = "r", encoding: str = None, **kwargs):
raise RemoteActionNotImplemented("open", self.scheme)
def exists(self, path) -> bool:
raise NotImplementedError
# pylint: disable=unused-argument
def isdir(self, path):
"""Optional: Overwrite only if the remote has a way to distinguish
between a directory and a file.
"""
return False
def isfile(self, path):
"""Optional: Overwrite only if the remote has a way to distinguish
between a directory and a file.
"""
return True
def isexec(self, path):
"""Optional: Overwrite only if the remote has a way to distinguish
between executable and non-executable file.
"""
return False
def iscopy(self, path):
"""Check if this file is an independent copy."""
return False # We can't be sure by default
def walk(self, top, topdown=True, onerror=None, **kwargs):
"""Return a generator with (root, dirs, files)."""
raise NotImplementedError
def find(self, path, prefix=None):
"""Return a generator with `str`s to all the files.
Optional kwargs:
prefix (bool): If true `path` will be treated as a prefix
rather than directory path.
"""
raise NotImplementedError
def is_empty(self, path):
return False
def info(self, path):
raise NotImplementedError
def getsize(self, path):
return self.info(path).get("size")
def remove(self, path):
raise RemoteActionNotImplemented("remove", self.scheme)
def makedirs(self, path, **kwargs):
"""Optional: Implement only if the remote needs to create
directories before copying/linking/moving data
"""
def move(self, from_info, to_info):
self.copy(from_info, to_info)
self.remove(from_info)
def copy(self, from_info, to_info):
raise RemoteActionNotImplemented("copy", self.scheme)
def symlink(self, from_info, to_info):
raise RemoteActionNotImplemented("symlink", self.scheme)
def hardlink(self, from_info, to_info):
raise RemoteActionNotImplemented("hardlink", self.scheme)
def reflink(self, from_info, to_info):
raise RemoteActionNotImplemented("reflink", self.scheme)
# pylint: enable=unused-argument
@classmethod
def is_dir_hash(cls, hash_):
if not hash_:
return False
return hash_.endswith(cls.CHECKSUM_DIR_SUFFIX)
def upload(
self,
from_info,
to_info,
total=None,
desc=None,
callback=None,
no_progress_bar=False,
**pbar_args,
):
is_file_obj = hasattr(from_info, "read")
method = "upload_fobj" if is_file_obj else "put_file"
if not hasattr(self, method):
raise RemoteActionNotImplemented(method, self.scheme)
if not is_file_obj:
from .local import localfs
desc = desc or localfs.path.name(from_info)
stack = contextlib.ExitStack()
if not callback:
pbar = ui.progress(
desc=desc,
disable=no_progress_bar,
bytes=True,
total=total or -1,
**pbar_args,
)
stack.enter_context(pbar)
callback = pbar.as_callback()
if total:
callback.set_size(total)
with stack:
if is_file_obj:
wrapped = CallbackIOWrapper(
callback.relative_update, from_info
)
# `size` is used to provide hints to the WebdavFileSystem
# for legacy servers.
# pylint: disable=no-member
return self.upload_fobj(wrapped, to_info, size=total)
logger.debug("Uploading '%s' to '%s'", from_info, to_info)
# pylint: disable=no-member
return self.put_file(
os.fspath(from_info), to_info, callback=callback
)
def download(
self,
from_info,
to_info,
name=None,
callback=None,
no_progress_bar=False,
jobs=None,
_only_file=False,
**kwargs,
):
from .local import localfs
if not hasattr(self, "get_file"):
raise RemoteActionNotImplemented("get_file", self.scheme)
download_dir = not _only_file and self.isdir(from_info)
desc = name or localfs.path.name(to_info)
stack = contextlib.ExitStack()
if not callback:
pbar_kwargs = {"unit": "Files"} if download_dir else {}
pbar = ui.progress(
total=-1,
desc="Downloading directory" if download_dir else desc,
bytes=not download_dir,
disable=no_progress_bar,
**pbar_kwargs,
)
stack.enter_context(pbar)
callback = pbar.as_callback()
with stack:
if download_dir:
return self._download_dir(
from_info, to_info, callback=callback, jobs=jobs, **kwargs
)
return self._download_file(from_info, to_info, callback=callback)
download_file = partialmethod(download, _only_file=True)
def _download_dir(
self,
from_info,
to_info,
callback=DEFAULT_CALLBACK,
jobs=None,
**kwargs,
):
from .local import localfs
from_infos = list(self.find(from_info, **kwargs))
if not from_infos:
makedirs(to_info, exist_ok=True)
return None
to_infos = (
localfs.path.join(to_info, *self.path.relparts(info, from_info))
for info in from_infos
)
callback.set_size(len(from_infos))
download_files = FsspecCallback.wrap_fn(callback, self._download_file)
max_workers = jobs or self.jobs
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = [
executor.submit(download_files, from_info, to_info)
for from_info, to_info in zip(from_infos, to_infos)
]
# NOTE: unlike pulling/fetching cache, where we need to
# download everything we can, not raising an error here might
# turn very ugly, as the user might think that he has
# downloaded a complete directory, while having a partial one,
# which might cause unexpected results in his pipeline.
for future in as_completed(futures):
# NOTE: executor won't let us raise until all futures that
# it has are finished, so we need to cancel them ourselves
# before re-raising.
exc = future.exception()
if exc:
for entry in futures:
entry.cancel()
raise exc
def _download_file(
self,
from_info,
to_info,
callback=DEFAULT_CALLBACK,
):
from .local import localfs
makedirs(localfs.path.parent(to_info), exist_ok=True)
tmp_file = tmp_fname(to_info)
logger.debug("Downloading '%s' to '%s'", from_info, to_info)
try:
# noqa, pylint: disable=no-member
self.get_file(from_info, tmp_file, callback=callback)
except Exception: # pylint: disable=broad-except
# do we need to rollback makedirs for previously not-existing
# directories?
with contextlib.suppress(FileNotFoundError):
os.unlink(tmp_file)
raise
move(tmp_file, to_info)