Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-01 23:08:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

deps: bump dvc-objects, dvc-data and scmrepo (#10280)

Browse Source 
* deps: bump dvc-objects, dvc-data and scmrepo

* bump dvc-azure

* fix typo
This commit is contained in:
skshetry
2024-02-06 03:36:37 +02:00
committed by GitHub
parent 79238fd2a3
commit d25cb79e0a
5 changed files with 65 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/commands/ls/__init__.py
+3
View File
@@ -57,6 +57,9 @@ class CmdList(CmdBaseNoRepo):
elif entries:
show_entries(entries, with_color=True, with_size=self.args.size)
return 0
except FileNotFoundError:
logger.exception("")
return 1
except DvcException:
logger.exception("failed to list '%s'", self.args.url)
return 1
dvc/fs/callbacks.py
+18 -57
View File
@@ -1,67 +1,38 @@
from contextlib import ExitStack
from typing import TYPE_CHECKING, Any, BinaryIO, Optional, Union
from typing import TYPE_CHECKING, BinaryIO, Optional, Union
import fsspec
from fsspec.callbacks import DEFAULT_CALLBACK, Callback # noqa: F401
from dvc.progress import Tqdm
from dvc.utils.objects import cached_property
from dvc_objects.fs.callbacks import DEFAULT_CALLBACK, Callback # noqa: F401
if TYPE_CHECKING:
from rich.progress import TaskID
from tqdm import tqdm
from dvc.ui._rich_progress import RichTransferProgress
# NOTE: this is very similar to dvc-objects.fs.callbacks.TqdmCallback,
# but it works with our own Tqdm instance.
class TqdmCallback(Callback):
class TqdmCallback(fsspec.callbacks.TqdmCallback):
def __init__(
self,
size: Optional[int] = None,
value: int = 0,
progress_bar: Optional["Tqdm"] = None,
progress_bar: Optional["tqdm"] = None,
tqdm_cls: Optional[type["tqdm"]] = None,
**tqdm_kwargs,
):
tqdm_kwargs["total"] = size or -1
self._tqdm_kwargs = tqdm_kwargs
self._progress_bar = progress_bar
self._stack = ExitStack()
super().__init__(size=size, value=value)
@cached_property
def progress_bar(self):
progress_bar = (
self._progress_bar
if self._progress_bar is not None
else Tqdm(**self._tqdm_kwargs)
tqdm_kwargs.pop("total", None)
super().__init__(
tqdm_kwargs=tqdm_kwargs, tqdm_cls=tqdm_cls or Tqdm, size=size, value=value
)
return self._stack.enter_context(progress_bar)
if progress_bar is not None:
self.tqdm = progress_bar
def __enter__(self):
return self
def close(self):
self._stack.close()
def set_size(self, size):
# Tqdm tries to be smart when to refresh,
# so we try to force it to re-render.
super().set_size(size)
self.progress_bar.refresh()
def call(self, hook_name=None, **kwargs): # noqa: ARG002
self.progress_bar.update_to(self.value, total=self.size)
def branch(
self,
path_1: "Union[str, BinaryIO]",
path_2: str,
kwargs: dict[str, Any],
child: Optional[Callback] = None,
):
child = child or TqdmCallback(
bytes=True, desc=path_1 if isinstance(path_1, str) else path_2
)
return super().branch(path_1, path_2, kwargs, child=child)
def branched(self, path_1: "Union[str, BinaryIO]", path_2: str, **kwargs):
desc = path_1 if isinstance(path_1, str) else path_2
return TqdmCallback(bytes=True, desc=desc)
class RichCallback(Callback):
@@ -110,9 +81,6 @@ class RichCallback(Callback):
def task(self) -> "TaskID":
return self.progress.add_task(**self._task_kwargs) # type: ignore[arg-type]
def __enter__(self):
return self
def close(self):
if self._transient:
self.progress.clear_task(self.task)
@@ -126,17 +94,10 @@ class RichCallback(Callback):
visible=not self.disable,
)
def branch(
self,
path_1: Union[str, BinaryIO],
path_2: str,
kwargs: dict[str, Any],
child: Optional["Callback"] = None,
):
child = child or RichCallback(
def branched(self, path_1: Union[str, BinaryIO], path_2: str, **kwargs):
return RichCallback(
progress=self.progress,
desc=path_1 if isinstance(path_1, str) else path_2,
bytes=True,
transient=self._transient,
)
return super().branch(path_1, path_2, kwargs, child=child)
dvc/fs/dvc.py
+24 -15
View File
@@ -360,38 +360,47 @@ class _DVCFileSystem(AbstractFileSystem):
except FileNotFoundError:
return False
def ls(self, path, detail=True, dvc_only=False, **kwargs):
def ls(self, path, detail=True, dvc_only=False, **kwargs): # noqa: C901
key = self._get_key_from_relative(path)
repo, dvc_fs, subkey = self._get_subrepo_info(key)
dvc_exists = False
dvc_infos = {}
dvc_info = {}
if dvc_fs:
dvc_path = _get_dvc_path(dvc_fs, subkey)
with suppress(FileNotFoundError, NotADirectoryError):
for info in dvc_fs.ls(dvc_path, detail=True):
dvc_infos[dvc_fs.name(info["name"])] = info
dvc_exists = True
with suppress(FileNotFoundError):
dvc_info = dvc_fs.info(dvc_path)
if dvc_info["type"] == "file":
dvc_infos[""] = dvc_info
else:
for info in dvc_fs.ls(dvc_path, detail=True):
dvc_infos[dvc_fs.name(info["name"])] = info
fs_exists = False
fs_infos = {}
fs_info = {}
ignore_subrepos = kwargs.get("ignore_subrepos", True)
if not dvc_only:
fs = self.repo.fs
fs_path = self._from_key(key)
try:
for info in repo.dvcignore.ls(
fs, fs_path, detail=True, ignore_subrepos=ignore_subrepos
):
fs_infos[fs.name(info["name"])] = info
fs_exists = True
fs_info = fs.info(fs_path)
if fs_info["type"] == "file":
fs_infos[""] = fs_info
else:
for info in repo.dvcignore.ls(
fs, fs_path, detail=True, ignore_subrepos=ignore_subrepos
):
fs_infos[fs.name(info["name"])] = info
except (FileNotFoundError, NotADirectoryError):
pass
if not (dvc_exists or fs_exists):
if not (fs_info or dvc_info):
# broken symlink or TreeError
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path)
if fs_info and dvc_info and dvc_info["type"] != fs_info["type"]:
dvc_infos.clear() # invalidate dvc_info if file type differs
dvcfiles = kwargs.get("dvcfiles", False)
infos = []
@@ -402,7 +411,7 @@ class _DVCFileSystem(AbstractFileSystem):
if not dvcfiles and _is_dvc_file(name):
continue
entry_path = self.join(path, name)
entry_path = self.join(path, name) if name else path
info = _merge_info(
repo, (*subkey, name), fs_infos.get(name), dvc_infos.get(name)
)
@@ -459,7 +468,7 @@ class _DVCFileSystem(AbstractFileSystem):
continue
if not dvc_info and not fs_info:
raise FileNotFoundError
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path)
info = _merge_info(repo, subkey, fs_info, dvc_info)
info["name"] = path
dvc/repo/ls.py
+17 -17
View File
@@ -83,24 +83,24 @@ def _ls(
fs_path = fs.info(fs_path)["name"]
infos = {}
for root, dirs, files in fs.walk(
fs_path, dvcfiles=True, dvc_only=dvc_only, detail=True
):
if not recursive:
files.update(dirs)
parts = fs.relparts(root, fs_path)
if parts == (".",):
parts = ()
for name, entry in files.items():
infos[os.path.join(*parts, name)] = entry
if not recursive:
break
if not infos and fs.isfile(fs_path):
if fs.isfile(fs_path):
infos[os.path.basename(path)] = fs.info(fs_path)
else:
for root, dirs, files in fs.walk(
fs_path, dvcfiles=True, dvc_only=dvc_only, detail=True
):
if not recursive:
files.update(dirs)
parts = fs.relparts(root, fs_path)
if parts == (".",):
parts = ()
for name, entry in files.items():
infos[os.path.join(*parts, name)] = entry
if not recursive:
break
ret = {}
for name, info in infos.items():
pyproject.toml
+3 -3
View File
@@ -35,7 +35,7 @@ dependencies = [
"configobj>=5.0.6",
"distro>=1.3",
"dpath<3,>=2.1.0",
"dvc-data>=3.9,<3.10",
"dvc-data>=3.10,<3.11",
"dvc-http>=2.29.0",
"dvc-render>=1.0.1,<2",
"dvc-studio-client>=0.17.1,<1",
@@ -59,7 +59,7 @@ dependencies = [
"requests>=2.22",
"rich>=12",
"ruamel.yaml>=0.17.11",
"scmrepo>=2.0.2,<3",
"scmrepo>=3,<4",
"shortuuid>=0.5",
"shtab<2,>=1.3.4",
"tabulate>=0.8.7",
@@ -71,7 +71,7 @@ dependencies = [
[project.optional-dependencies]
all = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs]"]
azure = ["dvc-azure>=3,<4"]
azure = ["dvc-azure>=3.1.0,<4"]
dev = ["dvc[azure,gdrive,gs,hdfs,lint,oss,s3,ssh,tests,webdav,webhdfs]"]
gdrive = ["dvc-gdrive>=3,<4"]
gs = ["dvc-gs>=3,<4"]