diff --git a/dvc/commands/ls/__init__.py b/dvc/commands/ls/__init__.py index be7150bb5..455e3d9a3 100644 --- a/dvc/commands/ls/__init__.py +++ b/dvc/commands/ls/__init__.py @@ -57,6 +57,9 @@ class CmdList(CmdBaseNoRepo): elif entries: show_entries(entries, with_color=True, with_size=self.args.size) return 0 + except FileNotFoundError: + logger.exception("") + return 1 except DvcException: logger.exception("failed to list '%s'", self.args.url) return 1 diff --git a/dvc/fs/callbacks.py b/dvc/fs/callbacks.py index aa6f7aec9..d30ad4f24 100644 --- a/dvc/fs/callbacks.py +++ b/dvc/fs/callbacks.py @@ -1,67 +1,38 @@ from contextlib import ExitStack -from typing import TYPE_CHECKING, Any, BinaryIO, Optional, Union +from typing import TYPE_CHECKING, BinaryIO, Optional, Union + +import fsspec +from fsspec.callbacks import DEFAULT_CALLBACK, Callback # noqa: F401 from dvc.progress import Tqdm from dvc.utils.objects import cached_property -from dvc_objects.fs.callbacks import DEFAULT_CALLBACK, Callback # noqa: F401 if TYPE_CHECKING: from rich.progress import TaskID + from tqdm import tqdm from dvc.ui._rich_progress import RichTransferProgress -# NOTE: this is very similar to dvc-objects.fs.callbacks.TqdmCallback, -# but it works with our own Tqdm instance. -class TqdmCallback(Callback): +class TqdmCallback(fsspec.callbacks.TqdmCallback): def __init__( self, size: Optional[int] = None, value: int = 0, - progress_bar: Optional["Tqdm"] = None, + progress_bar: Optional["tqdm"] = None, + tqdm_cls: Optional[type["tqdm"]] = None, **tqdm_kwargs, ): - tqdm_kwargs["total"] = size or -1 - self._tqdm_kwargs = tqdm_kwargs - self._progress_bar = progress_bar - self._stack = ExitStack() - super().__init__(size=size, value=value) - - @cached_property - def progress_bar(self): - progress_bar = ( - self._progress_bar - if self._progress_bar is not None - else Tqdm(**self._tqdm_kwargs) + tqdm_kwargs.pop("total", None) + super().__init__( + tqdm_kwargs=tqdm_kwargs, tqdm_cls=tqdm_cls or Tqdm, size=size, value=value ) - return self._stack.enter_context(progress_bar) + if progress_bar is not None: + self.tqdm = progress_bar - def __enter__(self): - return self - - def close(self): - self._stack.close() - - def set_size(self, size): - # Tqdm tries to be smart when to refresh, - # so we try to force it to re-render. - super().set_size(size) - self.progress_bar.refresh() - - def call(self, hook_name=None, **kwargs): # noqa: ARG002 - self.progress_bar.update_to(self.value, total=self.size) - - def branch( - self, - path_1: "Union[str, BinaryIO]", - path_2: str, - kwargs: dict[str, Any], - child: Optional[Callback] = None, - ): - child = child or TqdmCallback( - bytes=True, desc=path_1 if isinstance(path_1, str) else path_2 - ) - return super().branch(path_1, path_2, kwargs, child=child) + def branched(self, path_1: "Union[str, BinaryIO]", path_2: str, **kwargs): + desc = path_1 if isinstance(path_1, str) else path_2 + return TqdmCallback(bytes=True, desc=desc) class RichCallback(Callback): @@ -110,9 +81,6 @@ class RichCallback(Callback): def task(self) -> "TaskID": return self.progress.add_task(**self._task_kwargs) # type: ignore[arg-type] - def __enter__(self): - return self - def close(self): if self._transient: self.progress.clear_task(self.task) @@ -126,17 +94,10 @@ class RichCallback(Callback): visible=not self.disable, ) - def branch( - self, - path_1: Union[str, BinaryIO], - path_2: str, - kwargs: dict[str, Any], - child: Optional["Callback"] = None, - ): - child = child or RichCallback( + def branched(self, path_1: Union[str, BinaryIO], path_2: str, **kwargs): + return RichCallback( progress=self.progress, desc=path_1 if isinstance(path_1, str) else path_2, bytes=True, transient=self._transient, ) - return super().branch(path_1, path_2, kwargs, child=child) diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index 6a99f08f5..fefb1ea45 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -360,38 +360,47 @@ class _DVCFileSystem(AbstractFileSystem): except FileNotFoundError: return False - def ls(self, path, detail=True, dvc_only=False, **kwargs): + def ls(self, path, detail=True, dvc_only=False, **kwargs): # noqa: C901 key = self._get_key_from_relative(path) repo, dvc_fs, subkey = self._get_subrepo_info(key) - dvc_exists = False dvc_infos = {} + dvc_info = {} if dvc_fs: dvc_path = _get_dvc_path(dvc_fs, subkey) - with suppress(FileNotFoundError, NotADirectoryError): - for info in dvc_fs.ls(dvc_path, detail=True): - dvc_infos[dvc_fs.name(info["name"])] = info - dvc_exists = True + with suppress(FileNotFoundError): + dvc_info = dvc_fs.info(dvc_path) + if dvc_info["type"] == "file": + dvc_infos[""] = dvc_info + else: + for info in dvc_fs.ls(dvc_path, detail=True): + dvc_infos[dvc_fs.name(info["name"])] = info - fs_exists = False fs_infos = {} + fs_info = {} ignore_subrepos = kwargs.get("ignore_subrepos", True) if not dvc_only: fs = self.repo.fs fs_path = self._from_key(key) try: - for info in repo.dvcignore.ls( - fs, fs_path, detail=True, ignore_subrepos=ignore_subrepos - ): - fs_infos[fs.name(info["name"])] = info - fs_exists = True + fs_info = fs.info(fs_path) + if fs_info["type"] == "file": + fs_infos[""] = fs_info + else: + for info in repo.dvcignore.ls( + fs, fs_path, detail=True, ignore_subrepos=ignore_subrepos + ): + fs_infos[fs.name(info["name"])] = info except (FileNotFoundError, NotADirectoryError): pass - if not (dvc_exists or fs_exists): + if not (fs_info or dvc_info): # broken symlink or TreeError raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) + if fs_info and dvc_info and dvc_info["type"] != fs_info["type"]: + dvc_infos.clear() # invalidate dvc_info if file type differs + dvcfiles = kwargs.get("dvcfiles", False) infos = [] @@ -402,7 +411,7 @@ class _DVCFileSystem(AbstractFileSystem): if not dvcfiles and _is_dvc_file(name): continue - entry_path = self.join(path, name) + entry_path = self.join(path, name) if name else path info = _merge_info( repo, (*subkey, name), fs_infos.get(name), dvc_infos.get(name) ) @@ -459,7 +468,7 @@ class _DVCFileSystem(AbstractFileSystem): continue if not dvc_info and not fs_info: - raise FileNotFoundError + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) info = _merge_info(repo, subkey, fs_info, dvc_info) info["name"] = path diff --git a/dvc/repo/ls.py b/dvc/repo/ls.py index 0dd09c95f..be9d04d5b 100644 --- a/dvc/repo/ls.py +++ b/dvc/repo/ls.py @@ -83,24 +83,24 @@ def _ls( fs_path = fs.info(fs_path)["name"] infos = {} - for root, dirs, files in fs.walk( - fs_path, dvcfiles=True, dvc_only=dvc_only, detail=True - ): - if not recursive: - files.update(dirs) - - parts = fs.relparts(root, fs_path) - if parts == (".",): - parts = () - - for name, entry in files.items(): - infos[os.path.join(*parts, name)] = entry - - if not recursive: - break - - if not infos and fs.isfile(fs_path): + if fs.isfile(fs_path): infos[os.path.basename(path)] = fs.info(fs_path) + else: + for root, dirs, files in fs.walk( + fs_path, dvcfiles=True, dvc_only=dvc_only, detail=True + ): + if not recursive: + files.update(dirs) + + parts = fs.relparts(root, fs_path) + if parts == (".",): + parts = () + + for name, entry in files.items(): + infos[os.path.join(*parts, name)] = entry + + if not recursive: + break ret = {} for name, info in infos.items(): diff --git a/pyproject.toml b/pyproject.toml index e49974206..58a28af0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "configobj>=5.0.6", "distro>=1.3", "dpath<3,>=2.1.0", - "dvc-data>=3.9,<3.10", + "dvc-data>=3.10,<3.11", "dvc-http>=2.29.0", "dvc-render>=1.0.1,<2", "dvc-studio-client>=0.17.1,<1", @@ -59,7 +59,7 @@ dependencies = [ "requests>=2.22", "rich>=12", "ruamel.yaml>=0.17.11", - "scmrepo>=2.0.2,<3", + "scmrepo>=3,<4", "shortuuid>=0.5", "shtab<2,>=1.3.4", "tabulate>=0.8.7", @@ -71,7 +71,7 @@ dependencies = [ [project.optional-dependencies] all = ["dvc[azure,gdrive,gs,hdfs,oss,s3,ssh,webdav,webhdfs]"] -azure = ["dvc-azure>=3,<4"] +azure = ["dvc-azure>=3.1.0,<4"] dev = ["dvc[azure,gdrive,gs,hdfs,lint,oss,s3,ssh,tests,webdav,webhdfs]"] gdrive = ["dvc-gdrive>=3,<4"] gs = ["dvc-gs>=3,<4"]