From 24d9a05d69f359b8032660085df32515e9ee0459 Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Wed, 28 Jun 2023 16:21:27 +0900 Subject: [PATCH] fetch: warn if cloud-versioned file has no version ID --- dvc/repo/fetch.py | 31 +++++++++++++++++++++++++++++++ dvc/repo/pull.py | 19 ++++++++++++------- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index c29a088b2..7a061c5cc 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -1,6 +1,8 @@ import logging +from typing import Dict, List, Tuple from dvc.exceptions import DownloadError +from dvc_data.index import DataIndex, FileStorage from . import locked @@ -84,6 +86,7 @@ def fetch( # noqa: C901, PLR0913 data = collect( indexes, cache_index=self.data_index, cache_key=cache_key, callback=cb ) + failed_count += _log_unversioned(data) with Callback.as_tqdm_callback( desc="Fetching", @@ -109,3 +112,31 @@ def fetch( # noqa: C901, PLR0913 raise DownloadError(failed_count) return transferred_count + + +def _log_unversioned(data: Dict[Tuple[str, str], "DataIndex"]) -> int: + unversioned: List[str] = [] + for by_fs, fs_index in data.items(): + remote = fs_index.storage_map[()].remote + if not isinstance(remote, FileStorage) or not remote.fs.version_aware: + continue + + fs = remote.fs + index = DataIndex() + index.storage_map = fs_index.storage_map + for key, entry in fs_index.iteritems(): + if entry.meta and not entry.meta.isdir and entry.meta.version_id is None: + unversioned.append(fs.unstrip_protocol(fs.path.join(remote.path, *key))) + else: + index[key] = entry + fs_index.close() + data[by_fs] = index + if unversioned: + logger.warning( + ( + "Some files are missing cloud version information and will not be " + "fetched from the remote:\n%s" + ), + "\n".join(unversioned), + ) + return len(unversioned) diff --git a/dvc/repo/pull.py b/dvc/repo/pull.py index dbb75261b..66c628225 100644 --- a/dvc/repo/pull.py +++ b/dvc/repo/pull.py @@ -1,5 +1,6 @@ import logging +from dvc.exceptions import CheckoutError from dvc.repo import locked from dvc.utils import glob_targets @@ -38,13 +39,17 @@ def pull( # noqa: PLR0913 recursive=recursive, run_cache=run_cache, ) - stats = self.checkout( - targets=expanded_targets, - with_deps=with_deps, - force=force, - recursive=recursive, - allow_missing=allow_missing, - ) + try: + stats = self.checkout( + targets=expanded_targets, + with_deps=with_deps, + force=force, + recursive=recursive, + allow_missing=allow_missing, + ) + except CheckoutError as exc: + exc.stats["fetched"] = processed_files_count + raise stats["fetched"] = processed_files_count return stats