fetch: warn if cloud-versioned file has no version ID

This commit is contained in:
Peter Rowlands
2023-07-03 14:05:15 +09:00
committed by Peter Rowlands (변기호)
parent 1f31a9c696
commit 24d9a05d69
2 changed files with 43 additions and 7 deletions
+31
View File
@@ -1,6 +1,8 @@
import logging
from typing import Dict, List, Tuple
from dvc.exceptions import DownloadError
from dvc_data.index import DataIndex, FileStorage
from . import locked
@@ -84,6 +86,7 @@ def fetch( # noqa: C901, PLR0913
data = collect(
indexes, cache_index=self.data_index, cache_key=cache_key, callback=cb
)
failed_count += _log_unversioned(data)
with Callback.as_tqdm_callback(
desc="Fetching",
@@ -109,3 +112,31 @@ def fetch( # noqa: C901, PLR0913
raise DownloadError(failed_count)
return transferred_count
def _log_unversioned(data: Dict[Tuple[str, str], "DataIndex"]) -> int:
unversioned: List[str] = []
for by_fs, fs_index in data.items():
remote = fs_index.storage_map[()].remote
if not isinstance(remote, FileStorage) or not remote.fs.version_aware:
continue
fs = remote.fs
index = DataIndex()
index.storage_map = fs_index.storage_map
for key, entry in fs_index.iteritems():
if entry.meta and not entry.meta.isdir and entry.meta.version_id is None:
unversioned.append(fs.unstrip_protocol(fs.path.join(remote.path, *key)))
else:
index[key] = entry
fs_index.close()
data[by_fs] = index
if unversioned:
logger.warning(
(
"Some files are missing cloud version information and will not be "
"fetched from the remote:\n%s"
),
"\n".join(unversioned),
)
return len(unversioned)
+12 -7
View File
@@ -1,5 +1,6 @@
import logging
from dvc.exceptions import CheckoutError
from dvc.repo import locked
from dvc.utils import glob_targets
@@ -38,13 +39,17 @@ def pull( # noqa: PLR0913
recursive=recursive,
run_cache=run_cache,
)
stats = self.checkout(
targets=expanded_targets,
with_deps=with_deps,
force=force,
recursive=recursive,
allow_missing=allow_missing,
)
try:
stats = self.checkout(
targets=expanded_targets,
with_deps=with_deps,
force=force,
recursive=recursive,
allow_missing=allow_missing,
)
except CheckoutError as exc:
exc.stats["fetched"] = processed_files_count
raise
stats["fetched"] = processed_files_count
return stats