From 455260b29c699d2930b5d316a4ff063973a2299e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?= =?UTF-8?q?=E0=A4=A4=29?= Date: Thu, 19 May 2022 17:16:56 +0545 Subject: [PATCH] data/objects: reduce log levels in dvc-objects and dvc-data Also introduces a `validate_status` kwarg in `data.transfer`, which is used in dvc to log for missing version info. --- dvc/data/gc.py | 2 +- dvc/data/status.py | 13 +------------ dvc/data/transfer.py | 10 ++++++++-- dvc/data/tree.py | 2 +- dvc/data_cloud.py | 14 ++++++++++++++ dvc/objects/db.py | 2 +- 6 files changed, 26 insertions(+), 17 deletions(-) diff --git a/dvc/data/gc.py b/dvc/data/gc.py index 9d754d38f..7ff320143 100644 --- a/dvc/data/gc.py +++ b/dvc/data/gc.py @@ -1,8 +1,8 @@ def gc(odb, used, jobs=None, cache_odb=None, shallow=True): - from dvc.data.tree import Tree from dvc.objects.errors import ObjectDBPermissionError from ._progress import QueryingProgress + from .tree import Tree if odb.read_only: raise ObjectDBPermissionError("Cannot gc read-only ODB") diff --git a/dvc/data/status.py b/dvc/data/status.py index 3d942f17f..5160dfa3c 100644 --- a/dvc/data/status.py +++ b/dvc/data/status.py @@ -160,7 +160,6 @@ def compare_status( src: "ObjectDB", dest: "ObjectDB", obj_ids: Iterable["HashInfo"], - log_missing: bool = True, check_deleted: bool = True, src_index: Optional["ObjectDBIndexBase"] = None, dest_index: Optional["ObjectDBIndexBase"] = None, @@ -189,19 +188,9 @@ def compare_status( else: src_exists = dest_exists src_missing = set() - result = CompareStatusResult( + return CompareStatusResult( src_exists & dest_exists, src_missing & dest_missing, src_exists - dest_exists, dest_exists - src_exists, ) - if log_missing and result.missing: - missing_desc = "\n".join( - f"name: {hash_info.obj_name}, {hash_info}" - for hash_info in result.missing - ) - logger.warning( - "Some of the cache files do not exist neither locally " - f"nor on remote. Missing cache files:\n{missing_desc}" - ) - return result diff --git a/dvc/data/transfer.py b/dvc/data/transfer.py index 9e2f970c4..16b191e34 100644 --- a/dvc/data/transfer.py +++ b/dvc/data/transfer.py @@ -13,6 +13,7 @@ if TYPE_CHECKING: from dvc.objects.hash_info import HashInfo from .db.index import ObjectDBIndexBase + from .status import CompareStatusResult from .tree import Tree logger = logging.getLogger(__name__) @@ -37,7 +38,7 @@ def _log_exceptions(func): if isinstance(exc, OSError) and exc.errno == errno.EMFILE: raise - logger.exception("failed to transfer '%s'", fs_path) + logger.debug("failed to transfer '%s'", fs_path) return 1 return wrapper @@ -90,7 +91,7 @@ def _do_transfer( "aborting .dir file upload", dir_obj.name, ) - logger.error( + logger.debug( "failed to upload '%s' to '%s'", src.get(dir_obj.hash_info).fs_path, dest.get(dir_obj.hash_info).fs_path, @@ -139,6 +140,7 @@ def transfer( jobs: Optional[int] = None, verify: bool = False, hardlink: bool = False, + validate_status: Callable[["CompareStatusResult"], None] = None, **kwargs, ) -> int: """Transfer (copy) the specified objects from one ODB to another. @@ -158,6 +160,10 @@ def transfer( status = compare_status( src, dest, obj_ids, check_deleted=False, jobs=jobs, **kwargs ) + + if validate_status: + validate_status(status) + if not status.new: return 0 diff --git a/dvc/data/tree.py b/dvc/data/tree.py index 1a6083040..3f06f560a 100644 --- a/dvc/data/tree.py +++ b/dvc/data/tree.py @@ -153,7 +153,7 @@ class Tree(HashFile): raise ObjectFormatError(f"{obj} is corrupted") from exc if not isinstance(raw, list): - logger.error( + logger.debug( "dir cache file format error '%s' [skipping the file]", obj.fs_path, ) diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 7f2942ce6..9bf8dddee 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Iterable, Optional from dvc.data.db import get_index if TYPE_CHECKING: + from dvc.data.status import CompareStatusResult from dvc.objects.db import ObjectDB from dvc.objects.hash_info import HashInfo @@ -62,6 +63,17 @@ class DataCloud: config["tmp_dir"] = self.repo.index_db_dir return get_odb(cls(**config), fs_path, **config) + def _log_missing(self, status: "CompareStatusResult"): + if status.missing: + missing_desc = "\n".join( + f"name: {hash_info.obj_name}, {hash_info}" + for hash_info in status.missing + ) + logger.warning( + "Some of the cache files do not exist neither locally " + f"nor on remote. Missing cache files:\n{missing_desc}" + ) + def transfer( self, src_odb: "ObjectDB", @@ -101,6 +113,7 @@ class DataCloud: jobs=jobs, dest_index=get_index(odb), cache_odb=self.repo.odb.local, + validate_status=self._log_missing, ) def pull( @@ -128,6 +141,7 @@ class DataCloud: src_index=get_index(odb), cache_odb=self.repo.odb.local, verify=odb.verify, + validate_status=self._log_missing, ) def status( diff --git a/dvc/objects/db.py b/dvc/objects/db.py index 38d9771ad..135180cc3 100644 --- a/dvc/objects/db.py +++ b/dvc/objects/db.py @@ -197,7 +197,7 @@ class ObjectDB: try: obj.check(self, check_hash=check_hash) except ObjectFormatError: - logger.warning("corrupted cache file '%s'.", obj.fs_path) + logger.debug("corrupted cache file '%s'.", obj.fs_path) with suppress(FileNotFoundError): self.fs.remove(obj.fs_path) raise