data/objects: reduce log levels in dvc-objects and dvc-data

Also introduces a `validate_status` kwarg in `data.transfer`,
which is used in dvc to log for missing version info.
This commit is contained in:
Saugat Pachhai (सौगात)
2022-05-19 18:33:39 +05:45
committed by skshetry
parent 82dc67d890
commit 455260b29c
6 changed files with 26 additions and 17 deletions
+1 -1
View File
@@ -1,8 +1,8 @@
def gc(odb, used, jobs=None, cache_odb=None, shallow=True):
from dvc.data.tree import Tree
from dvc.objects.errors import ObjectDBPermissionError
from ._progress import QueryingProgress
from .tree import Tree
if odb.read_only:
raise ObjectDBPermissionError("Cannot gc read-only ODB")
+1 -12
View File
@@ -160,7 +160,6 @@ def compare_status(
src: "ObjectDB",
dest: "ObjectDB",
obj_ids: Iterable["HashInfo"],
log_missing: bool = True,
check_deleted: bool = True,
src_index: Optional["ObjectDBIndexBase"] = None,
dest_index: Optional["ObjectDBIndexBase"] = None,
@@ -189,19 +188,9 @@ def compare_status(
else:
src_exists = dest_exists
src_missing = set()
result = CompareStatusResult(
return CompareStatusResult(
src_exists & dest_exists,
src_missing & dest_missing,
src_exists - dest_exists,
dest_exists - src_exists,
)
if log_missing and result.missing:
missing_desc = "\n".join(
f"name: {hash_info.obj_name}, {hash_info}"
for hash_info in result.missing
)
logger.warning(
"Some of the cache files do not exist neither locally "
f"nor on remote. Missing cache files:\n{missing_desc}"
)
return result
+8 -2
View File
@@ -13,6 +13,7 @@ if TYPE_CHECKING:
from dvc.objects.hash_info import HashInfo
from .db.index import ObjectDBIndexBase
from .status import CompareStatusResult
from .tree import Tree
logger = logging.getLogger(__name__)
@@ -37,7 +38,7 @@ def _log_exceptions(func):
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
raise
logger.exception("failed to transfer '%s'", fs_path)
logger.debug("failed to transfer '%s'", fs_path)
return 1
return wrapper
@@ -90,7 +91,7 @@ def _do_transfer(
"aborting .dir file upload",
dir_obj.name,
)
logger.error(
logger.debug(
"failed to upload '%s' to '%s'",
src.get(dir_obj.hash_info).fs_path,
dest.get(dir_obj.hash_info).fs_path,
@@ -139,6 +140,7 @@ def transfer(
jobs: Optional[int] = None,
verify: bool = False,
hardlink: bool = False,
validate_status: Callable[["CompareStatusResult"], None] = None,
**kwargs,
) -> int:
"""Transfer (copy) the specified objects from one ODB to another.
@@ -158,6 +160,10 @@ def transfer(
status = compare_status(
src, dest, obj_ids, check_deleted=False, jobs=jobs, **kwargs
)
if validate_status:
validate_status(status)
if not status.new:
return 0
+1 -1
View File
@@ -153,7 +153,7 @@ class Tree(HashFile):
raise ObjectFormatError(f"{obj} is corrupted") from exc
if not isinstance(raw, list):
logger.error(
logger.debug(
"dir cache file format error '%s' [skipping the file]",
obj.fs_path,
)
+14
View File
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Iterable, Optional
from dvc.data.db import get_index
if TYPE_CHECKING:
from dvc.data.status import CompareStatusResult
from dvc.objects.db import ObjectDB
from dvc.objects.hash_info import HashInfo
@@ -62,6 +63,17 @@ class DataCloud:
config["tmp_dir"] = self.repo.index_db_dir
return get_odb(cls(**config), fs_path, **config)
def _log_missing(self, status: "CompareStatusResult"):
if status.missing:
missing_desc = "\n".join(
f"name: {hash_info.obj_name}, {hash_info}"
for hash_info in status.missing
)
logger.warning(
"Some of the cache files do not exist neither locally "
f"nor on remote. Missing cache files:\n{missing_desc}"
)
def transfer(
self,
src_odb: "ObjectDB",
@@ -101,6 +113,7 @@ class DataCloud:
jobs=jobs,
dest_index=get_index(odb),
cache_odb=self.repo.odb.local,
validate_status=self._log_missing,
)
def pull(
@@ -128,6 +141,7 @@ class DataCloud:
src_index=get_index(odb),
cache_odb=self.repo.odb.local,
verify=odb.verify,
validate_status=self._log_missing,
)
def status(
+1 -1
View File
@@ -197,7 +197,7 @@ class ObjectDB:
try:
obj.check(self, check_hash=check_hash)
except ObjectFormatError:
logger.warning("corrupted cache file '%s'.", obj.fs_path)
logger.debug("corrupted cache file '%s'.", obj.fs_path)
with suppress(FileNotFoundError):
self.fs.remove(obj.fs_path)
raise