mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 07:18:27 +01:00
data/objects: reduce log levels in dvc-objects and dvc-data
Also introduces a `validate_status` kwarg in `data.transfer`, which is used in dvc to log for missing version info.
This commit is contained in:
committed by
skshetry
parent
82dc67d890
commit
455260b29c
+1
-1
@@ -1,8 +1,8 @@
|
||||
def gc(odb, used, jobs=None, cache_odb=None, shallow=True):
|
||||
from dvc.data.tree import Tree
|
||||
from dvc.objects.errors import ObjectDBPermissionError
|
||||
|
||||
from ._progress import QueryingProgress
|
||||
from .tree import Tree
|
||||
|
||||
if odb.read_only:
|
||||
raise ObjectDBPermissionError("Cannot gc read-only ODB")
|
||||
|
||||
+1
-12
@@ -160,7 +160,6 @@ def compare_status(
|
||||
src: "ObjectDB",
|
||||
dest: "ObjectDB",
|
||||
obj_ids: Iterable["HashInfo"],
|
||||
log_missing: bool = True,
|
||||
check_deleted: bool = True,
|
||||
src_index: Optional["ObjectDBIndexBase"] = None,
|
||||
dest_index: Optional["ObjectDBIndexBase"] = None,
|
||||
@@ -189,19 +188,9 @@ def compare_status(
|
||||
else:
|
||||
src_exists = dest_exists
|
||||
src_missing = set()
|
||||
result = CompareStatusResult(
|
||||
return CompareStatusResult(
|
||||
src_exists & dest_exists,
|
||||
src_missing & dest_missing,
|
||||
src_exists - dest_exists,
|
||||
dest_exists - src_exists,
|
||||
)
|
||||
if log_missing and result.missing:
|
||||
missing_desc = "\n".join(
|
||||
f"name: {hash_info.obj_name}, {hash_info}"
|
||||
for hash_info in result.missing
|
||||
)
|
||||
logger.warning(
|
||||
"Some of the cache files do not exist neither locally "
|
||||
f"nor on remote. Missing cache files:\n{missing_desc}"
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -13,6 +13,7 @@ if TYPE_CHECKING:
|
||||
from dvc.objects.hash_info import HashInfo
|
||||
|
||||
from .db.index import ObjectDBIndexBase
|
||||
from .status import CompareStatusResult
|
||||
from .tree import Tree
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -37,7 +38,7 @@ def _log_exceptions(func):
|
||||
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
|
||||
raise
|
||||
|
||||
logger.exception("failed to transfer '%s'", fs_path)
|
||||
logger.debug("failed to transfer '%s'", fs_path)
|
||||
return 1
|
||||
|
||||
return wrapper
|
||||
@@ -90,7 +91,7 @@ def _do_transfer(
|
||||
"aborting .dir file upload",
|
||||
dir_obj.name,
|
||||
)
|
||||
logger.error(
|
||||
logger.debug(
|
||||
"failed to upload '%s' to '%s'",
|
||||
src.get(dir_obj.hash_info).fs_path,
|
||||
dest.get(dir_obj.hash_info).fs_path,
|
||||
@@ -139,6 +140,7 @@ def transfer(
|
||||
jobs: Optional[int] = None,
|
||||
verify: bool = False,
|
||||
hardlink: bool = False,
|
||||
validate_status: Callable[["CompareStatusResult"], None] = None,
|
||||
**kwargs,
|
||||
) -> int:
|
||||
"""Transfer (copy) the specified objects from one ODB to another.
|
||||
@@ -158,6 +160,10 @@ def transfer(
|
||||
status = compare_status(
|
||||
src, dest, obj_ids, check_deleted=False, jobs=jobs, **kwargs
|
||||
)
|
||||
|
||||
if validate_status:
|
||||
validate_status(status)
|
||||
|
||||
if not status.new:
|
||||
return 0
|
||||
|
||||
|
||||
+1
-1
@@ -153,7 +153,7 @@ class Tree(HashFile):
|
||||
raise ObjectFormatError(f"{obj} is corrupted") from exc
|
||||
|
||||
if not isinstance(raw, list):
|
||||
logger.error(
|
||||
logger.debug(
|
||||
"dir cache file format error '%s' [skipping the file]",
|
||||
obj.fs_path,
|
||||
)
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Iterable, Optional
|
||||
from dvc.data.db import get_index
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.data.status import CompareStatusResult
|
||||
from dvc.objects.db import ObjectDB
|
||||
from dvc.objects.hash_info import HashInfo
|
||||
|
||||
@@ -62,6 +63,17 @@ class DataCloud:
|
||||
config["tmp_dir"] = self.repo.index_db_dir
|
||||
return get_odb(cls(**config), fs_path, **config)
|
||||
|
||||
def _log_missing(self, status: "CompareStatusResult"):
|
||||
if status.missing:
|
||||
missing_desc = "\n".join(
|
||||
f"name: {hash_info.obj_name}, {hash_info}"
|
||||
for hash_info in status.missing
|
||||
)
|
||||
logger.warning(
|
||||
"Some of the cache files do not exist neither locally "
|
||||
f"nor on remote. Missing cache files:\n{missing_desc}"
|
||||
)
|
||||
|
||||
def transfer(
|
||||
self,
|
||||
src_odb: "ObjectDB",
|
||||
@@ -101,6 +113,7 @@ class DataCloud:
|
||||
jobs=jobs,
|
||||
dest_index=get_index(odb),
|
||||
cache_odb=self.repo.odb.local,
|
||||
validate_status=self._log_missing,
|
||||
)
|
||||
|
||||
def pull(
|
||||
@@ -128,6 +141,7 @@ class DataCloud:
|
||||
src_index=get_index(odb),
|
||||
cache_odb=self.repo.odb.local,
|
||||
verify=odb.verify,
|
||||
validate_status=self._log_missing,
|
||||
)
|
||||
|
||||
def status(
|
||||
|
||||
+1
-1
@@ -197,7 +197,7 @@ class ObjectDB:
|
||||
try:
|
||||
obj.check(self, check_hash=check_hash)
|
||||
except ObjectFormatError:
|
||||
logger.warning("corrupted cache file '%s'.", obj.fs_path)
|
||||
logger.debug("corrupted cache file '%s'.", obj.fs_path)
|
||||
with suppress(FileNotFoundError):
|
||||
self.fs.remove(obj.fs_path)
|
||||
raise
|
||||
|
||||
Reference in New Issue
Block a user