serialize: Wrap UnicodeDecodeErrors (#6193)

Also expose `encoding` and `path` as attributes, so programmatic
consumers can access it easily.
This commit is contained in:
Laurens Duijvesteijn
2021-06-19 11:35:50 +05:45
committed by GitHub
parent 1d3524f144
commit 9a7b6d2a1a
2 changed files with 26 additions and 3 deletions
+14 -2
View File
@@ -3,6 +3,7 @@ import os
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any, Callable, ContextManager, Dict, Union
from funcy import reraise
from typing_extensions import Protocol
from dvc.exceptions import DvcException
@@ -42,13 +43,24 @@ class ParseError(DvcException):
from dvc.utils import relpath
path = relpath(path)
self.path = path
super().__init__(f"unable to read: '{path}', {message}")
class EncodingError(ParseError):
"""We could not read a file with the given encoding"""
def __init__(self, path: "AnyPath", encoding: str):
self.encoding = encoding
super().__init__(path, f"is not valid {encoding}")
def _load_data(path: "AnyPath", parser: ParserFn, fs: "BaseFileSystem" = None):
open_fn = fs.open if fs else open
with open_fn(path, encoding="utf-8") as fd: # type: ignore
return parser(fd.read(), path)
encoding = "utf-8"
with open_fn(path, encoding=encoding) as fd: # type: ignore
with reraise(UnicodeDecodeError, EncodingError(path, encoding)):
return parser(fd.read(), path)
def _dump_data(path, data: Any, dumper: DumperFn, fs: "BaseFileSystem" = None):
+12 -1
View File
@@ -1,6 +1,6 @@
import pytest
from dvc.utils.serialize._yaml import YAMLError, parse_yaml
from dvc.utils.serialize import EncodingError, YAMLError, load_yaml, parse_yaml
def test_parse_yaml_duplicate_key_error():
@@ -12,3 +12,14 @@ def test_parse_yaml_duplicate_key_error():
"""
with pytest.raises(YAMLError, match='found duplicate key "mykey"'):
parse_yaml(text, "mypath")
def test_parse_yaml_invalid_unicode(tmp_dir):
filename = "invalid_utf8.yaml"
tmp_dir.gen(filename, b"\x80some: stuff")
with pytest.raises(EncodingError) as excinfo:
load_yaml(tmp_dir / filename)
assert filename in excinfo.value.path
assert excinfo.value.encoding == "utf-8"