[PyOV] Allow replacing of invalid bytes in string-based data (#21761)
* [PyOV] Allow replacing of invalid bytes in string-based data * Fix linter
This commit is contained in:
1 parent
27c9ca749d
commit
712e374519
2 files changed
+11
-1
No files matched your search
@@ -153,7 +153,7 @@ py::array string_array_from_tensor(ov::Tensor&& t) {
|
||||
auto data = t.data<std::string>();
|
||||
py::list _list;
|
||||
for (size_t i = 0; i < t.get_size(); ++i) {
|
||||
PyObject* _unicode_obj = PyUnicode_DecodeUTF8(&data[i][0], data[i].length(), "strict");
|
||||
PyObject* _unicode_obj = PyUnicode_DecodeUTF8(&data[i][0], data[i].length(), "replace");
|
||||
_list.append(_unicode_obj);
|
||||
Py_XDECREF(_unicode_obj);
|
||||
}
|
||||
|
||||
@@ -254,3 +254,13 @@ def test_empty_tensor_populate(init_type, init_shape, string_data, data_getter):
|
||||
check_bytes_based(tensor, _string_data, to_flat=True)
|
||||
# Decoded:
|
||||
check_string_based(tensor, _string_data, to_flat=True)
|
||||
|
||||
|
||||
def test_invalid_bytes_replaced():
|
||||
string_data = np.array(b"\xe2\x80")
|
||||
tensor = ov.Tensor(string_data)
|
||||
|
||||
# Encoded:
|
||||
check_bytes_based(tensor, string_data, to_flat=True)
|
||||
# Decoded:
|
||||
check_string_based(tensor, np.char.decode(string_data, encoding="utf=8", errors="replace"), to_flat=True)
|
||||
Reference in new issue
Block a user