[PyOV] Allow replacing of invalid bytes in string-based data (#21761)

* [PyOV] Allow replacing of invalid bytes in string-based data

* Fix linter
This commit is contained in:
Jan Iwaszkiewicz authored and GitHub committed 2023-12-19 23:25:52 +04:00
1 parent 27c9ca749d
commit 712e374519
2 files changed
+11 -1

No files matched your search

@@ -153,7 +153,7 @@ py::array string_array_from_tensor(ov::Tensor&& t) {
auto data = t.data<std::string>();
py::list _list;
for (size_t i = 0; i < t.get_size(); ++i) {
PyObject* _unicode_obj = PyUnicode_DecodeUTF8(&data[i][0], data[i].length(), "strict");
PyObject* _unicode_obj = PyUnicode_DecodeUTF8(&data[i][0], data[i].length(), "replace");
_list.append(_unicode_obj);
Py_XDECREF(_unicode_obj);
}
@@ -254,3 +254,13 @@ def test_empty_tensor_populate(init_type, init_shape, string_data, data_getter):
check_bytes_based(tensor, _string_data, to_flat=True)
# Decoded:
check_string_based(tensor, _string_data, to_flat=True)
def test_invalid_bytes_replaced():
string_data = np.array(b"\xe2\x80")
tensor = ov.Tensor(string_data)
# Encoded:
check_bytes_based(tensor, string_data, to_flat=True)
# Decoded:
check_string_based(tensor, np.char.decode(string_data, encoding="utf=8", errors="replace"), to_flat=True)