From a6c9f9aeab56742e6988c634916e2ac7ecd41e25 Mon Sep 17 00:00:00 2001 From: Mikhail Nosov Date: Wed, 12 Jan 2022 14:29:03 +0300 Subject: [PATCH] Move 'NV12toRGB/BGR' reference evaluates to template plugin (#9548) * Move 'NV12toRGB/BGR' reference evaluates to template plugin CPU doesn't need this fallback, so implementation can be moved to reduce core binary size * Moved evaluate_nv12 to 'runtime::reference' * Fix arm build --- .../template_plugin/backend/evaluates_map.cpp | 21 +++++ .../template_plugin/backend/opset_int_tbl.hpp | 2 + .../op/util/convert_color_nv12_base.hpp | 6 -- .../runtime/reference/convert_color_nv12.hpp | 46 ++++++++++ .../src/op/util/convert_color_nv12_base.cpp | 83 ------------------- 5 files changed, 69 insertions(+), 89 deletions(-) diff --git a/docs/template_plugin/backend/evaluates_map.cpp b/docs/template_plugin/backend/evaluates_map.cpp index c6a864f9727..0d5fcf27d19 100644 --- a/docs/template_plugin/backend/evaluates_map.cpp +++ b/docs/template_plugin/backend/evaluates_map.cpp @@ -78,6 +78,7 @@ #include "backend.hpp" #include "ngraph/ops.hpp" +#include "ngraph/runtime/reference/convert_color_nv12.hpp" using namespace ngraph; using namespace std; @@ -2761,6 +2762,26 @@ bool evaluate(const shared_ptr& op, const HostTensorVector& outp return true; } +template +inline bool evaluate(const shared_ptr& op, + const HostTensorVector& outputs, + const HostTensorVector& inputs) { + return runtime::reference::color_convert_nv12(op, + outputs, + inputs, + ov::op::util::ConvertColorNV12Base::ColorConversion::NV12_TO_RGB); +} + +template +inline bool evaluate(const shared_ptr& op, + const HostTensorVector& outputs, + const HostTensorVector& inputs) { + return runtime::reference::color_convert_nv12(op, + outputs, + inputs, + ov::op::util::ConvertColorNV12Base::ColorConversion::NV12_TO_BGR); +} + template bool evaluate_node(std::shared_ptr node, const HostTensorVector& outputs, const HostTensorVector& inputs) { auto element_type = node->get_output_element_type(0); diff --git a/docs/template_plugin/backend/opset_int_tbl.hpp b/docs/template_plugin/backend/opset_int_tbl.hpp index 4d4383da233..f7630515d7b 100644 --- a/docs/template_plugin/backend/opset_int_tbl.hpp +++ b/docs/template_plugin/backend/opset_int_tbl.hpp @@ -110,6 +110,8 @@ NGRAPH_OP(DeformableConvolution, ngraph::op::v8) NGRAPH_OP(If, ngraph::op::v8) NGRAPH_OP(GatherND, op::v8) NGRAPH_OP(DetectionOutput, op::v8) +NGRAPH_OP(NV12toRGB, op::v8) +NGRAPH_OP(NV12toBGR, op::v8) NGRAPH_OP(Sigmoid, op::v0) NGRAPH_OP(Tanh, op::v0) diff --git a/src/core/include/openvino/op/util/convert_color_nv12_base.hpp b/src/core/include/openvino/op/util/convert_color_nv12_base.hpp index ee4ed99422b..0afb88ac17e 100644 --- a/src/core/include/openvino/op/util/convert_color_nv12_base.hpp +++ b/src/core/include/openvino/op/util/convert_color_nv12_base.hpp @@ -67,12 +67,6 @@ public: bool visit_attributes(AttributeVisitor& visitor) override; - OPENVINO_SUPPRESS_DEPRECATED_START - bool evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const override; - OPENVINO_SUPPRESS_DEPRECATED_END - - bool has_evaluate() const override; - protected: bool is_type_supported(const ov::element::Type& type) const; diff --git a/src/core/reference/include/ngraph/runtime/reference/convert_color_nv12.hpp b/src/core/reference/include/ngraph/runtime/reference/convert_color_nv12.hpp index a40eb0dbc3c..3f0cffe5947 100644 --- a/src/core/reference/include/ngraph/runtime/reference/convert_color_nv12.hpp +++ b/src/core/reference/include/ngraph/runtime/reference/convert_color_nv12.hpp @@ -108,6 +108,52 @@ void color_convert_i420(const T* arg_y, } } +template +inline bool color_convert_nv12(const std::shared_ptr& op, + const ov::HostTensorVector& outputs, + const ov::HostTensorVector& inputs, + ov::op::util::ConvertColorNV12Base::ColorConversion type) { + static const size_t N_DIM = 0; + static const size_t H_DIM = 1; + static const size_t W_DIM = 2; + NGRAPH_CHECK(op->get_input_size() == 1 || op->get_input_size() == 2, + "NV12 conversion shall have one or 2 inputs, but it is ", + op->get_input_size()); + auto single_plane = op->get_input_size() == 1; + + const auto& y_tensor = inputs[0]; + auto batch_size = y_tensor->get_shape()[N_DIM]; + auto image_w = y_tensor->get_shape()[W_DIM]; + auto image_h = y_tensor->get_shape()[H_DIM]; + if (single_plane) { + image_h = image_h * 2 / 3; + } + outputs[0]->set_shape({batch_size, image_h, image_w, 3}); // 3 is RGB + if (single_plane) { + color_convert_nv12(y_tensor->get_data_ptr(), + y_tensor->get_data_ptr() + image_w * image_h, + outputs[0]->get_data_ptr(), + batch_size, + image_h, + image_w, + image_w * image_h * 3 / 2, + image_w * image_h * 3 / 2, + type); + } else { + const auto& uv_tensor = inputs[1]; + color_convert_nv12(y_tensor->get_data_ptr(), + uv_tensor->get_data_ptr(), + outputs[0]->get_data_ptr(), + batch_size, + image_h, + image_w, + image_w * image_h, + image_w * image_h / 2, + type); + } + return true; +} + } // namespace reference } // namespace runtime } // namespace ngraph diff --git a/src/core/src/op/util/convert_color_nv12_base.cpp b/src/core/src/op/util/convert_color_nv12_base.cpp index 7179e550495..2e751f38a6e 100644 --- a/src/core/src/op/util/convert_color_nv12_base.cpp +++ b/src/core/src/op/util/convert_color_nv12_base.cpp @@ -141,93 +141,10 @@ void ov::op::util::ConvertColorNV12Base::validate_and_infer_types() { set_output_type(0, out_type, out_shape); } -namespace color_convert_nv12_op { -namespace { - -template -inline bool evaluate(const ov::HostTensorVector& input_values, - const ov::HostTensorPtr& output_value, - bool single_tensor, - ov::op::util::ConvertColorNV12Base::ColorConversion color_format) { - using namespace ov::op::util; - const auto& y_tensor = input_values[0]; - auto batch_size = y_tensor->get_shape()[N_DIM]; - auto image_w = y_tensor->get_shape()[W_DIM]; - auto image_h = y_tensor->get_shape()[H_DIM]; - if (single_tensor) { - OPENVINO_ASSERT(ngraph::validate_host_tensor_vector(input_values, 1)); - image_h = image_h * 2 / 3; - } else { - OPENVINO_ASSERT(ngraph::validate_host_tensor_vector(input_values, 2)); - } - output_value->set_shape({batch_size, image_h, image_w, 3}); // 3 is RGB - if (single_tensor) { - ngraph::runtime::reference::color_convert_nv12(y_tensor->get_data_ptr(), - y_tensor->get_data_ptr() + image_w * image_h, - output_value->get_data_ptr(), - batch_size, - image_h, - image_w, - image_w * image_h * 3 / 2, - image_w * image_h * 3 / 2, - color_format); - } else { - const auto& uv_tensor = input_values[1]; - ngraph::runtime::reference::color_convert_nv12(y_tensor->get_data_ptr(), - uv_tensor->get_data_ptr(), - output_value->get_data_ptr(), - batch_size, - image_h, - image_w, - image_w * image_h, - image_w * image_h / 2, - color_format); - } - return true; -} - -bool evaluate_nv12_convert(const ov::HostTensorVector& input_values, - const ov::HostTensorPtr& output_value, - bool single_tensor, - ov::op::util::ConvertColorNV12Base::ColorConversion conv_format) { - bool rc = false; - switch (input_values[0]->get_element_type()) { - NGRAPH_TYPE_CASE(evaluate_nv12_convert, u8, input_values, output_value, single_tensor, conv_format); - NGRAPH_TYPE_CASE(evaluate_nv12_convert, f16, input_values, output_value, single_tensor, conv_format); - NGRAPH_TYPE_CASE(evaluate_nv12_convert, bf16, input_values, output_value, single_tensor, conv_format); - NGRAPH_TYPE_CASE(evaluate_nv12_convert, f32, input_values, output_value, single_tensor, conv_format); - NGRAPH_TYPE_CASE(evaluate_nv12_convert, f64, input_values, output_value, single_tensor, conv_format); - default: - break; - } - return rc; -} - -} // namespace -} // namespace color_convert_nv12_op - bool ov::op::util::ConvertColorNV12Base::visit_attributes(AttributeVisitor& visitor) { return true; } -bool ov::op::util::ConvertColorNV12Base::evaluate(const HostTensorVector& output_values, - const HostTensorVector& input_values) const { - NGRAPH_OP_SCOPE(v0_ConvertColorNV12_evaluate); - OPENVINO_ASSERT(ngraph::validate_host_tensor_vector(output_values, 1)); - NODE_VALIDATION_CHECK(this, - get_input_size() == 1 || get_input_size() == 2, - "NV12 conversion shall have one or 2 inputs, but it is ", - get_input_size()); - auto single_plane = get_input_size() == 1; - return color_convert_nv12_op::evaluate_nv12_convert(input_values, output_values[0], single_plane, m_format); -} - -bool ov::op::util::ConvertColorNV12Base::has_evaluate() const { - NGRAPH_OP_SCOPE(v0_ConvertColorNV12Base_has_evaluate); - - return is_type_supported(get_input_element_type(0)); -} - bool ov::op::util::ConvertColorNV12Base::is_type_supported(const ov::element::Type& type) const { return type.is_dynamic() || type.is_real() || type == ov::element::u8; }