[OV20] Convert NV12 to RGB operation + preprocessing (#7508)
* # Conflicts: # docs/template_plugin/tests/functional/op_reference/convert_color_nv12.cpp # inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/convert_color_nv12.cpp # inference-engine/tests/functional/shared_test_classes/include/shared_test_classes/single_layer/convert_color_nv12.hpp # inference-engine/tests/functional/shared_test_classes/src/single_layer/convert_color_nv12.cpp # ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp # ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp # ngraph/core/include/openvino/op/nv12_to_bgr.hpp # ngraph/core/include/openvino/op/nv12_to_rgb.hpp # ngraph/core/src/op/nv12_to_bgr.cpp # ngraph/core/src/op/nv12_to_rgb.cpp # ngraph/core/src/preprocess/pre_post_process.cpp # ngraph/core/src/preprocess/preprocess_steps_impl.hpp # ngraph/test/CMakeLists.txt * Added more test to cover 100% of code Allow convert element type for 'multi-plane' color format * Inherit tensor names for 'convert_color' * Clang * Fix tests * Disable 'int8' preprocessing resize test * Fix review comments * Add more restrictions and tests for planes sub-names * 1) Added check for uniqueness of tensor names generated for nodes Raise error if user's plane sub-name conflicts with some node in a function 2) Added exception safety to preprocess build. Before, when input #2 fail, only one preprocess will be applied to function and it will be corrupted Exception guard will restore function to original state if exception occurs * Fix clang-format
This commit is contained in:
Mikhail Nosov
GitHub
parent
659daf610f
commit
f57dc05c66
@ -24,6 +24,8 @@ struct RefPreprocessParams {
|
||||
std::function<std::shared_ptr<ov::Function>()> function;
|
||||
std::vector<Tensor> inputs;
|
||||
std::vector<Tensor> expected;
|
||||
float abs_threshold = 0.01f;
|
||||
float rel_threshold = 0.01f;
|
||||
std::string name;
|
||||
};
|
||||
|
||||
@ -39,6 +41,8 @@ public:
|
||||
for (const auto& exp : params.expected) {
|
||||
refOutData.push_back(exp.data);
|
||||
}
|
||||
abs_threshold = params.abs_threshold;
|
||||
threshold = params.rel_threshold;
|
||||
}
|
||||
static std::string getTestCaseName(const testing::TestParamInfo<RefPreprocessParams>& obj) {
|
||||
const auto& param = obj.param;
|
||||
@ -327,6 +331,26 @@ static RefPreprocessParams resize_from_spatial_dims() {
|
||||
return res;
|
||||
}
|
||||
|
||||
static RefPreprocessParams resize_i8() {
|
||||
RefPreprocessParams res("resize_i8");
|
||||
res.function = []() {
|
||||
auto f = create_simple_function(element::i8, PartialShape{1, 3, 1, 1});
|
||||
f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo()
|
||||
.set_spatial_dynamic_shape())
|
||||
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_LINEAR))
|
||||
.network(InputNetworkInfo().set_layout("NCHW")))
|
||||
.build(f);
|
||||
return f;
|
||||
};
|
||||
res.inputs.emplace_back(element::i8, Shape{1, 3, 2, 2}, std::vector<int8_t>{0, 0, 0, 0,
|
||||
1, 1, 1, 1,
|
||||
2, 2, 2, 2});
|
||||
res.expected.emplace_back(Shape{1, 3, 1, 1}, element::i8, std::vector<int8_t>{0, 1, 2});
|
||||
return res;
|
||||
}
|
||||
|
||||
static RefPreprocessParams resize_to_network_width_height() {
|
||||
RefPreprocessParams res("resize_to_network_width_height");
|
||||
res.function = []() {
|
||||
@ -505,6 +529,152 @@ static RefPreprocessParams resize_and_convert_layout() {
|
||||
return res;
|
||||
}
|
||||
|
||||
static RefPreprocessParams convert_color_nv12_to_bgr_two_planes() {
|
||||
RefPreprocessParams res("convert_color_nv12_to_bgr_two_planes");
|
||||
res.abs_threshold = 2.f; // Allow small color conversion deviations
|
||||
res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%)
|
||||
res.function = []() {
|
||||
auto f = create_simple_function(element::u8, PartialShape{1, 4, 4, 3});
|
||||
f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo()
|
||||
.set_color_format(ColorFormat::NV12_TWO_PLANES))
|
||||
.preprocess(PreProcessSteps()
|
||||
.convert_color(ColorFormat::BGR)))
|
||||
.build(f);
|
||||
return f;
|
||||
};
|
||||
|
||||
// clang-format off
|
||||
auto input_y = std::vector<uint8_t> {81, 81, 145, 145, // RRGG
|
||||
81, 81, 145, 145, // RRGG
|
||||
41, 41, 81, 81, // BBRR
|
||||
41, 41, 81, 81}; // BBRR
|
||||
auto input_shape_y = Shape{1, 4, 4, 1};
|
||||
auto input_uv = std::vector<uint8_t> {240, 90, // R (2x2)
|
||||
34, 54, // G (2x2)
|
||||
110, 240, // B (2x2)
|
||||
240, 90}; // R (2x2)
|
||||
auto input_shape_uv = Shape{1, 2, 2, 2};
|
||||
auto exp_out = std::vector<uint8_t> {0, 0, 255, 0, 0, 255, 0, 255, 0, 0, 255, 0,
|
||||
0, 0, 255, 0, 0, 255, 0, 255, 0, 0, 255, 0,
|
||||
255, 0, 0, 255, 0, 0, 0, 0, 255, 0, 0, 255,
|
||||
255, 0, 0, 255, 0, 0, 0, 0, 255, 0, 0, 255};
|
||||
auto out_shape = Shape{1, 4, 4, 3};
|
||||
// clang-format on
|
||||
res.inputs.emplace_back(element::u8, input_shape_y, input_y);
|
||||
res.inputs.emplace_back(element::u8, input_shape_uv, input_uv);
|
||||
res.expected.emplace_back(out_shape, element::u8, exp_out);
|
||||
return res;
|
||||
}
|
||||
|
||||
static RefPreprocessParams convert_color_nv12_single_plane() {
|
||||
RefPreprocessParams res("convert_color_nv12_single_plane");
|
||||
res.abs_threshold = 2.f; // Allow small color conversion deviations
|
||||
res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%)
|
||||
res.function = []() {
|
||||
auto f = create_simple_function(element::f32, PartialShape{1, 4, 4, 3});
|
||||
f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo()
|
||||
.set_color_format(ColorFormat::NV12_SINGLE_PLANE))
|
||||
.preprocess(PreProcessSteps()
|
||||
.convert_color(ColorFormat::RGB)))
|
||||
.build(f);
|
||||
return f;
|
||||
};
|
||||
|
||||
// clang-format off
|
||||
auto input = std::vector<float> { 81, 81, 145, 145, // RRGG
|
||||
81, 81, 145, 145, // RRGG
|
||||
41, 41, 81, 81, // BBRR
|
||||
41, 41, 81, 81, // BBRR
|
||||
240, 90, 34, 54, 110, 240, 240, 90}; // UV (RGBR)
|
||||
auto input_shape = Shape{1, 6, 4, 1};
|
||||
auto exp_out = std::vector<float> {255, 0, 0, 255, 0, 0, 0, 255, 0, 0, 255, 0, // RRGG
|
||||
255, 0, 0, 255, 0, 0, 0, 255, 0, 0, 255, 0, // RRGG
|
||||
0, 0, 255, 0, 0, 255, 255, 0, 0, 255, 0, 0, // BBRR
|
||||
0, 0, 255, 0, 0, 255, 255, 0, 0, 255, 0, 0, // BBRR
|
||||
};
|
||||
auto out_shape = Shape{1, 4, 4, 3};
|
||||
// clang-format on
|
||||
res.inputs.emplace_back(element::f32, input_shape, input);
|
||||
res.expected.emplace_back(out_shape, element::f32, exp_out);
|
||||
return res;
|
||||
}
|
||||
|
||||
static RefPreprocessParams convert_color_nv12_layout_resize() {
|
||||
RefPreprocessParams res("convert_color_nv12_layout_resize");
|
||||
res.abs_threshold = 2.f; // Allow small color conversion deviations
|
||||
res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%)
|
||||
res.function = []() {
|
||||
auto f = create_simple_function(element::f32, PartialShape{1, 3, 2, 2});
|
||||
f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo()
|
||||
.set_color_format(ColorFormat::NV12_SINGLE_PLANE)
|
||||
.set_element_type(element::u8)
|
||||
.set_spatial_dynamic_shape())
|
||||
.preprocess(PreProcessSteps()
|
||||
.convert_color(ColorFormat::RGB)
|
||||
.convert_layout()
|
||||
.convert_element_type(element::f32)
|
||||
.resize(ResizeAlgorithm::RESIZE_NEAREST))
|
||||
.network(InputNetworkInfo().set_layout("NCHW")))
|
||||
.build(f);
|
||||
return f;
|
||||
};
|
||||
|
||||
auto result = std::make_shared<HostTensor>();
|
||||
// clang-format off
|
||||
auto input = std::vector<uint8_t> {81, 81, 145, 145, // RRGG
|
||||
81, 81, 145, 145, // RRGG
|
||||
41, 41, 81, 81, // BBRR
|
||||
41, 41, 81, 81, // BBRR
|
||||
240, 90, 34, 54, 110, 240, 240, 90}; // UV (RGBR)
|
||||
auto input_shape = Shape{1, 6, 4, 1};
|
||||
auto exp_out = std::vector<float> {255, 0, 0, 255, // R channel
|
||||
0, 255, 0, 0, // G channel
|
||||
0, 0, 255, 0}; // B channel
|
||||
auto out_shape = Shape{1, 2, 2, 3};
|
||||
// clang-format on
|
||||
res.inputs.emplace_back(element::u8, input_shape, input);
|
||||
res.expected.emplace_back(out_shape, element::f32, exp_out);
|
||||
return res;
|
||||
}
|
||||
|
||||
static RefPreprocessParams element_type_before_convert_color_nv12() {
|
||||
RefPreprocessParams res("element_type_before_convert_color_nv12");
|
||||
res.abs_threshold = 2.f; // Allow small color conversion deviations
|
||||
res.rel_threshold = 1.f; // Ignore relative pixel values comparison (100%)
|
||||
res.function = []() {
|
||||
auto f = create_simple_function(element::f32, PartialShape{1, 2, 2, 3});
|
||||
f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo()
|
||||
.set_element_type(element::u8)
|
||||
.set_color_format(ColorFormat::NV12_TWO_PLANES))
|
||||
.preprocess(PreProcessSteps()
|
||||
.convert_element_type(element::f32)
|
||||
.convert_color(ColorFormat::RGB))
|
||||
.network(InputNetworkInfo().set_layout("NHWC")))
|
||||
.build(f);
|
||||
return f;
|
||||
};
|
||||
|
||||
// clang-format off
|
||||
auto input_y = std::vector<uint8_t> {81, 81, 81, 81};
|
||||
auto input_shape_y = Shape{1, 2, 2, 1};
|
||||
auto input_uv = std::vector<uint8_t> {240, 90};
|
||||
auto input_shape_uv = Shape{1, 1, 1, 2};
|
||||
auto exp_out = std::vector<float> {255, 0, 0, 255, 0, 0, 255, 0, 0, 255, 0, 0};
|
||||
auto out_shape = Shape{1, 2, 2, 3};
|
||||
// clang-format on
|
||||
res.inputs.emplace_back(element::u8, input_shape_y, input_y);
|
||||
res.inputs.emplace_back(element::u8, input_shape_uv, input_uv);
|
||||
res.expected.emplace_back(out_shape, element::f32, exp_out);
|
||||
return res;
|
||||
}
|
||||
|
||||
std::vector<RefPreprocessParams> allPreprocessTests() {
|
||||
return std::vector<RefPreprocessParams> {
|
||||
@ -521,12 +691,17 @@ std::vector<RefPreprocessParams> allPreprocessTests() {
|
||||
resize_to_network_height(),
|
||||
resize_to_network_width(),
|
||||
resize_from_spatial_dims(),
|
||||
resize_i8(),
|
||||
resize_to_network_width_height(),
|
||||
resize_to_specified_width_height(),
|
||||
resize_lvalues(),
|
||||
convert_layout_nhwc_to_nchw_lvalue(),
|
||||
convert_layout_nhwc_to_net_no_tensor_shape(),
|
||||
resize_and_convert_layout()
|
||||
resize_and_convert_layout(),
|
||||
convert_color_nv12_to_bgr_two_planes(),
|
||||
convert_color_nv12_single_plane(),
|
||||
convert_color_nv12_layout_resize(),
|
||||
element_type_before_convert_color_nv12(),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -111,6 +111,8 @@ std::vector<std::string> disabledTestPatterns() {
|
||||
|
||||
// Issue 66685
|
||||
R"(smoke_PrePostProcess.*resize_linear_nhwc.*)",
|
||||
// Issue 67214
|
||||
R"(smoke_PrePostProcess.*resize_and_convert_layout_i8.*)",
|
||||
};
|
||||
|
||||
#define FIX_62820 0
|
||||
|
||||
@ -30,6 +30,7 @@ void PrePostProcessTest::SetUp() {
|
||||
std::tie(func, targetDevice) = GetParam();
|
||||
function = (std::get<0>(func))();
|
||||
threshold = std::get<2>(func);
|
||||
abs_threshold = std::get<2>(func);
|
||||
}
|
||||
|
||||
TEST_P(PrePostProcessTest, CompareWithRefs) {
|
||||
|
||||
@ -270,6 +270,63 @@ inline std::shared_ptr<Function> resize_and_convert_layout() {
|
||||
return function;
|
||||
}
|
||||
|
||||
inline std::shared_ptr<Function> resize_and_convert_layout_i8() {
|
||||
using namespace ov::preprocess;
|
||||
auto function = create_preprocess_1input(element::i8, PartialShape{1, 30, 20, 3});
|
||||
function = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo()
|
||||
.set_layout("NHWC")
|
||||
.set_spatial_static_shape(40, 30))
|
||||
.preprocess(PreProcessSteps()
|
||||
.convert_layout()
|
||||
.resize(ResizeAlgorithm::RESIZE_LINEAR))
|
||||
.network(InputNetworkInfo().set_layout("NCHW")))
|
||||
.build(function);
|
||||
return function;
|
||||
}
|
||||
|
||||
inline std::shared_ptr<Function> cvt_color_nv12_to_rgb_single_plane() {
|
||||
using namespace ov::preprocess;
|
||||
auto function = create_preprocess_1input(element::f32, PartialShape{1, 20, 20, 3});
|
||||
function = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_SINGLE_PLANE))
|
||||
.preprocess(PreProcessSteps().convert_color(ColorFormat::RGB)))
|
||||
.build(function);
|
||||
return function;
|
||||
}
|
||||
|
||||
inline std::shared_ptr<Function> cvt_color_nv12_to_bgr_two_planes() {
|
||||
using namespace ov::preprocess;
|
||||
auto function = create_preprocess_1input(element::f32, PartialShape{1, 20, 20, 3});
|
||||
function = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES))
|
||||
.preprocess(PreProcessSteps().convert_color(ColorFormat::BGR)))
|
||||
.build(function);
|
||||
return function;
|
||||
}
|
||||
|
||||
inline std::shared_ptr<Function> cvt_color_nv12_cvt_layout_resize() {
|
||||
using namespace ov::preprocess;
|
||||
auto function = create_preprocess_1input(element::f32, PartialShape{1, 3, 10, 10});
|
||||
function = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo()
|
||||
.set_color_format(ColorFormat::NV12_TWO_PLANES)
|
||||
.set_element_type(element::u8)
|
||||
.set_spatial_static_shape(20, 20))
|
||||
.preprocess(PreProcessSteps()
|
||||
.convert_color(ColorFormat::RGB)
|
||||
.convert_layout()
|
||||
.convert_element_type(element::f32)
|
||||
.resize(ResizeAlgorithm::RESIZE_LINEAR))
|
||||
.network(InputNetworkInfo().set_layout("NCHW")))
|
||||
.build(function);
|
||||
return function;
|
||||
}
|
||||
|
||||
inline std::vector<preprocess_func> generic_preprocess_functions() {
|
||||
return std::vector<preprocess_func> {
|
||||
preprocess_func(mean_only, "mean_only", 0.01f),
|
||||
@ -290,6 +347,10 @@ inline std::vector<preprocess_func> generic_preprocess_functions() {
|
||||
preprocess_func(resize_linear_nhwc, "resize_linear_nhwc", 0.01f),
|
||||
preprocess_func(resize_cubic, "resize_cubic", 0.01f),
|
||||
preprocess_func(resize_and_convert_layout, "resize_and_convert_layout", 0.01f),
|
||||
preprocess_func(resize_and_convert_layout_i8, "resize_and_convert_layout_i8", 0.01f),
|
||||
preprocess_func(cvt_color_nv12_to_rgb_single_plane, "cvt_color_nv12_to_rgb_single_plane", 2.f),
|
||||
preprocess_func(cvt_color_nv12_to_bgr_two_planes, "cvt_color_nv12_to_bgr_two_planes", 2.f),
|
||||
preprocess_func(cvt_color_nv12_cvt_layout_resize, "cvt_color_nv12_cvt_layout_resize", 2.f),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,20 @@
|
||||
// Copyright (C) 2018-2021 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace ov {
|
||||
namespace preprocess {
|
||||
|
||||
/// \brief Color format enumeration for conversion
|
||||
enum class ColorFormat {
|
||||
UNDEFINED,
|
||||
NV12_SINGLE_PLANE, // Image in NV12 format as single tensor
|
||||
NV12_TWO_PLANES, // Image in NV12 format represented as separate tensors for Y and UV planes
|
||||
RGB,
|
||||
BGR
|
||||
};
|
||||
|
||||
} // namespace preprocess
|
||||
} // namespace ov
|
||||
@ -6,6 +6,7 @@
|
||||
|
||||
#include "openvino/core/core_visibility.hpp"
|
||||
#include "openvino/core/layout.hpp"
|
||||
#include "openvino/core/preprocess/color_format.hpp"
|
||||
#include "openvino/core/type/element_type.hpp"
|
||||
|
||||
namespace ov {
|
||||
@ -117,6 +118,44 @@ public:
|
||||
///
|
||||
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner.
|
||||
InputTensorInfo&& set_spatial_static_shape(size_t height, size_t width) &&;
|
||||
|
||||
/// \brief Set color format for user's input tensor.
|
||||
///
|
||||
/// In general way, some formats support multi-plane input, e.g. NV12 image can be represented as 2 separate tensors
|
||||
/// (planes): Y plane and UV plane. set_color_format API also allows to set sub_names for such parameters for
|
||||
/// convenient usage of plane parameters.
|
||||
///
|
||||
/// This version allows chaining for Lvalue objects.
|
||||
///
|
||||
/// \param format Color format of input image.
|
||||
///
|
||||
/// \param sub_names Optional list of sub-names assigned for each plane (e.g. {"Y", "UV"}). If not specified,
|
||||
/// sub-names for plane parameters are auto-generated, exact names auto-generation rules depend on specific color
|
||||
/// format, and client's code shall not rely on these rules. It is not allowed to specify sub-names for single-plane
|
||||
/// inputs, also is specified, number of sub-names shall match with number of planes.
|
||||
///
|
||||
/// \return Reference to 'this' to allow chaining with other calls in a builder-like manner.
|
||||
InputTensorInfo& set_color_format(const ov::preprocess::ColorFormat& format,
|
||||
const std::vector<std::string>& sub_names = {}) &;
|
||||
|
||||
/// \brief Set color format for user's input tensor.
|
||||
///
|
||||
/// In general way, some formats support multi-plane input, e.g. NV12 image can be represented as 2 separate tensors
|
||||
/// (planes): Y plane and UV plane. set_color_format API also allows to set sub_names for such parameters for
|
||||
/// convenient usage of plane parameters.
|
||||
///
|
||||
/// This version allows chaining for Rvalue objects.
|
||||
///
|
||||
/// \param format Color format of input image.
|
||||
///
|
||||
/// \param sub_names Optional list of sub-names assigned for each plane (e.g. {"Y", "UV"}). If not specified,
|
||||
/// sub-names for plane parameters are auto-generated, exact names auto-generation rules depend on specific color
|
||||
/// format, and client's code shall not rely on these rules. It is not allowed to specify sub-names for single-plane
|
||||
/// inputs, also is specified, number of sub-names shall match with number of planes.
|
||||
///
|
||||
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner.
|
||||
InputTensorInfo&& set_color_format(const ov::preprocess::ColorFormat& format,
|
||||
const std::vector<std::string>& sub_names = {}) &&;
|
||||
};
|
||||
|
||||
} // namespace preprocess
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#pragma once
|
||||
|
||||
#include "openvino/core/core_visibility.hpp"
|
||||
#include "openvino/core/preprocess/color_format.hpp"
|
||||
#include "openvino/core/preprocess/resize_algorithm.hpp"
|
||||
#include "openvino/core/type/element_type.hpp"
|
||||
|
||||
@ -56,6 +57,26 @@ public:
|
||||
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner
|
||||
PreProcessSteps&& convert_element_type(const ov::element::Type& type) &&;
|
||||
|
||||
/// \brief Converts color format for user's input tensor. Requires source color format to be specified by
|
||||
/// InputTensorInfo::set_color_format.
|
||||
///
|
||||
/// This version allows chaining for Lvalue objects
|
||||
///
|
||||
/// \param dst_format Destination color format of input image
|
||||
///
|
||||
/// \return Reference to 'this' to allow chaining with other calls in a builder-like manner
|
||||
PreProcessSteps& convert_color(const ov::preprocess::ColorFormat& dst_format) &;
|
||||
|
||||
/// \brief Converts color format for user's input tensor. Requires source color format to be specified by
|
||||
/// InputTensorInfo::set_color_format.
|
||||
///
|
||||
/// This version allows chaining for Rvalue objects.
|
||||
///
|
||||
/// \param dst_format Color format of input image.
|
||||
///
|
||||
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner
|
||||
PreProcessSteps&& convert_color(const ov::preprocess::ColorFormat& dst_format) &&;
|
||||
|
||||
/// \brief Add scale preprocess operation - Lvalue version
|
||||
/// Divide each element of input by specified value
|
||||
///
|
||||
|
||||
23
ngraph/core/src/preprocess/color_utils.cpp
Normal file
23
ngraph/core/src/preprocess/color_utils.cpp
Normal file
@ -0,0 +1,23 @@
|
||||
// Copyright (C) 2018-2021 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#include "color_utils.hpp"
|
||||
|
||||
using namespace ov::preprocess;
|
||||
|
||||
std::unique_ptr<ColorFormatInfo> ColorFormatInfo::get(ColorFormat format) {
|
||||
std::unique_ptr<ColorFormatInfo> res;
|
||||
switch (format) {
|
||||
case ColorFormat::NV12_SINGLE_PLANE:
|
||||
res.reset(new ColorFormatInfoNV12_Single(format));
|
||||
break;
|
||||
case ColorFormat::NV12_TWO_PLANES:
|
||||
res.reset(new ColorFormatInfoNV12_TwoPlanes(format));
|
||||
break;
|
||||
default:
|
||||
res.reset(new ColorFormatInfo(format));
|
||||
break;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
143
ngraph/core/src/preprocess/color_utils.hpp
Normal file
143
ngraph/core/src/preprocess/color_utils.hpp
Normal file
@ -0,0 +1,143 @@
|
||||
// Copyright (C) 2018-2021 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "openvino/core/layout.hpp"
|
||||
#include "openvino/core/partial_shape.hpp"
|
||||
#include "openvino/core/preprocess/color_format.hpp"
|
||||
|
||||
namespace ov {
|
||||
namespace preprocess {
|
||||
|
||||
/// \brief Helper function to check if color format represents RGB family
|
||||
inline bool is_rgb_family(const ColorFormat& format) {
|
||||
return format == ColorFormat::RGB || format == ColorFormat::BGR;
|
||||
}
|
||||
|
||||
inline std::string color_format_name(ColorFormat format) {
|
||||
std::string name;
|
||||
switch (format) {
|
||||
case ColorFormat::RGB:
|
||||
name = "RGB";
|
||||
break;
|
||||
case ColorFormat::BGR:
|
||||
name = "BGR";
|
||||
break;
|
||||
case ColorFormat::NV12_TWO_PLANES:
|
||||
name = "NV12 (multi-plane)";
|
||||
break;
|
||||
case ColorFormat::NV12_SINGLE_PLANE:
|
||||
name = "NV12 (single plane)";
|
||||
break;
|
||||
default:
|
||||
name = "Unknown";
|
||||
break;
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
/// \brief Internal helper class to get information depending on color format
|
||||
class ColorFormatInfo {
|
||||
public:
|
||||
static std::unique_ptr<ColorFormatInfo> get(ColorFormat format);
|
||||
|
||||
virtual ~ColorFormatInfo() = default;
|
||||
|
||||
virtual size_t planes_count() const {
|
||||
return 1;
|
||||
}
|
||||
|
||||
virtual Layout default_layout() const {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Calculate shape of plane based image shape in NHWC format
|
||||
PartialShape shape(size_t plane_num, const PartialShape& image_src_shape) const {
|
||||
OPENVINO_ASSERT(plane_num < planes_count(),
|
||||
"Internal error: incorrect plane number specified for color format");
|
||||
return calculate_shape(plane_num, image_src_shape);
|
||||
}
|
||||
|
||||
std::string friendly_suffix(size_t plane_num) const {
|
||||
OPENVINO_ASSERT(plane_num < planes_count(),
|
||||
"Internal error: incorrect plane number specified for color format");
|
||||
return calc_name_suffix(plane_num);
|
||||
}
|
||||
|
||||
protected:
|
||||
virtual PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const {
|
||||
return image_shape;
|
||||
}
|
||||
virtual std::string calc_name_suffix(size_t plane_num) const {
|
||||
return {};
|
||||
}
|
||||
explicit ColorFormatInfo(ColorFormat format) : m_format(format) {}
|
||||
ColorFormat m_format;
|
||||
};
|
||||
|
||||
// --- Derived classes ---
|
||||
class ColorFormatInfoNV12_Single : public ColorFormatInfo {
|
||||
public:
|
||||
explicit ColorFormatInfoNV12_Single(ColorFormat format) : ColorFormatInfo(format) {}
|
||||
|
||||
protected:
|
||||
PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const override {
|
||||
PartialShape result = image_shape;
|
||||
if (image_shape.rank().is_static() && image_shape.rank().get_length() == 4) {
|
||||
result[3] = 1;
|
||||
if (result[1].is_static()) {
|
||||
result[1] = result[1].get_length() * 3 / 2;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
Layout default_layout() const override {
|
||||
return "NHWC";
|
||||
}
|
||||
};
|
||||
|
||||
class ColorFormatInfoNV12_TwoPlanes : public ColorFormatInfo {
|
||||
public:
|
||||
explicit ColorFormatInfoNV12_TwoPlanes(ColorFormat format) : ColorFormatInfo(format) {}
|
||||
|
||||
size_t planes_count() const override {
|
||||
return 2;
|
||||
}
|
||||
|
||||
protected:
|
||||
PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const override {
|
||||
PartialShape result = image_shape;
|
||||
if (image_shape.rank().is_static() && image_shape.rank().get_length() == 4) {
|
||||
if (plane_num == 0) {
|
||||
result[3] = 1;
|
||||
return result;
|
||||
} else {
|
||||
// UV plane has half or width and half of height. Number of channels is 2
|
||||
if (result[1].is_static()) {
|
||||
result[1] = result[1].get_length() / 2;
|
||||
}
|
||||
if (result[2].is_static()) {
|
||||
result[2] = result[2].get_length() / 2;
|
||||
}
|
||||
result[3] = 2;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
std::string calc_name_suffix(size_t plane_num) const override {
|
||||
if (plane_num == 0) {
|
||||
return "/Y";
|
||||
}
|
||||
return "/UV";
|
||||
}
|
||||
|
||||
Layout default_layout() const override {
|
||||
return "NHWC";
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace preprocess
|
||||
} // namespace ov
|
||||
55
ngraph/core/src/preprocess/function_guard.hpp
Normal file
55
ngraph/core/src/preprocess/function_guard.hpp
Normal file
@ -0,0 +1,55 @@
|
||||
// Copyright (C) 2018-2021 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "openvino/core/function.hpp"
|
||||
|
||||
namespace ov {
|
||||
namespace preprocess {
|
||||
|
||||
/// \brief Internal guard to make preprocess builder exception-safe
|
||||
class FunctionGuard {
|
||||
std::shared_ptr<Function> m_function;
|
||||
ParameterVector m_parameters;
|
||||
std::map<std::shared_ptr<op::v0::Parameter>, std::set<Input<Node>>> m_backup;
|
||||
bool m_done = false;
|
||||
|
||||
public:
|
||||
FunctionGuard(const std::shared_ptr<Function>& f) : m_function(f) {
|
||||
m_parameters = f->get_parameters();
|
||||
for (const auto& param : f->get_parameters()) {
|
||||
m_backup.insert({param, param->output(0).get_target_inputs()});
|
||||
}
|
||||
}
|
||||
virtual ~FunctionGuard() {
|
||||
if (!m_done) {
|
||||
try {
|
||||
auto params = m_function->get_parameters();
|
||||
// Remove parameters added by preprocessing
|
||||
for (const auto& param : params) {
|
||||
m_function->remove_parameter(param);
|
||||
}
|
||||
// Insert old parameters and update consumers
|
||||
for (const auto& item : m_backup) {
|
||||
// Replace consumers
|
||||
for (auto consumer : item.second) {
|
||||
consumer.replace_source_output(item.first);
|
||||
}
|
||||
}
|
||||
m_function->add_parameters(m_parameters);
|
||||
} catch (std::exception& ex) {
|
||||
// Stress condition, can't recover function to original state
|
||||
std::cerr << "Unrecoverable error occurred during preprocessing. Function is corrupted, exiting\n";
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
}
|
||||
void reset() noexcept {
|
||||
m_done = true;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace preprocess
|
||||
} // namespace ov
|
||||
@ -4,6 +4,8 @@
|
||||
|
||||
#include "openvino/core/preprocess/pre_post_process.hpp"
|
||||
|
||||
#include "color_utils.hpp"
|
||||
#include "function_guard.hpp"
|
||||
#include "ngraph/opsets/opset1.hpp"
|
||||
#include "openvino/core/function.hpp"
|
||||
#include "preprocess_steps_impl.hpp"
|
||||
@ -66,7 +68,39 @@ public:
|
||||
m_spatial_width = static_cast<int>(width);
|
||||
}
|
||||
|
||||
const ColorFormat& get_color_format() const {
|
||||
return m_color_format;
|
||||
}
|
||||
|
||||
void set_color_format(ColorFormat format, const std::vector<std::string>& sub_names) {
|
||||
auto info = ColorFormatInfo::get(format);
|
||||
if (info->planes_count() == 1) {
|
||||
OPENVINO_ASSERT(sub_names.empty(),
|
||||
"Plane names are not allowed for single plane color format '",
|
||||
color_format_name(format),
|
||||
"'");
|
||||
} else if (!sub_names.empty()) {
|
||||
OPENVINO_ASSERT(sub_names.size() == info->planes_count(),
|
||||
"Number of sub-names (",
|
||||
sub_names.size(),
|
||||
") shall match with number of planes for '",
|
||||
color_format_name(format),
|
||||
"' color format (",
|
||||
info->planes_count(),
|
||||
")");
|
||||
}
|
||||
m_planes_sub_names = sub_names;
|
||||
m_color_format = format;
|
||||
}
|
||||
|
||||
const std::vector<std::string>& planes_sub_names() const {
|
||||
return m_planes_sub_names;
|
||||
}
|
||||
|
||||
private:
|
||||
ColorFormat m_color_format = ColorFormat::UNDEFINED;
|
||||
std::vector<std::string> m_planes_sub_names;
|
||||
|
||||
element::Type m_type = element::dynamic;
|
||||
bool m_type_set = false;
|
||||
|
||||
@ -120,6 +154,7 @@ struct InputInfo::InputInfoImpl {
|
||||
std::unique_ptr<InputTensorInfo::InputTensorInfoImpl> m_tensor_data;
|
||||
std::unique_ptr<PreProcessSteps::PreProcessStepsImpl> m_preprocess;
|
||||
std::unique_ptr<InputNetworkInfo::InputNetworkInfoImpl> m_network_data;
|
||||
std::shared_ptr<op::v0::Parameter> m_resolved_param;
|
||||
};
|
||||
|
||||
//-------------- InputInfo ------------------
|
||||
@ -181,6 +216,7 @@ PrePostProcessor&& PrePostProcessor::input(InputInfo&& builder) && {
|
||||
}
|
||||
|
||||
std::shared_ptr<Function> PrePostProcessor::build(const std::shared_ptr<Function>& function) {
|
||||
FunctionGuard guard(function);
|
||||
bool tensor_data_updated = false;
|
||||
for (const auto& input : m_impl->in_contexts) {
|
||||
std::shared_ptr<op::v0::Parameter> param;
|
||||
@ -200,19 +236,30 @@ std::shared_ptr<Function> PrePostProcessor::build(const std::shared_ptr<Function
|
||||
if (input->m_network_data && input->m_network_data->is_layout_set() && param->get_layout().empty()) {
|
||||
param->set_layout(input->m_network_data->get_layout());
|
||||
}
|
||||
input->m_resolved_param = param;
|
||||
}
|
||||
|
||||
for (const auto& input : m_impl->in_contexts) {
|
||||
auto param = input->m_resolved_param;
|
||||
auto consumers = param->output(0).get_target_inputs();
|
||||
if (!input->m_tensor_data) {
|
||||
input->create_tensor_data(param->get_element_type(), param->get_layout());
|
||||
}
|
||||
if (!input->m_tensor_data->is_layout_set() && param->get_layout() != Layout()) {
|
||||
input->m_tensor_data->set_layout(param->get_layout());
|
||||
}
|
||||
if (!input->m_tensor_data->is_element_type_set()) {
|
||||
input->m_tensor_data->set_element_type(param->get_element_type());
|
||||
}
|
||||
auto color_info = ColorFormatInfo::get(input->m_tensor_data->get_color_format());
|
||||
if (!input->m_tensor_data->is_layout_set()) {
|
||||
if (!color_info->default_layout().empty()) {
|
||||
input->m_tensor_data->set_layout(color_info->default_layout());
|
||||
} else if (!param->get_layout().empty()) {
|
||||
input->m_tensor_data->set_layout(param->get_layout());
|
||||
}
|
||||
}
|
||||
|
||||
auto net_shape = param->get_partial_shape();
|
||||
auto new_param_shape = net_shape;
|
||||
if (input->m_tensor_data->is_layout_set() && param->get_layout() != Layout() &&
|
||||
if (input->m_tensor_data->is_layout_set() && !param->get_layout().empty() &&
|
||||
param->get_layout() != input->m_tensor_data->get_layout()) {
|
||||
// Find transpose between network and tensor layouts and update tensor shape
|
||||
auto net_to_tensor =
|
||||
@ -236,26 +283,54 @@ std::shared_ptr<Function> PrePostProcessor::build(const std::shared_ptr<Function
|
||||
new_param_shape[width_idx] = input->m_tensor_data->get_spatial_width();
|
||||
}
|
||||
}
|
||||
auto new_param = std::make_shared<op::v0::Parameter>(input->m_tensor_data->get_element_type(), new_param_shape);
|
||||
if (input->m_tensor_data->is_layout_set()) {
|
||||
new_param->set_layout(input->m_tensor_data->get_layout());
|
||||
|
||||
std::vector<std::shared_ptr<ov::Node>> nodes;
|
||||
std::vector<std::shared_ptr<op::v0::Parameter>> new_params;
|
||||
|
||||
// Create separate parameter for each plane. Shape and friendly name is based on color format
|
||||
for (size_t plane = 0; plane < color_info->planes_count(); plane++) {
|
||||
auto plane_shape = color_info->shape(plane, new_param_shape);
|
||||
auto plane_param =
|
||||
std::make_shared<op::v0::Parameter>(input->m_tensor_data->get_element_type(), plane_shape);
|
||||
if (plane < input->m_tensor_data->planes_sub_names().size()) {
|
||||
auto sub_name = std::string("/") + input->m_tensor_data->planes_sub_names()[plane];
|
||||
inherit_friendly_names(function, param, plane_param, sub_name, false);
|
||||
} else {
|
||||
auto sub_name = color_info->friendly_suffix(plane);
|
||||
inherit_friendly_names(function, param, plane_param, sub_name);
|
||||
}
|
||||
if (!input->m_tensor_data->get_layout().empty()) {
|
||||
plane_param->set_layout(input->m_tensor_data->get_layout());
|
||||
}
|
||||
new_params.push_back(plane_param);
|
||||
nodes.push_back(plane_param);
|
||||
}
|
||||
// Old param will be removed, so friendly name can be reused
|
||||
new_param->set_friendly_name(param->get_friendly_name());
|
||||
|
||||
// Also reuse names of original tensor
|
||||
new_param->get_output_tensor(0).set_names(param->get_output_tensor(0).get_names());
|
||||
|
||||
std::shared_ptr<Node> node = new_param;
|
||||
PreprocessingContext context(new_param->get_layout());
|
||||
PreprocessingContext context(input->m_tensor_data->get_layout());
|
||||
context.color_format() = input->m_tensor_data->get_color_format();
|
||||
context.network_layout() = param->get_layout();
|
||||
context.network_shape() = param->get_partial_shape();
|
||||
|
||||
// 2. Apply preprocessing
|
||||
for (const auto& action : input->m_preprocess->actions()) {
|
||||
node = std::get<0>(action)({node}, context);
|
||||
tensor_data_updated |= std::get<1>(action);
|
||||
if (input->m_preprocess) {
|
||||
for (const auto& action : input->m_preprocess->actions()) {
|
||||
auto node = std::get<0>(action)(nodes, function, context);
|
||||
nodes = {node};
|
||||
tensor_data_updated |= std::get<1>(action);
|
||||
}
|
||||
}
|
||||
|
||||
OPENVINO_ASSERT(nodes.size() == 1,
|
||||
"Multiple plane input is not allowed as network input. Consider using of convert_color "
|
||||
"preprocessing operation. Current format is '",
|
||||
color_format_name(context.color_format()),
|
||||
"'");
|
||||
OPENVINO_ASSERT(is_rgb_family(context.color_format()) || context.color_format() == ColorFormat::UNDEFINED,
|
||||
"Network shall have RGB/BGR color format. Consider add 'convert_color' preprocessing operation "
|
||||
"to convert current color format '",
|
||||
color_format_name(context.color_format()),
|
||||
"'to RGB/BGR");
|
||||
auto node = nodes[0];
|
||||
// Check final type
|
||||
OPENVINO_ASSERT(node->get_element_type() == param->get_element_type(),
|
||||
std::string("Element type after preprocessing {") + node->get_element_type().c_type_string() +
|
||||
@ -267,13 +342,14 @@ std::shared_ptr<Function> PrePostProcessor::build(const std::shared_ptr<Function
|
||||
for (auto consumer : consumers) {
|
||||
consumer.replace_source_output(node);
|
||||
}
|
||||
function->add_parameters({new_param});
|
||||
function->add_parameters(new_params);
|
||||
// remove old parameter
|
||||
function->remove_parameter(param);
|
||||
}
|
||||
if (tensor_data_updated) {
|
||||
function->validate_nodes_and_infer_types();
|
||||
}
|
||||
guard.reset();
|
||||
return function;
|
||||
}
|
||||
|
||||
@ -339,6 +415,18 @@ InputNetworkInfo&& InputNetworkInfo::set_layout(const Layout& layout) && {
|
||||
return std::move(*this);
|
||||
}
|
||||
|
||||
InputTensorInfo& InputTensorInfo::set_color_format(const ov::preprocess::ColorFormat& format,
|
||||
const std::vector<std::string>& sub_names) & {
|
||||
m_impl->set_color_format(format, sub_names);
|
||||
return *this;
|
||||
}
|
||||
|
||||
InputTensorInfo&& InputTensorInfo::set_color_format(const ov::preprocess::ColorFormat& format,
|
||||
const std::vector<std::string>& sub_names) && {
|
||||
m_impl->set_color_format(format, sub_names);
|
||||
return std::move(*this);
|
||||
}
|
||||
|
||||
// --------------------- PreProcessSteps ------------------
|
||||
|
||||
PreProcessSteps::PreProcessSteps() : m_impl(std::unique_ptr<PreProcessStepsImpl>(new PreProcessStepsImpl())) {}
|
||||
@ -432,14 +520,26 @@ PreProcessSteps&& PreProcessSteps::convert_layout(const Layout& dst_layout) && {
|
||||
return std::move(*this);
|
||||
}
|
||||
|
||||
PreProcessSteps& PreProcessSteps::convert_color(const ov::preprocess::ColorFormat& dst_format) & {
|
||||
m_impl->add_convert_color_impl(dst_format);
|
||||
return *this;
|
||||
}
|
||||
|
||||
PreProcessSteps&& PreProcessSteps::convert_color(const ov::preprocess::ColorFormat& dst_format) && {
|
||||
m_impl->add_convert_color_impl(dst_format);
|
||||
return std::move(*this);
|
||||
}
|
||||
|
||||
PreProcessSteps& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb) & {
|
||||
// 'true' indicates that custom preprocessing step will trigger validate_and_infer_types
|
||||
m_impl->actions().emplace_back(std::make_tuple(
|
||||
[preprocess_cb](const std::vector<std::shared_ptr<ov::Node>>& nodes, PreprocessingContext&) {
|
||||
[preprocess_cb](const std::vector<std::shared_ptr<ov::Node>>& nodes,
|
||||
const std::shared_ptr<ov::Function>&,
|
||||
PreprocessingContext&) -> std::vector<std::shared_ptr<ov::Node>> {
|
||||
OPENVINO_ASSERT(nodes.size() == 1,
|
||||
"Can't apply custom preprocessing step for multi-plane input. Suggesting to convert "
|
||||
"current image to RGB/BGR color format using 'convert_color'");
|
||||
return preprocess_cb(nodes[0]);
|
||||
return {preprocess_cb(nodes[0])};
|
||||
},
|
||||
true));
|
||||
return *this;
|
||||
@ -448,11 +548,13 @@ PreProcessSteps& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb
|
||||
PreProcessSteps&& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb) && {
|
||||
// 'true' indicates that custom preprocessing step will trigger validate_and_infer_types
|
||||
m_impl->actions().emplace_back(std::make_tuple(
|
||||
[preprocess_cb](const std::vector<std::shared_ptr<ov::Node>>& nodes, PreprocessingContext&) {
|
||||
[preprocess_cb](const std::vector<std::shared_ptr<ov::Node>>& nodes,
|
||||
const std::shared_ptr<ov::Function>&,
|
||||
PreprocessingContext&) -> std::vector<std::shared_ptr<ov::Node>> {
|
||||
OPENVINO_ASSERT(nodes.size() == 1,
|
||||
"Can't apply custom preprocessing step for multi-plane input. Suggesting to convert "
|
||||
"current image to RGB/BGR color format using 'convert_color'");
|
||||
return preprocess_cb(nodes[0]);
|
||||
return {preprocess_cb(nodes[0])};
|
||||
},
|
||||
true));
|
||||
return std::move(*this);
|
||||
|
||||
@ -4,9 +4,12 @@
|
||||
|
||||
#include "preprocess_steps_impl.hpp"
|
||||
|
||||
#include "color_utils.hpp"
|
||||
#include "ngraph/opsets/opset1.hpp"
|
||||
#include "openvino/core/node.hpp"
|
||||
#include "openvino/core/shape.hpp"
|
||||
#include "openvino/op/nv12_to_bgr.hpp"
|
||||
#include "openvino/op/nv12_to_rgb.hpp"
|
||||
|
||||
namespace ov {
|
||||
namespace preprocess {
|
||||
@ -31,7 +34,9 @@ static Shape construct_mean_scale_shape(const std::shared_ptr<Node>& node,
|
||||
|
||||
void PreProcessSteps::PreProcessStepsImpl::add_scale_impl(const std::vector<float>& values) {
|
||||
m_actions.emplace_back(std::make_tuple(
|
||||
[values](const std::vector<std::shared_ptr<Node>>& nodes, PreprocessingContext& context) {
|
||||
[values](const std::vector<std::shared_ptr<Node>>& nodes,
|
||||
const std::shared_ptr<ov::Function>& function,
|
||||
PreprocessingContext& context) -> std::vector<std::shared_ptr<ov::Node>> {
|
||||
OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't apply scale preprocessing for empty input.");
|
||||
OPENVINO_ASSERT(nodes.size() == 1,
|
||||
"Can't apply scale preprocessing for multi-plane input. Suggesting to convert current "
|
||||
@ -43,18 +48,20 @@ void PreProcessSteps::PreProcessStepsImpl::add_scale_impl(const std::vector<floa
|
||||
shape = construct_mean_scale_shape(nodes[0], values.size(), context);
|
||||
}
|
||||
auto constant = op::v0::Constant::create(element::f32, shape, values);
|
||||
constant->set_friendly_name(nodes[0]->get_friendly_name() + "/scale/Divide_Factor");
|
||||
inherit_friendly_names(function, nodes[0], constant, "/scale/Divide_Factor");
|
||||
|
||||
auto new_op = std::make_shared<op::v1::Divide>(nodes[0], constant);
|
||||
new_op->set_friendly_name(nodes[0]->get_friendly_name() + "/scale/Divide");
|
||||
return new_op;
|
||||
inherit_friendly_names(function, nodes[0], new_op, "/scale/Divide");
|
||||
return {new_op};
|
||||
},
|
||||
false));
|
||||
}
|
||||
|
||||
void PreProcessSteps::PreProcessStepsImpl::add_mean_impl(const std::vector<float>& values) {
|
||||
m_actions.emplace_back(std::make_tuple(
|
||||
[values](const std::vector<std::shared_ptr<Node>>& nodes, PreprocessingContext& context) {
|
||||
[values](const std::vector<std::shared_ptr<Node>>& nodes,
|
||||
const std::shared_ptr<ov::Function>& function,
|
||||
PreprocessingContext& context) -> std::vector<std::shared_ptr<ov::Node>> {
|
||||
OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't apply mean preprocessing for empty input.");
|
||||
OPENVINO_ASSERT(nodes.size() == 1,
|
||||
"Can't apply scale preprocessing for multi-plane input. Suggesting to convert current "
|
||||
@ -66,27 +73,30 @@ void PreProcessSteps::PreProcessStepsImpl::add_mean_impl(const std::vector<float
|
||||
shape = construct_mean_scale_shape(nodes[0], values.size(), context);
|
||||
}
|
||||
auto constant = op::v0::Constant::create(element::f32, shape, values);
|
||||
constant->set_friendly_name(nodes[0]->get_friendly_name() + "/mean/Mean_Const");
|
||||
inherit_friendly_names(function, nodes[0], constant, "/mean/Mean_Const");
|
||||
|
||||
auto new_op = std::make_shared<op::v1::Subtract>(nodes[0], constant);
|
||||
new_op->set_friendly_name(nodes[0]->get_friendly_name() + "/mean/Subtract");
|
||||
return new_op;
|
||||
inherit_friendly_names(function, nodes[0], new_op, "/mean/Subtract");
|
||||
return {new_op};
|
||||
},
|
||||
false));
|
||||
}
|
||||
|
||||
void PreProcessSteps::PreProcessStepsImpl::add_convert_impl(const ov::element::Type& type) {
|
||||
m_actions.emplace_back(std::make_tuple(
|
||||
[type](const std::vector<std::shared_ptr<Node>>& nodes, PreprocessingContext&) {
|
||||
[type](const std::vector<std::shared_ptr<Node>>& nodes,
|
||||
const std::shared_ptr<ov::Function>& function,
|
||||
PreprocessingContext&) -> std::vector<std::shared_ptr<ov::Node>> {
|
||||
OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't set element type for empty input.");
|
||||
OPENVINO_ASSERT(nodes.size() == 1,
|
||||
"Can't set element type for multi-plane input. Suggesting to convert current image to "
|
||||
"RGB/BGR color format using 'convert_color'");
|
||||
OPENVINO_ASSERT(nodes[0]->get_element_type().is_static(),
|
||||
"Can't insert 'convert_element_type' for dynamic source tensor type.");
|
||||
auto convert = std::make_shared<op::v0::Convert>(nodes[0], type);
|
||||
convert->set_friendly_name(nodes[0]->get_friendly_name() + "/convert_element_type");
|
||||
return convert;
|
||||
std::vector<std::shared_ptr<ov::Node>> res;
|
||||
for (const auto& node : nodes) {
|
||||
OPENVINO_ASSERT(node->get_element_type().is_static(),
|
||||
"Can't insert 'convert_element_type' for dynamic source tensor type.");
|
||||
auto convert = std::make_shared<op::v0::Convert>(node, type);
|
||||
inherit_friendly_names(function, node, convert, "/convert_element_type");
|
||||
res.emplace_back(convert);
|
||||
}
|
||||
return res;
|
||||
},
|
||||
true));
|
||||
}
|
||||
@ -94,7 +104,9 @@ void PreProcessSteps::PreProcessStepsImpl::add_convert_impl(const ov::element::T
|
||||
void PreProcessSteps::PreProcessStepsImpl::add_resize_impl(ResizeAlgorithm alg, int dst_height, int dst_width) {
|
||||
using InterpolateMode = op::v4::Interpolate::InterpolateMode;
|
||||
m_actions.emplace_back(std::make_tuple(
|
||||
[alg, dst_width, dst_height](const std::vector<std::shared_ptr<Node>>& nodes, PreprocessingContext& ctxt) {
|
||||
[alg, dst_width, dst_height](const std::vector<std::shared_ptr<Node>>& nodes,
|
||||
const std::shared_ptr<ov::Function>& function,
|
||||
PreprocessingContext& ctxt) -> std::vector<std::shared_ptr<ov::Node>> {
|
||||
OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't add resize for empty input.");
|
||||
OPENVINO_ASSERT(nodes.size() == 1,
|
||||
"Can't resize multi-plane input. Suggesting to convert current image to "
|
||||
@ -140,28 +152,88 @@ void PreProcessSteps::PreProcessStepsImpl::add_resize_impl(ResizeAlgorithm alg,
|
||||
{0, 0});
|
||||
|
||||
auto interp = std::make_shared<op::v4::Interpolate>(node, target_spatial_shape, scales, axes, attrs);
|
||||
interp->set_friendly_name(nodes[0]->get_friendly_name() + "/resize");
|
||||
return interp;
|
||||
inherit_friendly_names(function, nodes[0], interp, "/resize");
|
||||
return {interp};
|
||||
},
|
||||
true));
|
||||
}
|
||||
|
||||
void PreProcessSteps::PreProcessStepsImpl::add_convert_layout_impl(const Layout& layout) {
|
||||
m_actions.emplace_back(std::make_tuple(
|
||||
[layout](const std::vector<std::shared_ptr<Node>>& nodes, PreprocessingContext& context) {
|
||||
[layout](const std::vector<std::shared_ptr<Node>>& nodes,
|
||||
const std::shared_ptr<ov::Function>& function,
|
||||
PreprocessingContext& context) -> std::vector<std::shared_ptr<ov::Node>> {
|
||||
OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't convert layout for empty input.");
|
||||
OPENVINO_ASSERT(nodes.size() == 1,
|
||||
"Can't convert layout for multi-plane input. Suggesting to convert current image to "
|
||||
"RGB/BGR color format using 'convert_color'");
|
||||
"RGB/BGR color format using 'convert_color'. Current format is '",
|
||||
color_format_name(context.color_format()),
|
||||
"'");
|
||||
Layout dst_layout = layout.empty() ? context.network_layout() : layout;
|
||||
auto permutation =
|
||||
layout::find_permutation(context.layout(), nodes[0]->get_output_partial_shape(0).rank(), dst_layout);
|
||||
auto perm_constant =
|
||||
op::v0::Constant::create<int64_t>(element::i64, Shape{permutation.size()}, permutation);
|
||||
auto transpose = std::make_shared<op::v1::Transpose>(nodes[0], perm_constant);
|
||||
transpose->set_friendly_name(nodes[0]->get_friendly_name() + "/convert_layout");
|
||||
inherit_friendly_names(function, nodes[0], transpose, "/convert_layout");
|
||||
context.layout() = dst_layout; // Update context's current layout
|
||||
return transpose;
|
||||
return {transpose};
|
||||
},
|
||||
true));
|
||||
}
|
||||
|
||||
void PreProcessSteps::PreProcessStepsImpl::add_convert_color_impl(const ColorFormat& dst_format) {
|
||||
m_actions.emplace_back(std::make_tuple(
|
||||
[&, dst_format](const std::vector<std::shared_ptr<Node>>& nodes,
|
||||
const std::shared_ptr<ov::Function>& function,
|
||||
PreprocessingContext& context) -> std::vector<std::shared_ptr<ov::Node>> {
|
||||
if (context.color_format() == dst_format) {
|
||||
return nodes;
|
||||
}
|
||||
if (context.color_format() == ColorFormat::NV12_SINGLE_PLANE) {
|
||||
OPENVINO_ASSERT(nodes.size() == 1,
|
||||
"Internal error: single plane NV12 image can't have multiple inputs");
|
||||
std::shared_ptr<Node> convert;
|
||||
switch (dst_format) {
|
||||
case ColorFormat::RGB:
|
||||
convert = std::make_shared<op::v8::NV12toRGB>(nodes[0]);
|
||||
break;
|
||||
case ColorFormat::BGR:
|
||||
convert = std::make_shared<op::v8::NV12toBGR>(nodes[0]);
|
||||
break;
|
||||
default:
|
||||
OPENVINO_ASSERT(false,
|
||||
"Unsupported conversion from NV12 to '",
|
||||
color_format_name(dst_format),
|
||||
"' format:");
|
||||
}
|
||||
inherit_friendly_names(function, nodes[0], convert, "/convert_color_nv12_single");
|
||||
context.color_format() = dst_format;
|
||||
return {convert};
|
||||
} else if (context.color_format() == ColorFormat::NV12_TWO_PLANES) {
|
||||
OPENVINO_ASSERT(nodes.size() == 2, "Internal error: two-plane NV12 image must have exactly two inputs");
|
||||
std::shared_ptr<Node> convert;
|
||||
switch (dst_format) {
|
||||
case ColorFormat::RGB:
|
||||
convert = std::make_shared<op::v8::NV12toRGB>(nodes[0], nodes[1]);
|
||||
break;
|
||||
case ColorFormat::BGR:
|
||||
convert = std::make_shared<op::v8::NV12toBGR>(nodes[0], nodes[1]);
|
||||
break;
|
||||
default:
|
||||
OPENVINO_ASSERT(false,
|
||||
"Unsupported conversion from NV12 to '",
|
||||
color_format_name(dst_format),
|
||||
"' format:");
|
||||
}
|
||||
inherit_friendly_names(function, nodes[0], convert, "/convert_color_nv12_two_planes");
|
||||
context.color_format() = dst_format;
|
||||
return {convert};
|
||||
}
|
||||
OPENVINO_ASSERT(false,
|
||||
"Source color format '",
|
||||
color_format_name(context.color_format()),
|
||||
"' is not convertible to any other");
|
||||
},
|
||||
true));
|
||||
}
|
||||
|
||||
@ -7,8 +7,11 @@
|
||||
#include <list>
|
||||
|
||||
#include "openvino/core/layout.hpp"
|
||||
#include "openvino/core/node.hpp"
|
||||
#include "openvino/core/partial_shape.hpp"
|
||||
#include "openvino/core/preprocess/color_format.hpp"
|
||||
#include "openvino/core/preprocess/preprocess_steps.hpp"
|
||||
#include "tensor_name_util.hpp"
|
||||
|
||||
namespace ov {
|
||||
namespace preprocess {
|
||||
@ -55,11 +58,37 @@ inline size_t get_and_check_channels_idx(const Layout& layout, const PartialShap
|
||||
return idx;
|
||||
}
|
||||
|
||||
inline void inherit_friendly_names(const std::shared_ptr<ov::Function>& function,
|
||||
const std::shared_ptr<ov::Node>& src_node,
|
||||
const std::shared_ptr<ov::Node>& dst_node,
|
||||
const std::string& suffix,
|
||||
bool search_for_available_name = true) {
|
||||
OPENVINO_ASSERT(src_node->get_output_size() == 1 && dst_node->get_output_size() == 1,
|
||||
"Internal error. Preprocessing steps must contain nodes with one output");
|
||||
dst_node->set_friendly_name(src_node->get_friendly_name() + suffix);
|
||||
std::unordered_set<std::string> new_names;
|
||||
for (const auto& tensor_name : src_node->output(0).get_tensor().get_names()) {
|
||||
auto new_tensor_name = tensor_name + suffix;
|
||||
if (!suffix.empty()) {
|
||||
// Verify that new names are unique for a function
|
||||
if (!is_tensor_name_available(new_tensor_name, function) && search_for_available_name) {
|
||||
// Search for available name
|
||||
size_t idx = 0;
|
||||
do {
|
||||
new_tensor_name = tensor_name + suffix + std::to_string(idx++);
|
||||
} while (!is_tensor_name_available(new_tensor_name, function));
|
||||
}
|
||||
}
|
||||
new_names.emplace(new_tensor_name);
|
||||
}
|
||||
dst_node->output(0).get_tensor().set_names(new_names);
|
||||
}
|
||||
|
||||
/// \brief Preprocessing context passed to each preprocessing operation.
|
||||
/// This is internal structure which is not shared to custom operations yet.
|
||||
class PreprocessingContext {
|
||||
public:
|
||||
explicit PreprocessingContext(const Layout& layout) : m_layout(layout) {}
|
||||
explicit PreprocessingContext(Layout layout) : m_layout(std::move(layout)) {}
|
||||
|
||||
const Layout& layout() const {
|
||||
return m_layout;
|
||||
@ -99,15 +128,25 @@ public:
|
||||
return network_shape()[network_width_idx].get_length();
|
||||
}
|
||||
|
||||
const ColorFormat& color_format() const {
|
||||
return m_color_format;
|
||||
}
|
||||
|
||||
ColorFormat& color_format() {
|
||||
return m_color_format;
|
||||
}
|
||||
|
||||
private:
|
||||
Layout m_layout;
|
||||
PartialShape m_network_shape;
|
||||
Layout m_network_layout;
|
||||
ColorFormat m_color_format = ColorFormat::UNDEFINED;
|
||||
};
|
||||
|
||||
using InternalPreprocessOp =
|
||||
std::function<std::shared_ptr<ov::Node>(const std::vector<std::shared_ptr<ov::Node>>& nodes,
|
||||
PreprocessingContext& context)>;
|
||||
std::function<std::vector<std::shared_ptr<ov::Node>>(const std::vector<std::shared_ptr<ov::Node>>& nodes,
|
||||
const std::shared_ptr<ov::Function>& function,
|
||||
PreprocessingContext& context)>;
|
||||
|
||||
/// \brief PreProcessStepsImpl - internal data structure
|
||||
class PreProcessSteps::PreProcessStepsImpl {
|
||||
@ -117,6 +156,7 @@ public:
|
||||
void add_convert_impl(const element::Type& type);
|
||||
void add_resize_impl(ResizeAlgorithm alg, int dst_height, int dst_width);
|
||||
void add_convert_layout_impl(const Layout& layout);
|
||||
void add_convert_color_impl(const ColorFormat& dst_format);
|
||||
|
||||
const std::list<std::tuple<InternalPreprocessOp, bool>>& actions() const {
|
||||
return m_actions;
|
||||
|
||||
29
ngraph/core/src/tensor_name_util.hpp
Normal file
29
ngraph/core/src/tensor_name_util.hpp
Normal file
@ -0,0 +1,29 @@
|
||||
// Copyright (C) 2018-2021 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "openvino/core/except.hpp"
|
||||
#include "openvino/core/function.hpp"
|
||||
|
||||
namespace ov {
|
||||
|
||||
/// \brief Check that specified tensor name is unique for a given function.
|
||||
///
|
||||
/// \param tensor_name Name to check across all tensors in a function.
|
||||
/// \param function Function.
|
||||
/// \return False if tensor name is already used in some function's node, True otherwise
|
||||
inline bool is_tensor_name_available(const std::string& tensor_name, const std::shared_ptr<Function>& function) {
|
||||
for (const auto& node : function->get_ordered_ops()) {
|
||||
for (const auto& output : node->outputs()) {
|
||||
const auto& tensor = output.get_tensor();
|
||||
if (tensor.get_names().count(tensor_name)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace ov
|
||||
@ -57,6 +57,13 @@ TEST(pre_post_process, convert_element_type_and_scale) {
|
||||
EXPECT_EQ(f->get_output_element_type(0), element::i8);
|
||||
}
|
||||
|
||||
TEST(pre_post_process, empty_preprocess) {
|
||||
auto f = create_simple_function(element::i8, Shape{1, 3, 2, 2});
|
||||
f = PrePostProcessor().input(InputInfo().tensor(InputTensorInfo().set_element_type(element::i8))).build(f);
|
||||
EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::i8);
|
||||
EXPECT_EQ(f->get_output_element_type(0), element::i8);
|
||||
}
|
||||
|
||||
TEST(pre_post_process, convert_element_type_from_unknown) {
|
||||
auto f = create_simple_function(element::i32, Shape{1, 3, 224, 224});
|
||||
ASSERT_THROW(
|
||||
@ -107,6 +114,265 @@ TEST(pre_post_process, tensor_element_type_and_scale) {
|
||||
EXPECT_EQ(f->get_parameters().front()->get_layout(), Layout());
|
||||
}
|
||||
|
||||
TEST(pre_post_process, convert_color_nv12_rgb_single) {
|
||||
auto f = create_simple_function(element::f32, PartialShape{Dimension::dynamic(), 2, 2, 3});
|
||||
auto name = f->get_parameters()[0]->get_friendly_name();
|
||||
auto tensor_names = f->get_parameters().front()->get_output_tensor(0).get_names();
|
||||
f = PrePostProcessor()
|
||||
.input(
|
||||
InputInfo()
|
||||
.tensor(InputTensorInfo()
|
||||
.set_element_type(element::u8)
|
||||
.set_color_format(ColorFormat::NV12_SINGLE_PLANE))
|
||||
.preprocess(PreProcessSteps().convert_color(ColorFormat::RGB).convert_element_type(element::f32)))
|
||||
.build(f);
|
||||
|
||||
EXPECT_EQ(f->get_parameters().size(), 1);
|
||||
EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::u8);
|
||||
EXPECT_EQ(f->get_parameters().front()->get_layout(), "NHWC");
|
||||
EXPECT_EQ(f->get_parameters().front()->get_partial_shape(), (PartialShape{Dimension::dynamic(), 3, 2, 1}));
|
||||
EXPECT_EQ(f->get_parameters().front()->get_friendly_name(), name);
|
||||
EXPECT_EQ(f->get_parameters().front()->get_output_tensor(0).get_names(), tensor_names);
|
||||
}
|
||||
|
||||
TEST(pre_post_process, convert_color_nv12_bgr_single) {
|
||||
auto f = create_simple_function(element::f32, PartialShape{Dimension::dynamic(), 2, 2, 3});
|
||||
auto name = f->get_parameters()[0]->get_friendly_name();
|
||||
auto tensor_names = f->get_parameters().front()->get_output_tensor(0).get_names();
|
||||
f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_SINGLE_PLANE))
|
||||
.preprocess(PreProcessSteps().convert_color(ColorFormat::BGR)))
|
||||
.build(f);
|
||||
|
||||
EXPECT_EQ(f->get_parameters().size(), 1);
|
||||
EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::f32);
|
||||
EXPECT_EQ(f->get_parameters().front()->get_layout(), "NHWC");
|
||||
EXPECT_EQ(f->get_parameters().front()->get_partial_shape(), (PartialShape{Dimension::dynamic(), 3, 2, 1}));
|
||||
EXPECT_EQ(f->get_parameters().front()->get_friendly_name(), name);
|
||||
EXPECT_EQ(f->get_parameters().front()->get_output_tensor(0).get_names(), tensor_names);
|
||||
}
|
||||
|
||||
TEST(pre_post_process, convert_color_nv12_bgr_2_planes) {
|
||||
auto f = create_simple_function(element::f32, Shape{5, 2, 2, 3});
|
||||
f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES, {"TestY", "TestUV"}))
|
||||
.preprocess(PreProcessSteps().convert_color(ColorFormat::BGR)))
|
||||
.build(f);
|
||||
|
||||
EXPECT_EQ(f->get_parameters().size(), 2);
|
||||
EXPECT_EQ(f->get_parameters()[0]->get_friendly_name(), "input1/TestY");
|
||||
EXPECT_EQ(*f->get_parameters()[0]->output(0).get_tensor().get_names().begin(), "tensor_input1/TestY");
|
||||
EXPECT_EQ(f->get_parameters()[0]->get_element_type(), element::f32);
|
||||
EXPECT_EQ(f->get_parameters()[0]->get_partial_shape(), (PartialShape{5, 2, 2, 1}));
|
||||
|
||||
EXPECT_EQ(f->get_parameters()[1]->get_friendly_name(), "input1/TestUV");
|
||||
EXPECT_EQ(*f->get_parameters()[1]->output(0).get_tensor().get_names().begin(), "tensor_input1/TestUV");
|
||||
EXPECT_EQ(f->get_parameters()[1]->get_element_type(), element::f32);
|
||||
EXPECT_EQ(f->get_parameters()[1]->get_partial_shape(), (PartialShape{5, 1, 1, 2}));
|
||||
}
|
||||
|
||||
TEST(pre_post_process, convert_color_nv12_rgb_2_planes) {
|
||||
auto f = create_simple_function(element::f32, Shape{5, 2, 2, 3});
|
||||
f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES))
|
||||
.preprocess(PreProcessSteps().convert_color(ColorFormat::RGB)))
|
||||
.build(f);
|
||||
|
||||
EXPECT_EQ(f->get_parameters().size(), 2);
|
||||
EXPECT_EQ(f->get_parameters()[0]->get_element_type(), element::f32);
|
||||
EXPECT_EQ(f->get_parameters()[1]->get_element_type(), element::f32);
|
||||
EXPECT_EQ(f->get_parameters()[0]->get_partial_shape(), (PartialShape{5, 2, 2, 1}));
|
||||
EXPECT_EQ(f->get_parameters()[1]->get_partial_shape(), (PartialShape{5, 1, 1, 2}));
|
||||
|
||||
EXPECT_EQ(f->get_parameters()[0]->get_friendly_name(), "input1/Y");
|
||||
EXPECT_EQ(*f->get_parameters()[0]->output(0).get_tensor().get_names().begin(), "tensor_input1/Y");
|
||||
|
||||
EXPECT_EQ(f->get_parameters()[1]->get_friendly_name(), "input1/UV");
|
||||
EXPECT_EQ(*f->get_parameters()[1]->output(0).get_tensor().get_names().begin(), "tensor_input1/UV");
|
||||
}
|
||||
|
||||
TEST(pre_post_process, convert_color_nv12_bgr_2_planes_u8_lvalue) {
|
||||
auto f = create_simple_function(element::u8, Shape{1, 2, 2, 3});
|
||||
auto input_tensor_info = InputTensorInfo();
|
||||
input_tensor_info.set_color_format(ColorFormat::NV12_TWO_PLANES);
|
||||
auto steps = PreProcessSteps();
|
||||
steps.convert_color(ColorFormat::BGR);
|
||||
f = PrePostProcessor()
|
||||
.input(InputInfo().tensor(std::move(input_tensor_info)).preprocess(std::move(steps)))
|
||||
.build(f);
|
||||
|
||||
EXPECT_EQ(f->get_parameters().size(), 2);
|
||||
EXPECT_EQ(f->get_parameters()[0]->get_element_type(), element::u8);
|
||||
EXPECT_EQ(f->get_parameters()[0]->get_partial_shape(), (PartialShape{1, 2, 2, 1}));
|
||||
EXPECT_EQ(f->get_parameters()[1]->get_element_type(), element::u8);
|
||||
EXPECT_EQ(f->get_parameters()[1]->get_partial_shape(), (PartialShape{1, 1, 1, 2}));
|
||||
}
|
||||
|
||||
TEST(pre_post_process, convert_color_nv12_bgr_2_planes_el_type) {
|
||||
auto f = create_simple_function(element::u8, Shape{1, 2, 2, 3});
|
||||
EXPECT_NO_THROW(
|
||||
f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo()
|
||||
.set_element_type(element::f32)
|
||||
.set_color_format(ColorFormat::NV12_TWO_PLANES))
|
||||
.preprocess(
|
||||
PreProcessSteps().convert_element_type(element::u8).convert_color(ColorFormat::BGR)))
|
||||
.build(f));
|
||||
|
||||
EXPECT_EQ(f->get_parameters().size(), 2);
|
||||
EXPECT_EQ(f->get_parameters()[0]->get_element_type(), element::f32);
|
||||
EXPECT_EQ(f->get_parameters()[1]->get_element_type(), element::f32);
|
||||
}
|
||||
|
||||
TEST(pre_post_process, convert_color_same_type) {
|
||||
auto f = create_simple_function(element::u8, Shape{1, 2, 2, 3});
|
||||
EXPECT_NO_THROW(f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_color_format(ColorFormat::RGB))
|
||||
.preprocess(PreProcessSteps().convert_color(ColorFormat::RGB)))
|
||||
.build(f));
|
||||
|
||||
EXPECT_EQ(f->get_parameters().size(), 1);
|
||||
EXPECT_EQ(f->get_parameters()[0]->get_partial_shape(), (PartialShape{1, 2, 2, 3}));
|
||||
}
|
||||
|
||||
TEST(pre_post_process, convert_color_unsupported) {
|
||||
// Feel free to update this test when more color conversions are supported in future
|
||||
auto f = create_simple_function(element::f32, PartialShape{1, 4, 4, 3});
|
||||
EXPECT_THROW(f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_SINGLE_PLANE))
|
||||
.preprocess(PreProcessSteps().convert_color(ColorFormat::UNDEFINED)))
|
||||
.build(f),
|
||||
ov::AssertFailure);
|
||||
|
||||
EXPECT_THROW(f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES))
|
||||
.preprocess(PreProcessSteps().convert_color(ColorFormat::UNDEFINED)))
|
||||
.build(f),
|
||||
ov::AssertFailure);
|
||||
|
||||
auto colors = {ColorFormat::NV12_TWO_PLANES, ColorFormat::NV12_SINGLE_PLANE, ColorFormat::RGB, ColorFormat::BGR};
|
||||
for (const auto& color : colors) {
|
||||
EXPECT_THROW(f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_color_format(ColorFormat::UNDEFINED))
|
||||
.preprocess(PreProcessSteps().convert_color(color)))
|
||||
.build(f),
|
||||
ov::AssertFailure);
|
||||
|
||||
EXPECT_THROW(f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_color_format(color))
|
||||
.preprocess(PreProcessSteps().convert_color(ColorFormat::UNDEFINED)))
|
||||
.build(f),
|
||||
ov::AssertFailure);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(pre_post_process, convert_color_incorrect_subnames) {
|
||||
auto f = create_simple_function(element::f32, PartialShape{Dimension::dynamic(), 2, 2, 3});
|
||||
auto name = f->get_parameters()[0]->get_friendly_name();
|
||||
auto tensor_names = f->get_parameters().front()->get_output_tensor(0).get_names();
|
||||
EXPECT_THROW(
|
||||
f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_SINGLE_PLANE, {"Test"}))
|
||||
.preprocess(PreProcessSteps().convert_color(ColorFormat::RGB)))
|
||||
.build(f),
|
||||
ov::AssertFailure);
|
||||
|
||||
EXPECT_THROW(
|
||||
f = PrePostProcessor()
|
||||
.input(InputInfo().tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES, {"Test"})))
|
||||
.build(f),
|
||||
ov::AssertFailure);
|
||||
|
||||
EXPECT_THROW(f = PrePostProcessor()
|
||||
.input(InputInfo().tensor(
|
||||
InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES, {"1", "2", "3"})))
|
||||
.build(f),
|
||||
ov::AssertFailure);
|
||||
}
|
||||
|
||||
TEST(pre_post_process, convert_color_duplicate_subnames) {
|
||||
auto f = create_2inputs(element::f32, PartialShape{1, 2, 2, 3});
|
||||
f->get_parameters()[0]->get_output_tensor(0).set_names({"tensor_input1"});
|
||||
f->get_parameters()[1]->get_output_tensor(0).set_names({"tensor_input1/CustomUV"});
|
||||
EXPECT_THROW(f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_SINGLE_PLANE,
|
||||
{"CustomY", "CustomUV"}))
|
||||
.preprocess(PreProcessSteps().convert_color(ColorFormat::RGB)))
|
||||
.build(f),
|
||||
ov::AssertFailure);
|
||||
}
|
||||
|
||||
TEST(pre_post_process, convert_color_duplicate_internal_subnames_mean) {
|
||||
auto f = create_simple_function(element::f32, PartialShape{1, 2, 2, 3});
|
||||
for (int i = 0; i < 10; i++) {
|
||||
// Create preprocessing step several times (try to duplicate internal node names this way)
|
||||
EXPECT_NO_THROW(f = PrePostProcessor().input(InputInfo().preprocess(PreProcessSteps().mean(0.1f))).build(f));
|
||||
EXPECT_NO_THROW(f = PrePostProcessor().input(InputInfo().preprocess(PreProcessSteps().scale(1.1f))).build(f));
|
||||
EXPECT_NO_THROW(
|
||||
f = PrePostProcessor()
|
||||
.input(InputInfo().preprocess(
|
||||
PreProcessSteps().convert_element_type(element::u8).convert_element_type(element::f32)))
|
||||
.build(f));
|
||||
EXPECT_NO_THROW(f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_layout("NHWC"))
|
||||
.preprocess(PreProcessSteps().convert_layout("NCHW")))
|
||||
.build(f));
|
||||
EXPECT_NO_THROW(
|
||||
f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_layout("NHWC").set_spatial_static_shape(480, 640))
|
||||
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_LINEAR)))
|
||||
.build(f));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(pre_post_process, unsupported_network_color_format) {
|
||||
auto f = create_simple_function(element::f32, PartialShape{1, 4, 4, 3});
|
||||
EXPECT_THROW(f = PrePostProcessor()
|
||||
.input(InputInfo().tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_SINGLE_PLANE)))
|
||||
.build(f),
|
||||
ov::AssertFailure);
|
||||
|
||||
EXPECT_THROW(f = PrePostProcessor()
|
||||
.input(InputInfo().tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES)))
|
||||
.build(f),
|
||||
ov::AssertFailure);
|
||||
|
||||
EXPECT_THROW(
|
||||
f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES))
|
||||
.preprocess(PreProcessSteps().convert_layout("NCHW").convert_color(ColorFormat::RGB)))
|
||||
.build(f),
|
||||
ov::AssertFailure);
|
||||
|
||||
EXPECT_THROW(f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES))
|
||||
.preprocess(PreProcessSteps().mean(0.1f).convert_color(ColorFormat::RGB)))
|
||||
.build(f),
|
||||
ov::AssertFailure);
|
||||
|
||||
EXPECT_THROW(f = PrePostProcessor()
|
||||
.input(InputInfo()
|
||||
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES))
|
||||
.preprocess(PreProcessSteps().scale(2.1f).convert_color(ColorFormat::RGB)))
|
||||
.build(f),
|
||||
ov::AssertFailure);
|
||||
}
|
||||
|
||||
TEST(pre_post_process, custom_preprocessing) {
|
||||
auto f = create_simple_function(element::i32, Shape{1, 3, 1, 1});
|
||||
f = PrePostProcessor()
|
||||
@ -327,3 +593,34 @@ TEST(pre_post_process, resize_no_tensor_width) {
|
||||
.build(f),
|
||||
ov::AssertFailure);
|
||||
}
|
||||
|
||||
TEST(pre_post_process, exception_safety) {
|
||||
auto f = create_2inputs(element::f32, Shape{1, 3, 224, 224});
|
||||
auto name0 = f->get_parameters()[0]->get_friendly_name();
|
||||
auto tensor_names0 = f->get_parameters()[0]->get_output_tensor(0).get_names();
|
||||
auto name1 = f->get_parameters()[1]->get_friendly_name();
|
||||
auto tensor_names1 = f->get_parameters()[1]->get_output_tensor(0).get_names();
|
||||
EXPECT_THROW(f = PrePostProcessor()
|
||||
.input(InputInfo(0) // this one is correct
|
||||
.tensor(InputTensorInfo().set_element_type(element::u8))
|
||||
.preprocess(PreProcessSteps().convert_element_type(element::f32)))
|
||||
.input(InputInfo(1) // This one is not
|
||||
.tensor(InputTensorInfo().set_color_format(ColorFormat::NV12_TWO_PLANES))
|
||||
.preprocess(PreProcessSteps().custom(
|
||||
[](const std::shared_ptr<Node>& node) -> std::shared_ptr<Node> {
|
||||
throw ngraph::ngraph_error("test error");
|
||||
})))
|
||||
.build(f),
|
||||
ov::AssertFailure);
|
||||
EXPECT_EQ(f->get_parameters().size(), 2);
|
||||
|
||||
EXPECT_EQ(f->get_parameters()[0]->get_element_type(), element::f32);
|
||||
EXPECT_EQ(f->get_parameters()[0]->get_partial_shape(), (PartialShape{1, 3, 224, 224}));
|
||||
EXPECT_EQ(f->get_parameters()[0]->get_friendly_name(), name0);
|
||||
EXPECT_EQ(f->get_parameters()[0]->get_output_tensor(0).get_names(), tensor_names0);
|
||||
|
||||
EXPECT_EQ(f->get_parameters()[1]->get_element_type(), element::f32);
|
||||
EXPECT_EQ(f->get_parameters()[1]->get_partial_shape(), (PartialShape{1, 3, 224, 224}));
|
||||
EXPECT_EQ(f->get_parameters()[1]->get_friendly_name(), name1);
|
||||
EXPECT_EQ(f->get_parameters()[1]->get_output_tensor(0).get_names(), tensor_names1);
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user