diff --git a/docs/template_plugin/tests/functional/subgraph_reference/preprocess.cpp b/docs/template_plugin/tests/functional/subgraph_reference/preprocess.cpp index 157ea18cd08..ccca4f66bd1 100644 --- a/docs/template_plugin/tests/functional/subgraph_reference/preprocess.cpp +++ b/docs/template_plugin/tests/functional/subgraph_reference/preprocess.cpp @@ -419,6 +419,91 @@ static RefPreprocessParams resize_lvalues() { return res; } +static RefPreprocessParams convert_layout_nhwc_to_nchw_lvalue() { + RefPreprocessParams res("convert_layout_nhwc_to_nchw_lvalue"); + res.function = []() { + auto f = create_simple_function(element::u8, {1, 3, 2, 2}); + f->get_parameters()[0]->set_layout("NCHW"); + auto p = PreProcessSteps(); + p.convert_layout("NCHW"); + + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_layout("NHWC")) + .preprocess(std::move(p))) + .build(f); + return f; + }; + res.inputs.emplace_back(Shape{1, 2, 2, 3}, element::u8, std::vector{1, 2, 3, // [H=0, W=0, RGB] + 4, 5, 6, // [H=0, W=1] + 7, 8, 9, // [H=1, W=0] + 10, 11, 12}); // [H=1, W=1] + res.expected.emplace_back(Shape{1, 3, 2, 2}, element::u8, std::vector{1, 4, 7, 10, // R + 2, 5, 8, 11, // G + 3, 6, 9, 12}); // B + return res; +} + +static RefPreprocessParams convert_layout_nhwc_to_net_no_tensor_shape() { + RefPreprocessParams res("convert_layout_nhwc_to_net_no_tensor_shape"); + res.function = []() { + auto f = create_simple_function(element::u8, {1, 3, 2, 2}); + f->get_parameters()[0]->set_layout("NCHW"); + auto p = PreProcessSteps(); + p.convert_layout(); + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo().set_layout("NHWC")) + .preprocess(std::move(p))) + .build(f); + return f; + }; + res.inputs.emplace_back(Shape{1, 2, 2, 3}, element::u8, std::vector{1, 2, 3, // [H=0, W=0, RGB] + 4, 5, 6, // [H=0, W=1] + 7, 8, 9, // [H=1, W=0] + 10, 11, 12}); // [H=1, W=1] + res.expected.emplace_back(Shape{1, 3, 2, 2}, element::u8, std::vector{1, 4, 7, 10, // R + 2, 5, 8, 11, // G + 3, 6, 9, 12}); // B + return res; +} + +static RefPreprocessParams resize_and_convert_layout() { + RefPreprocessParams res("resize_and_convert_layout"); + res.function = []() { + auto f = create_simple_function(element::f32, PartialShape{1, 2, 2, 2}); + f = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo() + .set_layout("NCHW") + .set_spatial_dynamic_shape()) + .preprocess(PreProcessSteps() + .resize(ResizeAlgorithm::RESIZE_LINEAR) + .convert_layout()) + .network(InputNetworkInfo().set_layout("NHWC"))) + .build(f); + return f; + }; + + auto result = std::make_shared(); + // clang-format off + std::vector input = { + 1., 1., 1., 1., // channel 1 + 1., 1., 1., 1., + 1., 1., 1., 1., + 1., 1., 1., 1., + 2., 2., 2., 2., // channel 2 + 2., 2., 2., 2., + 2., 2., 2., 2., + 2., 2., 2., 2., + }; + std::vector expected = {1., 2., 1., 2., 1., 2., 1., 2.}; + // clang-format on + res.inputs.emplace_back(element::f32, Shape{1, 2, 4, 4}, input); + res.expected.emplace_back(Shape{1, 2, 2, 2}, element::f32, expected); + return res; +} + std::vector allPreprocessTests() { return std::vector { @@ -437,7 +522,10 @@ std::vector allPreprocessTests() { resize_from_spatial_dims(), resize_to_network_width_height(), resize_to_specified_width_height(), - resize_lvalues() + resize_lvalues(), + convert_layout_nhwc_to_nchw_lvalue(), + convert_layout_nhwc_to_net_no_tensor_shape(), + resize_and_convert_layout() }; } diff --git a/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/preprocess/preprocess_builders.hpp b/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/preprocess/preprocess_builders.hpp index 82a44a81e74..0ec6abc7c5c 100644 --- a/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/preprocess/preprocess_builders.hpp +++ b/inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/preprocess/preprocess_builders.hpp @@ -254,6 +254,22 @@ inline std::shared_ptr resize_cubic() { return function; } +inline std::shared_ptr resize_and_convert_layout() { + using namespace ov::preprocess; + auto function = create_preprocess_1input(element::f32, PartialShape{1, 30, 20, 3}); + function = PrePostProcessor() + .input(InputInfo() + .tensor(InputTensorInfo() + .set_layout("NHWC") + .set_spatial_static_shape(40, 30)) + .preprocess(PreProcessSteps() + .convert_layout() + .resize(ResizeAlgorithm::RESIZE_LINEAR)) + .network(InputNetworkInfo().set_layout("NCHW"))) + .build(function); + return function; +} + inline std::vector generic_preprocess_functions() { return std::vector { preprocess_func(mean_only, "mean_only", 0.01f), @@ -273,6 +289,7 @@ inline std::vector generic_preprocess_functions() { preprocess_func(resize_nearest, "resize_nearest", 0.01f), preprocess_func(resize_linear_nhwc, "resize_linear_nhwc", 0.01f), preprocess_func(resize_cubic, "resize_cubic", 0.01f), + preprocess_func(resize_and_convert_layout, "resize_and_convert_layout", 0.01f), }; } diff --git a/ngraph/core/include/openvino/core/layout.hpp b/ngraph/core/include/openvino/core/layout.hpp index 16c3b72dfd6..873d0c953d1 100644 --- a/ngraph/core/include/openvino/core/layout.hpp +++ b/ngraph/core/include/openvino/core/layout.hpp @@ -9,11 +9,20 @@ #include "ngraph/attribute_adapter.hpp" #include "openvino/core/core_visibility.hpp" +#include "openvino/core/partial_shape.hpp" #include "openvino/core/rank.hpp" #include "openvino/core/variant.hpp" namespace ov { +class Layout; + +namespace layout { + +std::vector find_permutation(const Layout& src_layout, const Rank& src_shape_rank, const Layout& dst_layout); + +} + class OPENVINO_API Layout { public: /// \brief Constructs a dynamic Layout with no layout information. @@ -59,6 +68,10 @@ public: /// \brief String representation of Layout std::string to_string() const; + bool empty() const { + return *this == Layout(); + } + private: /// stores dimension names map to index in a layout std::unordered_map m_names; @@ -70,6 +83,10 @@ private: bool m_dynamic = false; int64_t m_left_size = 0; int64_t m_right_size = 0; + + friend std::vector layout::find_permutation(const Layout& src_layout, + const Rank& src_shape_rank, + const Layout& dst_layout); }; namespace layout { diff --git a/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp b/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp index e57755e45bc..8ef191dba00 100644 --- a/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp +++ b/ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp @@ -176,6 +176,37 @@ public: /// /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner. PreProcessSteps&& resize(ResizeAlgorithm alg) &&; + + /// \brief Add 'convert layout' operation to specified layout - Lvalue version. + /// + /// \details Adds appropriate 'transpose' operation between user layout and target layout. + /// Current implementation requires source and destination layout to have same number of dimensions + /// + /// \example Example: when user data has 'NHWC' layout (example is RGB image, [1, 224, 224, 3]) but network expects + /// planar input image ('NCHW', [1, 3, 224, 224]). Preprocessing may look like this: + /// + /// \code{.cpp} auto proc = + /// PrePostProcessor() + /// .input(InputInfo() + /// .tensor(InputTensorInfo().set_layout("NHWC")) // User data is NHWC + /// .preprocess(PreProcessSteps() + /// .convert_layout("NCHW")) // Network expects input as NCHW + /// ); + /// \endcode + /// + /// \param dst_layout New layout after conversion. If not specified - destination layout is obtained from + /// appropriate network input properties. + /// + /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner. + PreProcessSteps& convert_layout(const Layout& dst_layout = {}) &; + + /// \brief Add resize operation to network dimensions - Rvalue version. + /// + /// \param dst_layout New layout after conversion. If not specified - destination layout is obtained from + /// appropriate network input properties. + /// + /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner. + PreProcessSteps&& convert_layout(const Layout& dst_layout = {}) &&; }; } // namespace preprocess diff --git a/ngraph/core/src/layout.cpp b/ngraph/core/src/layout.cpp index 3e0b8017562..0c23fdfd099 100644 --- a/ngraph/core/src/layout.cpp +++ b/ngraph/core/src/layout.cpp @@ -233,6 +233,33 @@ std::string Layout::to_string() const { return res.str(); } +namespace layout { + +std::vector find_permutation(const Layout& src_layout, const Rank& rank, const Layout& dst) { + // Basic implementation so far, can support partially-specified layouts later (shape rank will be needed for dynamic + // layouts) + OPENVINO_ASSERT(!src_layout.m_dynamic && !dst.m_dynamic, "Conversion is not supported for dynamic layouts"); + OPENVINO_ASSERT(src_layout.m_left_size == src_layout.m_left_size, + "Conversion is not supported for layouts with different sizes"); + std::vector res(src_layout.m_left_size); + for (int64_t i = 0; i < src_layout.m_left_size; i++) { + auto it = src_layout.m_index_map.find(i); + OPENVINO_ASSERT(it != src_layout.m_index_map.end(), + "Conversion is not supported for partially specified source layout: ", + src_layout.to_string()); + auto name = it->second; + OPENVINO_ASSERT(dst.has_name(name), + "Source dimension name '", + name, + "' is not found in destination layout: ", + dst.to_string()); + res[dst.get_index_by_name(name)] = i; + } + return res; +} + +} // namespace layout + #define DEFINE_NAMED_DIMENSION(NAME, name) \ bool layout::has_##name(const Layout& layout) { \ return layout.has_name(NAME); \ diff --git a/ngraph/core/src/preprocess/pre_post_process.cpp b/ngraph/core/src/preprocess/pre_post_process.cpp index c1011d733a8..0445bc90bfd 100644 --- a/ngraph/core/src/preprocess/pre_post_process.cpp +++ b/ngraph/core/src/preprocess/pre_post_process.cpp @@ -197,7 +197,7 @@ std::shared_ptr PrePostProcessor::build(const std::shared_ptrget_parameters().front(); } // Set parameter layout from 'network' information - if (input->m_network_data && input->m_network_data->is_layout_set() && param->get_layout() == Layout()) { + if (input->m_network_data && input->m_network_data->is_layout_set() && param->get_layout().empty()) { param->set_layout(input->m_network_data->get_layout()); } auto consumers = param->output(0).get_target_inputs(); @@ -210,7 +210,19 @@ std::shared_ptr PrePostProcessor::build(const std::shared_ptrm_tensor_data->is_element_type_set()) { input->m_tensor_data->set_element_type(param->get_element_type()); } - auto new_param_shape = param->get_partial_shape(); + auto net_shape = param->get_partial_shape(); + auto new_param_shape = net_shape; + if (input->m_tensor_data->is_layout_set() && param->get_layout() != Layout() && + param->get_layout() != input->m_tensor_data->get_layout()) { + // Find transpose between network and tensor layouts and update tensor shape + auto net_to_tensor = + layout::find_permutation(param->get_layout(), net_shape.rank(), input->m_tensor_data->get_layout()); + std::vector dims(new_param_shape.size()); + std::transform(net_to_tensor.begin(), net_to_tensor.end(), dims.begin(), [&](int64_t v) { + return new_param_shape[v]; + }); + new_param_shape = PartialShape(dims); + } if (input->m_tensor_data->is_spatial_shape_set()) { auto height_idx = get_and_check_height_idx(input->m_tensor_data->get_layout(), new_param_shape); auto width_idx = get_and_check_width_idx(input->m_tensor_data->get_layout(), new_param_shape); @@ -410,6 +422,16 @@ PreProcessSteps&& PreProcessSteps::resize(ResizeAlgorithm alg) && { return std::move(*this); } +PreProcessSteps& PreProcessSteps::convert_layout(const Layout& dst_layout) & { + m_impl->add_convert_layout_impl(dst_layout); + return *this; +} + +PreProcessSteps&& PreProcessSteps::convert_layout(const Layout& dst_layout) && { + m_impl->add_convert_layout_impl(dst_layout); + return std::move(*this); +} + PreProcessSteps& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb) & { // 'true' indicates that custom preprocessing step will trigger validate_and_infer_types m_impl->actions().emplace_back(std::make_tuple( diff --git a/ngraph/core/src/preprocess/preprocess_steps_impl.cpp b/ngraph/core/src/preprocess/preprocess_steps_impl.cpp index 3c47aae90d9..93c3b22df32 100644 --- a/ngraph/core/src/preprocess/preprocess_steps_impl.cpp +++ b/ngraph/core/src/preprocess/preprocess_steps_impl.cpp @@ -146,5 +146,25 @@ void PreProcessSteps::PreProcessStepsImpl::add_resize_impl(ResizeAlgorithm alg, true)); } +void PreProcessSteps::PreProcessStepsImpl::add_convert_layout_impl(const Layout& layout) { + m_actions.emplace_back(std::make_tuple( + [layout](const std::vector>& nodes, PreprocessingContext& context) { + OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't convert layout for empty input."); + OPENVINO_ASSERT(nodes.size() == 1, + "Can't convert layout for multi-plane input. Suggesting to convert current image to " + "RGB/BGR color format using 'convert_color'"); + Layout dst_layout = layout.empty() ? context.network_layout() : layout; + auto permutation = + layout::find_permutation(context.layout(), nodes[0]->get_output_partial_shape(0).rank(), dst_layout); + auto perm_constant = + op::v0::Constant::create(element::i64, Shape{permutation.size()}, permutation); + auto transpose = std::make_shared(nodes[0], perm_constant); + transpose->set_friendly_name(nodes[0]->get_friendly_name() + "/convert_layout"); + context.layout() = dst_layout; // Update context's current layout + return transpose; + }, + true)); +} + } // namespace preprocess } // namespace ov diff --git a/ngraph/core/src/preprocess/preprocess_steps_impl.hpp b/ngraph/core/src/preprocess/preprocess_steps_impl.hpp index a6dce4e90e7..120c6849fd9 100644 --- a/ngraph/core/src/preprocess/preprocess_steps_impl.hpp +++ b/ngraph/core/src/preprocess/preprocess_steps_impl.hpp @@ -116,6 +116,7 @@ public: void add_mean_impl(const std::vector& values); void add_convert_impl(const element::Type& type); void add_resize_impl(ResizeAlgorithm alg, int dst_height, int dst_width); + void add_convert_layout_impl(const Layout& layout); const std::list>& actions() const { return m_actions;