[OV2.0] Preprocessing - add 'convert_layout' (#7772)

* Preprocessing - add 'convert_layout'

* Fix comments, removed 'set_data_shape' for time being

docs/template_plugin/tests/functional/subgraph_reference/preprocess.cpp

@@ -419,6 +419,91 @@ static RefPreprocessParams resize_lvalues() {
     return res;
 }
 
+static RefPreprocessParams convert_layout_nhwc_to_nchw_lvalue() {
+    RefPreprocessParams res("convert_layout_nhwc_to_nchw_lvalue");
+    res.function = []() {
+        auto f = create_simple_function(element::u8, {1, 3, 2, 2});
+        f->get_parameters()[0]->set_layout("NCHW");
+        auto p = PreProcessSteps();
+        p.convert_layout("NCHW");
+
+        f = PrePostProcessor()
+                .input(InputInfo()
+                               .tensor(InputTensorInfo().set_layout("NHWC"))
+                               .preprocess(std::move(p)))
+                .build(f);
+        return f;
+    };
+    res.inputs.emplace_back(Shape{1, 2, 2, 3}, element::u8, std::vector<uint8_t>{1,  2,  3,       // [H=0, W=0, RGB]
+                                                                                 4,  5,  6,       // [H=0, W=1]
+                                                                                 7,  8,  9,       // [H=1, W=0]
+                                                                                 10, 11, 12});    // [H=1, W=1]
+    res.expected.emplace_back(Shape{1, 3, 2, 2}, element::u8, std::vector<uint8_t>{1, 4, 7, 10,    // R
+                                                                                   2, 5, 8, 11,    // G
+                                                                                   3, 6, 9, 12});  // B
+    return res;
+}
+
+static RefPreprocessParams convert_layout_nhwc_to_net_no_tensor_shape() {
+    RefPreprocessParams res("convert_layout_nhwc_to_net_no_tensor_shape");
+    res.function = []() {
+        auto f = create_simple_function(element::u8, {1, 3, 2, 2});
+        f->get_parameters()[0]->set_layout("NCHW");
+        auto p = PreProcessSteps();
+        p.convert_layout();
+        f = PrePostProcessor()
+                .input(InputInfo()
+                               .tensor(InputTensorInfo().set_layout("NHWC"))
+                               .preprocess(std::move(p)))
+                .build(f);
+        return f;
+    };
+    res.inputs.emplace_back(Shape{1, 2, 2, 3}, element::u8, std::vector<uint8_t>{1,  2,  3,       // [H=0, W=0, RGB]
+                                                                                 4,  5,  6,       // [H=0, W=1]
+                                                                                 7,  8,  9,       // [H=1, W=0]
+                                                                                 10, 11, 12});    // [H=1, W=1]
+    res.expected.emplace_back(Shape{1, 3, 2, 2}, element::u8, std::vector<uint8_t>{1, 4, 7, 10,    // R
+                                                                                   2, 5, 8, 11,    // G
+                                                                                   3, 6, 9, 12});  // B
+    return res;
+}
+
+static RefPreprocessParams resize_and_convert_layout() {
+    RefPreprocessParams res("resize_and_convert_layout");
+    res.function = []() {
+        auto f = create_simple_function(element::f32, PartialShape{1, 2, 2, 2});
+        f = PrePostProcessor()
+                .input(InputInfo()
+                               .tensor(InputTensorInfo()
+                                               .set_layout("NCHW")
+                                               .set_spatial_dynamic_shape())
+                               .preprocess(PreProcessSteps()
+                                                   .resize(ResizeAlgorithm::RESIZE_LINEAR)
+                                                   .convert_layout())
+                               .network(InputNetworkInfo().set_layout("NHWC")))
+                .build(f);
+        return f;
+    };
+
+    auto result = std::make_shared<HostTensor>();
+    // clang-format off
+    std::vector<float> input = {
+            1., 1., 1., 1., // channel 1
+            1., 1., 1., 1.,
+            1., 1., 1., 1.,
+            1., 1., 1., 1.,
+            2., 2., 2., 2., // channel 2
+            2., 2., 2., 2.,
+            2., 2., 2., 2.,
+            2., 2., 2., 2.,
+    };
+    std::vector<float> expected = {1., 2., 1., 2., 1., 2., 1., 2.};
+    // clang-format on
+    res.inputs.emplace_back(element::f32, Shape{1, 2, 4, 4}, input);
+    res.expected.emplace_back(Shape{1, 2, 2, 2}, element::f32, expected);
+    return res;
+}
+
 
 std::vector<RefPreprocessParams> allPreprocessTests() {
     return std::vector<RefPreprocessParams> {
@@ -437,7 +522,10 @@ std::vector<RefPreprocessParams> allPreprocessTests() {
         resize_from_spatial_dims(),
         resize_to_network_width_height(),
         resize_to_specified_width_height(),
-        resize_lvalues()
+        resize_lvalues(),
+        convert_layout_nhwc_to_nchw_lvalue(),
+        convert_layout_nhwc_to_net_no_tensor_shape(),
+        resize_and_convert_layout()
              };
 }
 

inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/preprocess/preprocess_builders.hpp

@@ -254,6 +254,22 @@ inline std::shared_ptr<Function> resize_cubic() {
     return function;
 }
 
+inline std::shared_ptr<Function> resize_and_convert_layout() {
+    using namespace ov::preprocess;
+    auto function = create_preprocess_1input(element::f32, PartialShape{1, 30, 20, 3});
+    function = PrePostProcessor()
+            .input(InputInfo()
+                           .tensor(InputTensorInfo()
+                                           .set_layout("NHWC")
+                                           .set_spatial_static_shape(40, 30))
+                           .preprocess(PreProcessSteps()
+                                               .convert_layout()
+                                               .resize(ResizeAlgorithm::RESIZE_LINEAR))
+                           .network(InputNetworkInfo().set_layout("NCHW")))
+            .build(function);
+    return function;
+}
+
 inline std::vector<preprocess_func> generic_preprocess_functions() {
     return std::vector<preprocess_func> {
             preprocess_func(mean_only, "mean_only", 0.01f),
@@ -273,6 +289,7 @@ inline std::vector<preprocess_func> generic_preprocess_functions() {
             preprocess_func(resize_nearest, "resize_nearest", 0.01f),
             preprocess_func(resize_linear_nhwc, "resize_linear_nhwc", 0.01f),
             preprocess_func(resize_cubic, "resize_cubic", 0.01f),
+            preprocess_func(resize_and_convert_layout, "resize_and_convert_layout", 0.01f),
     };
 }
 

ngraph/core/include/openvino/core/layout.hpp

@@ -9,11 +9,20 @@
 
 #include "ngraph/attribute_adapter.hpp"
 #include "openvino/core/core_visibility.hpp"
+#include "openvino/core/partial_shape.hpp"
 #include "openvino/core/rank.hpp"
 #include "openvino/core/variant.hpp"
 
 namespace ov {
 
+class Layout;
+
+namespace layout {
+
+std::vector<int64_t> find_permutation(const Layout& src_layout, const Rank& src_shape_rank, const Layout& dst_layout);
+
+}
+
 class OPENVINO_API Layout {
 public:
     /// \brief Constructs a dynamic Layout with no layout information.
@@ -59,6 +68,10 @@ public:
     /// \brief String representation of Layout
     std::string to_string() const;
 
+    bool empty() const {
+        return *this == Layout();
+    }
+
 private:
     /// stores dimension names map to index in a layout
     std::unordered_map<std::string, std::int64_t> m_names;
@@ -70,6 +83,10 @@ private:
     bool m_dynamic = false;
     int64_t m_left_size = 0;
     int64_t m_right_size = 0;
+
+    friend std::vector<int64_t> layout::find_permutation(const Layout& src_layout,
+                                                         const Rank& src_shape_rank,
+                                                         const Layout& dst_layout);
 };
 
 namespace layout {

ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp

@@ -176,6 +176,37 @@ public:
     ///
     /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner.
     PreProcessSteps&& resize(ResizeAlgorithm alg) &&;
+
+    /// \brief Add 'convert layout' operation to specified layout - Lvalue version.
+    ///
+    /// \details Adds appropriate 'transpose' operation between user layout and target layout.
+    /// Current implementation requires source and destination layout to have same number of dimensions
+    ///
+    /// \example Example: when user data has 'NHWC' layout (example is RGB image, [1, 224, 224, 3]) but network expects
+    /// planar input image ('NCHW', [1, 3, 224, 224]). Preprocessing may look like this:
+    ///
+    /// \code{.cpp} auto proc =
+    /// PrePostProcessor()
+    ///     .input(InputInfo()
+    ///            .tensor(InputTensorInfo().set_layout("NHWC")) // User data is NHWC
+    ///            .preprocess(PreProcessSteps()
+    ///                        .convert_layout("NCHW")) // Network expects input as NCHW
+    ///     );
+    /// \endcode
+    ///
+    /// \param dst_layout New layout after conversion. If not specified - destination layout is obtained from
+    /// appropriate network input properties.
+    ///
+    /// \return Reference to 'this' to allow chaining with other calls in a builder-like manner.
+    PreProcessSteps& convert_layout(const Layout& dst_layout = {}) &;
+
+    /// \brief Add resize operation to network dimensions - Rvalue version.
+    ///
+    /// \param dst_layout New layout after conversion. If not specified - destination layout is obtained from
+    /// appropriate network input properties.
+    ///
+    /// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner.
+    PreProcessSteps&& convert_layout(const Layout& dst_layout = {}) &&;
 };
 
 }  // namespace preprocess

ngraph/core/src/layout.cpp

@@ -233,6 +233,33 @@ std::string Layout::to_string() const {
     return res.str();
 }
 
+namespace layout {
+
+std::vector<int64_t> find_permutation(const Layout& src_layout, const Rank& rank, const Layout& dst) {
+    // Basic implementation so far, can support partially-specified layouts later (shape rank will be needed for dynamic
+    // layouts)
+    OPENVINO_ASSERT(!src_layout.m_dynamic && !dst.m_dynamic, "Conversion is not supported for dynamic layouts");
+    OPENVINO_ASSERT(src_layout.m_left_size == src_layout.m_left_size,
+                    "Conversion is not supported for layouts with different sizes");
+    std::vector<int64_t> res(src_layout.m_left_size);
+    for (int64_t i = 0; i < src_layout.m_left_size; i++) {
+        auto it = src_layout.m_index_map.find(i);
+        OPENVINO_ASSERT(it != src_layout.m_index_map.end(),
+                        "Conversion is not supported for partially specified source layout: ",
+                        src_layout.to_string());
+        auto name = it->second;
+        OPENVINO_ASSERT(dst.has_name(name),
+                        "Source dimension name '",
+                        name,
+                        "' is not found in destination layout: ",
+                        dst.to_string());
+        res[dst.get_index_by_name(name)] = i;
+    }
+    return res;
+}
+
+}  // namespace layout
+
 #define DEFINE_NAMED_DIMENSION(NAME, name)            \
     bool layout::has_##name(const Layout& layout) {   \
         return layout.has_name(NAME);                 \

ngraph/core/src/preprocess/pre_post_process.cpp

@@ -197,7 +197,7 @@ std::shared_ptr<Function> PrePostProcessor::build(const std::shared_ptr<Function
             param = function->get_parameters().front();
         }
         // Set parameter layout from 'network' information
-        if (input->m_network_data && input->m_network_data->is_layout_set() && param->get_layout() == Layout()) {
+        if (input->m_network_data && input->m_network_data->is_layout_set() && param->get_layout().empty()) {
             param->set_layout(input->m_network_data->get_layout());
         }
         auto consumers = param->output(0).get_target_inputs();
@@ -210,7 +210,19 @@ std::shared_ptr<Function> PrePostProcessor::build(const std::shared_ptr<Function
         if (!input->m_tensor_data->is_element_type_set()) {
             input->m_tensor_data->set_element_type(param->get_element_type());
         }
-        auto new_param_shape = param->get_partial_shape();
+        auto net_shape = param->get_partial_shape();
+        auto new_param_shape = net_shape;
+        if (input->m_tensor_data->is_layout_set() && param->get_layout() != Layout() &&
+            param->get_layout() != input->m_tensor_data->get_layout()) {
+            // Find transpose between network and tensor layouts and update tensor shape
+            auto net_to_tensor =
+                layout::find_permutation(param->get_layout(), net_shape.rank(), input->m_tensor_data->get_layout());
+            std::vector<ov::Dimension> dims(new_param_shape.size());
+            std::transform(net_to_tensor.begin(), net_to_tensor.end(), dims.begin(), [&](int64_t v) {
+                return new_param_shape[v];
+            });
+            new_param_shape = PartialShape(dims);
+        }
         if (input->m_tensor_data->is_spatial_shape_set()) {
             auto height_idx = get_and_check_height_idx(input->m_tensor_data->get_layout(), new_param_shape);
             auto width_idx = get_and_check_width_idx(input->m_tensor_data->get_layout(), new_param_shape);
@@ -410,6 +422,16 @@ PreProcessSteps&& PreProcessSteps::resize(ResizeAlgorithm alg) && {
     return std::move(*this);
 }
 
+PreProcessSteps& PreProcessSteps::convert_layout(const Layout& dst_layout) & {
+    m_impl->add_convert_layout_impl(dst_layout);
+    return *this;
+}
+
+PreProcessSteps&& PreProcessSteps::convert_layout(const Layout& dst_layout) && {
+    m_impl->add_convert_layout_impl(dst_layout);
+    return std::move(*this);
+}
+
 PreProcessSteps& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb) & {
     // 'true' indicates that custom preprocessing step will trigger validate_and_infer_types
     m_impl->actions().emplace_back(std::make_tuple(

ngraph/core/src/preprocess/preprocess_steps_impl.cpp

@@ -146,5 +146,25 @@ void PreProcessSteps::PreProcessStepsImpl::add_resize_impl(ResizeAlgorithm alg,
         true));
 }
 
+void PreProcessSteps::PreProcessStepsImpl::add_convert_layout_impl(const Layout& layout) {
+    m_actions.emplace_back(std::make_tuple(
+        [layout](const std::vector<std::shared_ptr<Node>>& nodes, PreprocessingContext& context) {
+            OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't convert layout for empty input.");
+            OPENVINO_ASSERT(nodes.size() == 1,
+                            "Can't convert layout for multi-plane input. Suggesting to convert current image to "
+                            "RGB/BGR color format using 'convert_color'");
+            Layout dst_layout = layout.empty() ? context.network_layout() : layout;
+            auto permutation =
+                layout::find_permutation(context.layout(), nodes[0]->get_output_partial_shape(0).rank(), dst_layout);
+            auto perm_constant =
+                op::v0::Constant::create<int64_t>(element::i64, Shape{permutation.size()}, permutation);
+            auto transpose = std::make_shared<op::v1::Transpose>(nodes[0], perm_constant);
+            transpose->set_friendly_name(nodes[0]->get_friendly_name() + "/convert_layout");
+            context.layout() = dst_layout;  // Update context's current layout
+            return transpose;
+        },
+        true));
+}
+
 }  // namespace preprocess
 }  // namespace ov

ngraph/core/src/preprocess/preprocess_steps_impl.hpp

@@ -116,6 +116,7 @@ public:
     void add_mean_impl(const std::vector<float>& values);
     void add_convert_impl(const element::Type& type);
     void add_resize_impl(ResizeAlgorithm alg, int dst_height, int dst_width);
+    void add_convert_layout_impl(const Layout& layout);
 
     const std::list<std::tuple<InternalPreprocessOp, bool>>& actions() const {
         return m_actions;