diff --git a/src/bindings/python/tests/test_graph/test_preprocess.py b/src/bindings/python/tests/test_graph/test_preprocess.py index 2432a31f023..34a1d15d610 100644 --- a/src/bindings/python/tests/test_graph/test_preprocess.py +++ b/src/bindings/python/tests/test_graph/test_preprocess.py @@ -393,7 +393,7 @@ def test_graph_preprocess_steps(algorithm, color_format1, color_format2, is_fail "Gather", "Interpolate", ] - assert len(model_operators) == 16 + assert len(model_operators) == 15 assert function.get_output_size() == 1 assert list(function.get_output_shape(0)) == [1, 3, 3, 3] assert function.get_output_element_type(0) == Type.f32 @@ -535,7 +535,7 @@ def test_graph_preprocess_resize_algorithm(): "Subtract", "Interpolate", ] - assert len(model_operators) == 8 + assert len(model_operators) == 7 assert function.get_output_size() == 1 assert list(function.get_output_shape(0)) == [1, 1, 3, 3] assert function.get_output_element_type(0) == Type.f32 diff --git a/src/common/transformations/src/transformations/op_conversions/convert_interpolate11_downgrade.cpp b/src/common/transformations/src/transformations/op_conversions/convert_interpolate11_downgrade.cpp index c9b2e15dd4c..8d3a288d1f9 100644 --- a/src/common/transformations/src/transformations/op_conversions/convert_interpolate11_downgrade.cpp +++ b/src/common/transformations/src/transformations/op_conversions/convert_interpolate11_downgrade.cpp @@ -11,6 +11,43 @@ #include #include "itt.hpp" +#include "utils.hpp" + +namespace { +// v4_sizes, v4_scales +std::pair, ov::Output> make_v4_inputs( + const std::shared_ptr& interpolate) { + ov::pass::NodeRegistry registry; + std::pair, ov::Output> ret; + std::shared_ptr broadcast_shape; + + if (interpolate->get_input_size() == 3) { + // broadcast dummy constant to the shape of axes + broadcast_shape = registry.make(interpolate->input_value(2)); + } else { + // broadcast dummy constant to the rank of data + broadcast_shape = registry.make(interpolate->input_value(0)); + broadcast_shape = registry.make(broadcast_shape); + } + + if (interpolate->get_attrs().shape_calculation_mode == ov::op::util::InterpolateBase::ShapeCalcMode::SCALES) { + ret.second = interpolate->input_value(1); + std::shared_ptr sizes_input = registry.make(ov::element::i32, ov::Shape{}, 1); + sizes_input = registry.make(sizes_input, broadcast_shape); + ret.first = sizes_input; + } else { + ret.first = interpolate->input_value(1); + std::shared_ptr scales_input = + registry.make(ov::element::f32, ov::Shape{}, 1.0f); + scales_input = registry.make(scales_input, broadcast_shape); + ret.second = scales_input; + } + + copy_runtime_info(interpolate, registry.get()); + + return ret; +} +} // namespace ov::pass::ConvertInterpolate11ToInterpolate4::ConvertInterpolate11ToInterpolate4() { MATCHER_SCOPE(ConvertInterpolate11ToInterpolate4); @@ -38,17 +75,7 @@ ov::pass::ConvertInterpolate11ToInterpolate4::ConvertInterpolate11ToInterpolate4 std::shared_ptr interpolate_v4; ov::Output v4_input_output_shape; ov::Output v4_input_scales; - - if (interpolate_v11->get_attrs().shape_calculation_mode == - ov::op::util::InterpolateBase::ShapeCalcMode::SCALES) { - v4_input_scales = interpolate_v11->input_value(1); - v4_input_output_shape = opset4::Constant::create(element::i32, Shape{1}, {1}); - copy_runtime_info(interpolate_v11, v4_input_output_shape.get_node_shared_ptr()); - } else { - v4_input_output_shape = interpolate_v11->input_value(1); - v4_input_scales = opset4::Constant::create(element::f32, Shape{1}, {1.0f}); - copy_runtime_info(interpolate_v11, v4_input_scales.get_node_shared_ptr()); - } + std::tie(v4_input_output_shape, v4_input_scales) = make_v4_inputs(interpolate_v11); if (interpolate_v11->get_input_size() == 3) { // with axes input interpolate_v4 = std::make_shared(interpolate_v11->input_value(0), diff --git a/src/common/transformations/tests/op_conversions/convert_interpolate11_downgrade_test.cpp b/src/common/transformations/tests/op_conversions/convert_interpolate11_downgrade_test.cpp index 7504cd378eb..d385493e3e8 100644 --- a/src/common/transformations/tests/op_conversions/convert_interpolate11_downgrade_test.cpp +++ b/src/common/transformations/tests/op_conversions/convert_interpolate11_downgrade_test.cpp @@ -59,7 +59,8 @@ std::shared_ptr create_v4_model(const bool with_axes, attributes.pads_begin = {0, 0}; attributes.pads_end = {0, 0}; - const auto input = std::make_shared(ov::element::i32, ov::Shape{1, 2, 10, 10}); + const auto input = std::make_shared(ov::element::i32, ov::Shape{1, 2, 10, 10}); + const auto axes = std::make_shared(ov::element::i32, ov::Shape{2}); std::shared_ptr output_shape; std::shared_ptr scales; std::shared_ptr interpolate; @@ -71,16 +72,29 @@ std::shared_ptr create_v4_model(const bool with_axes, if (shape_calc_mode == ov::opset4::Interpolate::ShapeCalcMode::SCALES) { scales = std::make_shared(ov::element::f32, ov::Shape{num_scales_or_sizes}); model_params.push_back(std::dynamic_pointer_cast(scales)); - output_shape = ov::opset4::Constant::create(ov::element::i32, ov::Shape{1}, {1}); - + output_shape = ov::opset4::Constant::create(ov::element::i32, ov::Shape{}, {1}); + if (with_axes) { + output_shape = + std::make_shared(output_shape, std::make_shared(axes)); + } else { + output_shape = std::make_shared( + output_shape, + std::make_shared(std::make_shared(input))); + } } else { output_shape = std::make_shared(ov::element::i32, ov::Shape{num_scales_or_sizes}); model_params.push_back(std::dynamic_pointer_cast(output_shape)); - scales = ov::opset4::Constant::create(ov::element::f32, ov::Shape{1}, {1.0f}); + scales = ov::opset4::Constant::create(ov::element::f32, ov::Shape{}, {1.0f}); + if (with_axes) { + scales = std::make_shared(scales, std::make_shared(axes)); + } else { + scales = std::make_shared( + scales, + std::make_shared(std::make_shared(input))); + } } if (with_axes) { - const auto axes = std::make_shared(ov::element::i32, ov::Shape{2}); model_params.push_back(axes); interpolate = std::make_shared(input, output_shape, scales, axes, attributes); } else { @@ -97,24 +111,32 @@ TEST_F(TransformationTestsF, ConvertInterpolate11ToInterpolate4_scales) { manager.register_pass(); function = create_v11_model(WITH_AXES, ov::opset11::Interpolate::ShapeCalcMode::SCALES); function_ref = create_v4_model(WITH_AXES, ov::opset4::Interpolate::ShapeCalcMode::SCALES); + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); } TEST_F(TransformationTestsF, ConvertInterpolate11ToInterpolate4_sizes) { manager.register_pass(); function = create_v11_model(WITH_AXES, ov::opset11::Interpolate::ShapeCalcMode::SIZES); function_ref = create_v4_model(WITH_AXES, ov::opset4::Interpolate::ShapeCalcMode::SIZES); + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); } TEST_F(TransformationTestsF, ConvertInterpolate11ToInterpolate4_scales_no_axes) { manager.register_pass(); function = create_v11_model(WITHOUT_AXES, ov::opset11::Interpolate::ShapeCalcMode::SCALES); function_ref = create_v4_model(WITHOUT_AXES, ov::opset4::Interpolate::ShapeCalcMode::SCALES); + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); } TEST_F(TransformationTestsF, ConvertInterpolate11ToInterpolate4_sizes_no_axes) { manager.register_pass(); function = create_v11_model(WITHOUT_AXES, ov::opset11::Interpolate::ShapeCalcMode::SIZES); function_ref = create_v4_model(WITHOUT_AXES, ov::opset4::Interpolate::ShapeCalcMode::SIZES); + comparator.enable(FunctionsComparator::CmpValues::CONST_VALUES); + comparator.enable(FunctionsComparator::CmpValues::ATTRIBUTES); } namespace { diff --git a/src/core/include/openvino/core/preprocess/resize_algorithm.hpp b/src/core/include/openvino/core/preprocess/resize_algorithm.hpp index eec0aa53658..98e100b5cd8 100644 --- a/src/core/include/openvino/core/preprocess/resize_algorithm.hpp +++ b/src/core/include/openvino/core/preprocess/resize_algorithm.hpp @@ -7,7 +7,14 @@ namespace ov { namespace preprocess { -enum class ResizeAlgorithm { RESIZE_LINEAR, RESIZE_CUBIC, RESIZE_NEAREST }; +/// \brief An enum containing all supported resize(interpolation) algorithms available in preprocessing +enum class ResizeAlgorithm { + RESIZE_LINEAR, //!< Linear interpolation matching the TensorFlow behavior + RESIZE_CUBIC, //!< Cubic interpolation + RESIZE_NEAREST, //!< Nearest interpolation + RESIZE_BILINEAR_PILLOW, //!< Bilinear interpolation matching the Pillow behavior + RESIZE_BICUBIC_PILLOW //!< Bicubic interpolation matching the Pillow behavior +}; } // namespace preprocess } // namespace ov diff --git a/src/core/src/preprocess/preprocess_steps_impl.cpp b/src/core/src/preprocess/preprocess_steps_impl.cpp index 9762fb7198c..d6ca402bc14 100644 --- a/src/core/src/preprocess/preprocess_steps_impl.cpp +++ b/src/core/src/preprocess/preprocess_steps_impl.cpp @@ -142,7 +142,7 @@ void PreStepsList::add_convert_impl(const element::Type& type) { } void PreStepsList::add_resize_impl(ResizeAlgorithm alg, int dst_height, int dst_width) { - using InterpolateMode = op::v4::Interpolate::InterpolateMode; + using InterpolateMode = op::util::InterpolateBase::InterpolateMode; std::string name; if (dst_width > 0 && dst_height > 0) { name = "resize to (" + std::to_string(dst_height) + ", " + std::to_string(dst_width) + ")"; @@ -157,48 +157,52 @@ void PreStepsList::add_resize_impl(ResizeAlgorithm alg, int dst_height, int dst_ OPENVINO_ASSERT(nodes.size() == 1, "Can't resize multi-plane input. Suggesting to convert current image to " "RGB/BGR color format using 'PreProcessSteps::convert_color'"); - auto to_mode = [](ResizeAlgorithm alg) -> InterpolateMode { + const auto to_mode = [](const ResizeAlgorithm alg) -> InterpolateMode { switch (alg) { case ResizeAlgorithm::RESIZE_NEAREST: return InterpolateMode::NEAREST; case ResizeAlgorithm::RESIZE_CUBIC: return InterpolateMode::CUBIC; + case ResizeAlgorithm::RESIZE_BILINEAR_PILLOW: + return InterpolateMode::BILINEAR_PILLOW; + case ResizeAlgorithm::RESIZE_BICUBIC_PILLOW: + return InterpolateMode::BICUBIC_PILLOW; case ResizeAlgorithm::RESIZE_LINEAR: default: return InterpolateMode::LINEAR; } }; - auto node = nodes.front(); - auto layout = ctxt.layout(); + const auto& layout = ctxt.layout(); OPENVINO_ASSERT(ov::layout::has_height(layout) && ov::layout::has_width(layout), "Can't add resize for layout without W/H specified. Use 'set_layout' API to define layout " "of image data, like `NCHW`"); - auto node_rank = node.get_partial_shape().rank(); - OPENVINO_ASSERT(node_rank.is_static(), "Resize operation is not supported for fully dynamic shape"); + const auto& node = nodes.front(); + OPENVINO_ASSERT(node.get_partial_shape().rank().is_static(), + "Resize operation is not supported for fully dynamic shape"); - auto height_idx = static_cast(get_and_check_height_idx(layout, node.get_partial_shape())); - auto width_idx = static_cast(get_and_check_width_idx(layout, node.get_partial_shape())); + const auto height_idx = static_cast(get_and_check_height_idx(layout, node.get_partial_shape())); + const auto width_idx = static_cast(get_and_check_width_idx(layout, node.get_partial_shape())); if (dst_height < 0 || dst_width < 0) { OPENVINO_ASSERT(ctxt.model_shape().rank().is_static(), "Resize is not fully specified while target model shape is dynamic"); } - int new_image_width = dst_width < 0 ? static_cast(ctxt.get_model_width_for_resize()) : dst_width; - int new_image_height = dst_height < 0 ? static_cast(ctxt.get_model_height_for_resize()) : dst_height; + const int new_image_width = dst_width < 0 ? static_cast(ctxt.get_model_width_for_resize()) : dst_width; + const int new_image_height = + dst_height < 0 ? static_cast(ctxt.get_model_height_for_resize()) : dst_height; - auto target_spatial_shape = + const auto target_spatial_shape = op::v0::Constant::create(element::i64, Shape{2}, {new_image_height, new_image_width}); - auto scales = op::v0::Constant::create(element::f32, Shape{2}, {1, 1}); // In future consider replacing this to set of new OV operations like `getDimByName(node, "H")` // This is to allow specifying layout on 'evaluation' stage - auto axes = op::v0::Constant::create(element::i64, Shape{2}, {height_idx, width_idx}); + const auto axes = op::v0::Constant::create(element::i64, Shape{2}, {height_idx, width_idx}); - op::v4::Interpolate::InterpolateAttrs attrs(to_mode(alg), - op::v4::Interpolate::ShapeCalcMode::SIZES, - {0, 0}, - {0, 0}); + op::util::InterpolateBase::InterpolateAttrs attrs(to_mode(alg), + op::util::InterpolateBase::ShapeCalcMode::SIZES, + {0, 0}, + {0, 0}); - auto interp = std::make_shared(node, target_spatial_shape, scales, axes, attrs); - return std::make_tuple(std::vector>{interp}, true); + const auto interp = std::make_shared(node, target_spatial_shape, axes, attrs); + return std::make_tuple(OutputVector{interp}, true); }, name); } diff --git a/src/core/tests/preprocess.cpp b/src/core/tests/preprocess.cpp index 4bed1518203..d714108aae9 100644 --- a/src/core/tests/preprocess.cpp +++ b/src/core/tests/preprocess.cpp @@ -923,6 +923,34 @@ TEST(pre_post_process, resize_no_model_layout) { EXPECT_NO_THROW(p.build()); } +TEST(pre_post_process, resize_with_bilinear_pillow) { + const auto f = create_simple_function(element::f32, PartialShape{1, 3, 224, 224}); + + auto p = PrePostProcessor(f); + // make the model accept images with spatial dimensions different than the original model's dims + p.input().tensor().set_shape({1, 3, 100, 150}).set_layout("NCHW"); + // resize the incoming images to the original model's dims (deduced by PPP) + p.input().preprocess().resize(ResizeAlgorithm::RESIZE_BILINEAR_PILLOW); + p.build(); + + EXPECT_EQ(f->input().get_partial_shape(), (PartialShape{1, 3, 100, 150})); + EXPECT_EQ(f->output().get_partial_shape(), (PartialShape{1, 3, 224, 224})); +} + +TEST(pre_post_process, resize_with_bicubic_pillow) { + const auto f = create_simple_function(element::f32, PartialShape{1, 3, 100, 100}); + + auto p = PrePostProcessor(f); + // make the model accept images with any spatial dimensions + p.input().tensor().set_shape({1, 3, -1, -1}).set_layout("NCHW"); + // resize the incoming images to the original model's dims (specified manually) + p.input().preprocess().resize(ResizeAlgorithm::RESIZE_BICUBIC_PILLOW, 100, 100); + p.build(); + + EXPECT_EQ(f->input().get_partial_shape(), (PartialShape{1, 3, -1, -1})); + EXPECT_EQ(f->output().get_partial_shape(), (PartialShape{1, 3, 100, 100})); +} + // Error cases for 'resize' TEST(pre_post_process, tensor_spatial_shape_no_layout_dims) { auto f = create_simple_function(element::f32, Shape{1, 3, 224, 224});