IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[OV2.0] Preprocessing - resize (#7583)

Browse Source 
* Initial version

* Added 'network' layout to preprocessing info
Moved existing resize tests to template plugin

* Fix clang

* More tests for 'resize' reference implementation + CPU tests + error cases
Coverage is 100%

* Align with new base_reference_test implementation

* Fixed comments

* Add assert to check that desired size is not out of bounds

* CPU: skip failed test
This commit is contained in:
Mikhail Nosov
2021-09-30 11:41:25 +03:00
committed by GitHub
parent c00b0b6ae4
commit 414c3dc133
14 changed files with 749 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
docs/template_plugin/tests/functional/op_reference/base_reference_test.cpp
View File

@@ -38,7 +38,12 @@ void CommonReferenceTest::FillInputs() {
for (size_t i = 0; i < functionParams.size(); i++) {
const auto& param = functionParams[i];
ov::runtime::Tensor blob(param->get_element_type(), param->get_shape());
ov::runtime::Tensor blob;
if (param->get_partial_shape().is_static()) {
blob = ov::runtime::Tensor(param->get_element_type(), param->get_shape());
} else {
blob = ov::runtime::Tensor(param->get_element_type(), inputData[i].get_shape());
}
ASSERT_EQ(blob.get_byte_size(), inputData[i].get_byte_size());
std::memcpy(blob.data(), inputData[i].data(), inputData[i].get_byte_size());

20
docs/template_plugin/tests/functional/op_reference/base_reference_test.hpp
View File

@@ -36,7 +36,9 @@ protected:
};
template <class T>
ov::runtime::Tensor CreateTensor(const ov::element::Type& element_type, const std::vector<T>& values, size_t size = 0) {
ov::runtime::Tensor CreateTensor(const ov::element::Type& element_type,
const std::vector<T>& values,
size_t size = 0) {
size_t real_size = size ? size : values.size() * sizeof(T) / element_type.size();
ov::runtime::Tensor tensor { element_type, {real_size} };
std::memcpy(tensor.data(), values.data(), std::min(real_size * element_type.size(), sizeof(T) * values.size()));
@@ -44,6 +46,17 @@ ov::runtime::Tensor CreateTensor(const ov::element::Type& element_type, const st
return tensor;
}
// Create blob with correct input shape (not 1-dimensional). Will be used in tests with dynamic input shapes
template <class T>
ov::runtime::Tensor CreateTensor(const ov::Shape& shape,
const ov::element::Type& element_type,
const std::vector<T>& values) {
ov::runtime::Tensor tensor { element_type, shape };
std::memcpy(tensor.data(), values.data(), sizeof(T) * values.size());
return tensor;
}
///
/// Class which should help to build data for single input
///
@@ -56,6 +69,11 @@ struct Tensor {
Tensor(const ov::Shape& shape, ov::element::Type type, const std::vector<T>& data_elements)
: Tensor {shape, type, CreateTensor(type, data_elements)} {}
// Temporary constructor to create blob with passed input shape (not 1-dimensional)
template <typename T>
Tensor(ov::element::Type type, const ov::Shape& shape, const std::vector<T>& data_elements)
: Tensor {shape, type, CreateTensor(shape, type, data_elements)} {}
ov::Shape shape;
ov::element::Type type;
ov::runtime::Tensor data;

157
docs/template_plugin/tests/functional/subgraph_reference/preprocess.cpp
View File

@@ -271,6 +271,155 @@ static RefPreprocessParams mean_scale_dynamic_layout() {
return res;
}
static RefPreprocessParams resize_to_network_height() {
RefPreprocessParams res("resize_to_network_height");
res.function = []() {
auto f = create_simple_function(element::f32, PartialShape{1, 2, 1, 1});
f = PrePostProcessor()
.input(InputInfo()
.tensor(InputTensorInfo().set_spatial_dynamic_shape())
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_LINEAR))
.network(InputNetworkInfo().set_layout("NHWC"))
)
.build(f);
return f;
};
res.inputs.emplace_back(element::f32, Shape{1, 4, 1, 1}, std::vector<float>{0., 2., 4., 6.});
res.expected.emplace_back(Shape{1, 2, 1, 1}, element::f32, std::vector<float>{1., 5.});
return res;
}
static RefPreprocessParams resize_to_network_width() {
RefPreprocessParams res("resize_to_network_width");
res.function = []() {
auto f = create_simple_function(element::f32, PartialShape{Dimension::dynamic(), 1, 2, 2});
f = PrePostProcessor()
.input(InputInfo()
.tensor(InputTensorInfo().set_spatial_dynamic_shape())
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_LINEAR))
.network(InputNetworkInfo().set_layout("NCHW")))
.build(f);
return f;
};
res.inputs.emplace_back(element::f32, Shape{1, 1, 2, 6}, std::vector<float>{0., 1., 2., 3., 4., 5.,
0., 1., 2., 3., 4., 5.});
res.expected.emplace_back(Shape{1, 1, 2, 2}, element::f32, std::vector<float>{1., 4., 1., 4.});
return res;
}
static RefPreprocessParams resize_from_spatial_dims() {
RefPreprocessParams res("resize_from_spatial_dims");
res.function = []() {
auto f = create_simple_function(element::f32, PartialShape{Dimension::dynamic(), 1, 1, 1});
auto t = InputTensorInfo();
t.set_spatial_static_shape(1, 4);
f = PrePostProcessor()
.input(InputInfo()
.tensor(std::move(t))
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_CUBIC))
.network(InputNetworkInfo().set_layout("NCHW")))
.build(f);
return f;
};
res.inputs.emplace_back(element::f32, Shape{1, 1, 1, 7}, std::vector<float>{0., 0.25, 1., 2.25, 4., 6.25, 9});
res.expected.emplace_back(Shape{1, 1, 1, 1}, element::f32, std::vector<float>{2.25});
return res;
}
static RefPreprocessParams resize_to_network_width_height() {
RefPreprocessParams res("resize_to_network_width_height");
res.function = []() {
auto f = create_simple_function(element::f32, PartialShape{1, 1, 4, 4});
f = PrePostProcessor()
.input(InputInfo()
.tensor(InputTensorInfo().set_spatial_static_shape(5, 5))
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_NEAREST))
.network(InputNetworkInfo().set_layout("...HW")))
.build(f);
return f;
};
auto result = std::make_shared<HostTensor>();
// clang-format off
std::vector<float> input = {0., 1., 2., 3., 4.,
1., 2., 3., 4., 5.,
2., 3., 4., 5., 6.,
3., 4., 5., 6., 7.,
2., 3., 4., 5., 6.};
std::vector<float> expected = {0., 1., 3., 4.,
1., 2., 4., 5.,
3., 4., 6., 7.,
2., 3., 5., 6.};
// clang-format on
res.inputs.emplace_back(element::f32, Shape{1, 1, 5, 5}, input);
res.expected.emplace_back(Shape{1, 1, 4, 4}, element::f32, expected);
return res;
}
static RefPreprocessParams resize_to_specified_width_height() {
RefPreprocessParams res("resize_to_specified_width_height");
res.function = []() {
auto f = create_simple_function(element::f32, PartialShape{1, 1, Dimension::dynamic(), Dimension::dynamic()});
f = PrePostProcessor()
.input(InputInfo()
.tensor(InputTensorInfo().set_spatial_dynamic_shape())
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_NEAREST, 4, 4))
.network(InputNetworkInfo().set_layout("...HW")))
.build(f);
return f;
};
auto result = std::make_shared<HostTensor>();
// clang-format off
std::vector<float> input = {0., 1., 2., 3., 4.,
1., 2., 3., 4., 5.,
2., 3., 4., 5., 6.,
3., 4., 5., 6., 7.,
2., 3., 4., 5., 6.};
std::vector<float> expected = {0., 1., 3., 4.,
1., 2., 4., 5.,
3., 4., 6., 7.,
2., 3., 5., 6.};
// clang-format on
res.inputs.emplace_back(element::f32, Shape{1, 1, 5, 5}, input);
res.expected.emplace_back(Shape{1, 1, 4, 4}, element::f32, expected);
return res;
}
static RefPreprocessParams resize_lvalues() {
RefPreprocessParams res("resize_lvalues");
res.function = []() {
auto f = create_simple_function(element::f32, PartialShape{Dimension::dynamic(), 1, 1, 2});
f->get_parameters().front()->set_layout("NCHW");
auto t = InputTensorInfo();
t.set_spatial_dynamic_shape();
auto s = PreProcessSteps();
s.resize(ResizeAlgorithm::RESIZE_LINEAR, 1, 6); // to specified shape
s.resize(ResizeAlgorithm::RESIZE_LINEAR); // to network's shape
auto n = InputNetworkInfo();
n.set_layout("NCHW");
auto i = InputInfo();
i.tensor(std::move(t));
i.preprocess(std::move(s));
i.network(std::move(n));
f = PrePostProcessor()
.input(std::move(i))
.build(f);
return f;
};
// clang-format off
res.inputs.emplace_back(element::f32, Shape{1, 1, 1, 18}, std::vector<float>{0., 0., 0.,
1., 1., 1.,
2., 2., 2.,
3., 3., 3.,
4., 4., 4.,
5., 5., 5.});
// clang-format on
res.expected.emplace_back(Shape{1, 1, 2, 1}, element::f32, std::vector<float>{1., 4.});
return res;
}
std::vector<RefPreprocessParams> allPreprocessTests() {
return std::vector<RefPreprocessParams> {
simple_mean_scale(),
@@ -282,7 +431,13 @@ std::vector<RefPreprocessParams> allPreprocessTests() {
test_lvalue(),
test_2_inputs_basic(),
mean_scale_vector_tensor_layout(),
mean_scale_dynamic_layout()
mean_scale_dynamic_layout(),
resize_to_network_height(),
resize_to_network_width(),
resize_from_spatial_dims(),
resize_to_network_width_height(),
resize_to_specified_width_height(),
resize_lvalues()
};
}

2
inference-engine/tests/functional/plugin/cpu/shared_tests_instances/skip_tests_config.cpp
View File

@@ -96,6 +96,8 @@ std::vector<std::string> disabledTestPatterns() {
// Issue: 62746
R"(smoke_CachingSupportCase_CPU/LoadNetworkCacheTestBase.CompareWithRefImpl/ReadConcatSplitAssign_f32_batch1_CPU)",
// Issue 66685
R"(smoke_PrePostProcess.*resize_linear_nhwc.*)",
};
#define FIX_62820 0

52
inference-engine/tests/ngraph_helpers/ngraph_functions/include/ngraph_functions/preprocess/preprocess_builders.hpp
View File

@@ -206,6 +206,54 @@ inline std::shared_ptr<Function> tensor_layout() {
return function;
}
inline std::shared_ptr<Function> resize_linear() {
using namespace ov::preprocess;
auto function = create_preprocess_1input(element::f32, PartialShape{1, 3, 10, 10});
function = PrePostProcessor()
.input(InputInfo()
.tensor(InputTensorInfo().set_spatial_static_shape(20, 20))
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_LINEAR))
.network(InputNetworkInfo().set_layout("NCHW")))
.build(function);
return function;
}
inline std::shared_ptr<Function> resize_nearest() {
using namespace ov::preprocess;
auto function = create_preprocess_1input(element::f32, PartialShape{1, 3, 10, 10});
function = PrePostProcessor()
.input(InputInfo()
.tensor(InputTensorInfo().set_spatial_static_shape(20, 20))
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_NEAREST))
.network(InputNetworkInfo().set_layout("NCHW")))
.build(function);
return function;
}
inline std::shared_ptr<Function> resize_linear_nhwc() {
using namespace ov::preprocess;
auto function = create_preprocess_1input(element::f32, PartialShape{1, 10, 10, 3});
function = PrePostProcessor()
.input(InputInfo()
.tensor(InputTensorInfo().set_spatial_static_shape(20, 20))
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_LINEAR))
.network(InputNetworkInfo().set_layout("NHWC")))
.build(function);
return function;
}
inline std::shared_ptr<Function> resize_cubic() {
using namespace ov::preprocess;
auto function = create_preprocess_1input(element::f32, PartialShape{1, 3, 20, 20});
function = PrePostProcessor()
.input(InputInfo()
.tensor(InputTensorInfo().set_spatial_static_shape(10, 10))
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_CUBIC))
.network(InputNetworkInfo().set_layout("NCHW")))
.build(function);
return function;
}
inline std::vector<preprocess_func> generic_preprocess_functions() {
return std::vector<preprocess_func> {
preprocess_func(mean_only, "mean_only", 0.01f),
@@ -221,6 +269,10 @@ inline std::vector<preprocess_func> generic_preprocess_functions() {
preprocess_func(two_inputs_basic, "two_inputs_basic", 0.01f),
preprocess_func(reuse_network_layout, "reuse_network_layout", 0.01f),
preprocess_func(tensor_layout, "tensor_layout", 0.01f),
preprocess_func(resize_linear, "resize_linear", 0.01f),
preprocess_func(resize_nearest, "resize_nearest", 0.01f),
preprocess_func(resize_linear_nhwc, "resize_linear_nhwc", 0.01f),
preprocess_func(resize_cubic, "resize_cubic", 0.01f),
};
}

23
ngraph/core/include/openvino/core/preprocess/input_info.hpp
View File

@@ -5,6 +5,7 @@
#pragma once
#include "openvino/core/core_visibility.hpp"
#include "openvino/core/preprocess/input_network_info.hpp"
#include "openvino/core/preprocess/input_tensor_info.hpp"
#include "openvino/core/preprocess/preprocess_steps.hpp"
@@ -12,9 +13,14 @@ namespace ov {
namespace preprocess {
/// \brief Class holding preprocessing information for one input
/// From preprocessing pipeline perspective, each input can be represented as:
/// - User's input parameter info (InputInfo::tensor)
/// - Preprocessing steps applied to user's input (InputInfo::preprocess)
/// - Network's input info, which is a final info after preprocessing (InputInfo::network)
///
/// API has Builder-like style to allow chaining calls in client's code, like
/// \code{.cpp}
/// auto proc = PrePostProcessor().input(InputInfo().tensor(...).preprocess(...);
/// auto proc = PrePostProcessor().input(InputInfo().tensor(...).preprocess(...).network(...);
/// \endcode
class OPENVINO_API InputInfo final {
class InputInfoImpl;
@@ -65,7 +71,22 @@ public:
/// \param builder Preprocessing operations.
///
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner
InputInfo&& preprocess(PreProcessSteps&& builder) &&;
/// \brief Set network's tensor information for input - Lvalue version
///
/// \param builder Input network tensor information.
///
/// \return Reference to 'this' to allow chaining with other calls in a builder-like manner
InputInfo& network(InputNetworkInfo&& builder) &;
/// \brief Set input tensor information for input - Rvalue version
///
/// \param builder Input network tensor information.
///
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner
InputInfo&& network(InputNetworkInfo&& builder) &&;
};
} // namespace preprocess

68
ngraph/core/include/openvino/core/preprocess/input_network_info.hpp Normal file
View File

@@ -0,0 +1,68 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include "openvino/core/core_visibility.hpp"
#include "openvino/core/layout.hpp"
#include "openvino/core/type/element_type.hpp"
namespace ov {
namespace preprocess {
/// \brief Information about network's input tensor. If all information is already included to loaded network, this info
/// may not be needed. However it can be set to specify additional information about network, like 'layout'.
///
/// Example of usage of network 'layout':
/// Support network has input parameter with shape {1, 3, 224, 224} and user needs to resize input image to network's
/// dimensions It can be done like this
///
/// \code{.cpp}
/// <network has input parameter with shape {1, 3, 224, 224}>
/// auto proc =
/// PrePostProcessor()
/// .input(InputInfo()
/// .tensor(<input tensor info>)
/// .preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_LINEAR))
/// .network(InputNetworkInfo()
/// .set_layout("NCHW"))
/// );
/// \endcode
class OPENVINO_API InputNetworkInfo final {
class InputNetworkInfoImpl;
std::unique_ptr<InputNetworkInfoImpl> m_impl;
friend class InputInfo;
public:
/// \brief Default empty constructor
InputNetworkInfo();
/// \brief Default move constructor
InputNetworkInfo(InputNetworkInfo&&) noexcept;
/// \brief Default move assignment
InputNetworkInfo& operator=(InputNetworkInfo&&) noexcept;
/// \brief Default destructor
~InputNetworkInfo();
/// \brief Set layout for network's input tensor
/// This version allows chaining for Lvalue objects
///
/// \param layout Layout for network's input tensor.
///
/// \return Reference to 'this' to allow chaining with other calls in a builder-like manner
InputNetworkInfo& set_layout(const ov::Layout& layout) &;
/// \brief Set layout for network's input tensor
/// This version allows chaining for Rvalue objects
///
/// \param layout Layout for network's input tensor.
///
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner
InputNetworkInfo&& set_layout(const ov::Layout& layout) &&;
};
} // namespace preprocess
} // namespace ov

46
ngraph/core/include/openvino/core/preprocess/input_tensor_info.hpp
View File

@@ -66,13 +66,57 @@ public:
/// \return Reference to 'this' to allow chaining with other calls in a builder-like manner
InputTensorInfo& set_layout(const ov::Layout& layout) &;
/// \briefSet layout for user's input tensor
/// \brief Set layout for user's input tensor
/// This version allows chaining for Rvalue objects
///
/// \param layout Layout for user's input tensor.
///
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner
InputTensorInfo&& set_layout(const ov::Layout& layout) &&;
/// \brief By default, input image shape is inherited from network input shape. This method specifies that user's
/// input image has dynamic spatial dimensions (width & height). This can be useful for adding resize preprocessing
/// from any input image to network's expected dimensions.
///
/// This version allows chaining for Lvalue objects.
///
/// \return Reference to 'this' to allow chaining with other calls in a builder-like manner.
InputTensorInfo& set_spatial_dynamic_shape() &;
/// \brief By default, input image shape is inherited from network input shape. This method specifies that user's
/// input image has dynamic spatial dimensions (width & height). This can be useful for adding resize preprocessing
/// from any input image to network's expected dimensions.
///
/// This version allows chaining for Rvalue objects.
///
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner.
InputTensorInfo&& set_spatial_dynamic_shape() &&;
/// \brief By default, input image shape is inherited from network input shape. Use this method to specify different
/// width and height of user's input image. In case if input image size is not known, use
/// `set_spatial_dynamic_shape` method.
///
/// This version allows chaining for Lvalue objects.
///
/// \param height Set fixed user's input image height.
///
/// \param width Set fixed user's input image width.
///
/// \return Reference to 'this' to allow chaining with other calls in a builder-like manner.
InputTensorInfo& set_spatial_static_shape(size_t height, size_t width) &;
/// \brief By default, input image shape is inherited from network input shape. Use this method to specify different
/// width and height of user's input image. In case if input image size is not known, use
/// `set_spatial_dynamic_shape` method.
///
/// This version allows chaining for Rvalue objects.
///
/// \param height Set fixed user's input image height.
///
/// \param width Set fixed user's input image width.
///
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner.
InputTensorInfo&& set_spatial_static_shape(size_t height, size_t width) &&;
};
} // namespace preprocess

37
ngraph/core/include/openvino/core/preprocess/preprocess_steps.hpp
View File

@@ -5,6 +5,7 @@
#pragma once
#include "openvino/core/core_visibility.hpp"
#include "openvino/core/preprocess/resize_algorithm.hpp"
#include "openvino/core/type/element_type.hpp"
namespace ov {
@@ -139,6 +140,42 @@ public:
///
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner
PreProcessSteps&& custom(const CustomPreprocessOp& preprocess_cb) &&;
/// \brief Add resize operation to known dimensions - Lvalue version.
///
/// \param alg Resize algorithm.
///
/// \param dst_height Desired height of resized image.
///
/// \param dst_width Desired width of resized image.
///
/// \return Reference to 'this' to allow chaining with other calls in a builder-like manner.
PreProcessSteps& resize(ResizeAlgorithm alg, size_t dst_height, size_t dst_width) &;
/// \brief Add resize operation to known dimensions - Rvalue version.
///
/// \param alg Resize algorithm.
///
/// \param dst_height Desired height of resized image.
///
/// \param dst_width Desired width of resized image.
///
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner.
PreProcessSteps&& resize(ResizeAlgorithm alg, size_t dst_height, size_t dst_width) &&;
/// \brief Add resize operation to network dimensions - Lvalue version.
///
/// \param alg Resize algorithm.
///
/// \return Reference to 'this' to allow chaining with other calls in a builder-like manner.
PreProcessSteps& resize(ResizeAlgorithm alg) &;
/// \brief Add resize operation to network dimensions - Rvalue version.
///
/// \param alg Resize algorithm.
///
/// \return Rvalue reference to 'this' to allow chaining with other calls in a builder-like manner.
PreProcessSteps&& resize(ResizeAlgorithm alg) &&;
};
} // namespace preprocess

13
ngraph/core/include/openvino/core/preprocess/resize_algorithm.hpp Normal file
View File

@@ -0,0 +1,13 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
namespace ov {
namespace preprocess {
enum class ResizeAlgorithm { RESIZE_LINEAR, RESIZE_CUBIC, RESIZE_NEAREST };
} // namespace preprocess
} // namespace ov

145
ngraph/core/src/preprocess/pre_post_process.cpp
View File

@@ -38,12 +38,65 @@ public:
return m_layout;
}
bool is_spatial_shape_set() const {
return m_spatial_shape_set;
}
int get_spatial_width() const {
return m_spatial_width;
}
int get_spatial_height() const {
return m_spatial_height;
}
bool is_spatial_shape_dynamic() const {
return m_spatial_shape_set && m_spatial_width == -1 && m_spatial_height == -1;
}
void set_spatial_dynamic_shape() {
m_spatial_shape_set = true;
m_spatial_width = -1;
m_spatial_height = -1;
}
void set_spatial_static_shape(size_t height, size_t width) & {
m_spatial_shape_set = true;
m_spatial_height = static_cast<int>(height);
m_spatial_width = static_cast<int>(width);
}
private:
element::Type m_type = element::dynamic;
bool m_type_set = false;
Layout m_layout = Layout();
bool m_layout_set = false;
int m_spatial_width = -1;
int m_spatial_height = -1;
bool m_spatial_shape_set = false;
};
/// \brief InputNetworkInfoImpl - internal data structure
class InputNetworkInfo::InputNetworkInfoImpl {
public:
InputNetworkInfoImpl() = default;
void set_layout(const Layout& layout) {
m_layout = layout;
m_layout_set = true;
}
bool is_layout_set() const {
return m_layout_set;
}
const Layout& get_layout() const {
return m_layout;
}
private:
Layout m_layout = Layout();
bool m_layout_set = false;
};
/// \brief InputInfoImpl - internal data structure
@@ -66,6 +119,7 @@ struct InputInfo::InputInfoImpl {
size_t m_index = 0;
std::unique_ptr<InputTensorInfo::InputTensorInfoImpl> m_tensor_data;
std::unique_ptr<PreProcessSteps::PreProcessStepsImpl> m_preprocess;
std::unique_ptr<InputNetworkInfo::InputNetworkInfoImpl> m_network_data;
};
//-------------- InputInfo ------------------
@@ -95,6 +149,16 @@ InputInfo& InputInfo::preprocess(PreProcessSteps&& builder) & {
return *this;
}
InputInfo& InputInfo::network(InputNetworkInfo&& builder) & {
m_impl->m_network_data = std::move(builder.m_impl);
return *this;
}
InputInfo&& InputInfo::network(InputNetworkInfo&& builder) && {
m_impl->m_network_data = std::move(builder.m_impl);
return std::move(*this);
}
// ------------------------ PrePostProcessor --------------------
struct PrePostProcessor::PrePostProcessorImpl {
public:
@@ -132,6 +196,10 @@ std::shared_ptr<Function> PrePostProcessor::build(const std::shared_ptr<Function
"particular input instead of default one");
param = function->get_parameters().front();
}
// Set parameter layout from 'network' information
if (input->m_network_data && input->m_network_data->is_layout_set() && param->get_layout() == Layout()) {
param->set_layout(input->m_network_data->get_layout());
}
auto consumers = param->output(0).get_target_inputs();
if (!input->m_tensor_data) {
input->create_tensor_data(param->get_element_type(), param->get_layout());
@@ -143,6 +211,19 @@ std::shared_ptr<Function> PrePostProcessor::build(const std::shared_ptr<Function
input->m_tensor_data->set_element_type(param->get_element_type());
}
auto new_param_shape = param->get_partial_shape();
if (input->m_tensor_data->is_spatial_shape_set()) {
auto height_idx = get_and_check_height_idx(input->m_tensor_data->get_layout(), new_param_shape);
auto width_idx = get_and_check_width_idx(input->m_tensor_data->get_layout(), new_param_shape);
if (input->m_tensor_data->is_spatial_shape_dynamic()) {
// Use dynamic spatial dimensions
new_param_shape[height_idx] = Dimension::dynamic();
new_param_shape[width_idx] = Dimension::dynamic();
} else {
// Use static spatial dimensions
new_param_shape[height_idx] = input->m_tensor_data->get_spatial_height();
new_param_shape[width_idx] = input->m_tensor_data->get_spatial_width();
}
}
auto new_param = std::make_shared<op::v0::Parameter>(input->m_tensor_data->get_element_type(), new_param_shape);
if (input->m_tensor_data->is_layout_set()) {
new_param->set_layout(input->m_tensor_data->get_layout());
@@ -155,6 +236,8 @@ std::shared_ptr<Function> PrePostProcessor::build(const std::shared_ptr<Function
std::shared_ptr<Node> node = new_param;
PreprocessingContext context(new_param->get_layout());
context.network_layout() = param->get_layout();
context.network_shape() = param->get_partial_shape();
// 2. Apply preprocessing
for (const auto& action : input->m_preprocess->actions()) {
node = std::get<0>(action)({node}, context);
@@ -208,6 +291,42 @@ InputTensorInfo&& InputTensorInfo::set_layout(const Layout& layout) && {
return std::move(*this);
}
InputTensorInfo& InputTensorInfo::set_spatial_dynamic_shape() & {
m_impl->set_spatial_dynamic_shape();
return *this;
}
InputTensorInfo&& InputTensorInfo::set_spatial_dynamic_shape() && {
m_impl->set_spatial_dynamic_shape();
return std::move(*this);
}
InputTensorInfo& InputTensorInfo::set_spatial_static_shape(size_t height, size_t width) & {
m_impl->set_spatial_static_shape(height, width);
return *this;
}
InputTensorInfo&& InputTensorInfo::set_spatial_static_shape(size_t height, size_t width) && {
m_impl->set_spatial_static_shape(height, width);
return std::move(*this);
}
// --------------------- InputNetworkInfo ------------------
InputNetworkInfo::InputNetworkInfo() : m_impl(std::unique_ptr<InputNetworkInfoImpl>(new InputNetworkInfoImpl())) {}
InputNetworkInfo::InputNetworkInfo(InputNetworkInfo&&) noexcept = default;
InputNetworkInfo& InputNetworkInfo::operator=(InputNetworkInfo&&) noexcept = default;
InputNetworkInfo::~InputNetworkInfo() = default;
InputNetworkInfo& InputNetworkInfo::set_layout(const Layout& layout) & {
m_impl->set_layout(layout);
return *this;
}
InputNetworkInfo&& InputNetworkInfo::set_layout(const Layout& layout) && {
m_impl->set_layout(layout);
return std::move(*this);
}
// --------------------- PreProcessSteps ------------------
PreProcessSteps::PreProcessSteps() : m_impl(std::unique_ptr<PreProcessStepsImpl>(new PreProcessStepsImpl())) {}
@@ -265,6 +384,32 @@ PreProcessSteps&& PreProcessSteps::convert_element_type(const element::Type& typ
return std::move(*this);
}
PreProcessSteps& PreProcessSteps::resize(ResizeAlgorithm alg, size_t dst_height, size_t dst_width) & {
OPENVINO_ASSERT(dst_height <= std::numeric_limits<int>::max() && dst_width <= std::numeric_limits<int>::max(),
"Resize: Width/Height dimensions cannot be greater than ",
std::to_string(std::numeric_limits<int>::max()));
m_impl->add_resize_impl(alg, static_cast<int>(dst_height), static_cast<int>(dst_width));
return *this;
}
PreProcessSteps&& PreProcessSteps::resize(ResizeAlgorithm alg, size_t dst_height, size_t dst_width) && {
OPENVINO_ASSERT(dst_height <= std::numeric_limits<int>::max() && dst_width <= std::numeric_limits<int>::max(),
"Resize: Width/Height dimensions cannot be greater than ",
std::to_string(std::numeric_limits<int>::max()));
m_impl->add_resize_impl(alg, static_cast<int>(dst_height), static_cast<int>(dst_width));
return std::move(*this);
}
PreProcessSteps& PreProcessSteps::resize(ResizeAlgorithm alg) & {
m_impl->add_resize_impl(alg, -1, -1);
return *this;
}
PreProcessSteps&& PreProcessSteps::resize(ResizeAlgorithm alg) && {
m_impl->add_resize_impl(alg, -1, -1);
return std::move(*this);
}
PreProcessSteps& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb) & {
// 'true' indicates that custom preprocessing step will trigger validate_and_infer_types
m_impl->actions().emplace_back(std::make_tuple(

68
ngraph/core/src/preprocess/preprocess_steps_impl.cpp
View File

@@ -15,17 +15,10 @@ static Shape construct_mean_scale_shape(const std::shared_ptr<Node>& node,
size_t values_size,
const PreprocessingContext& context) {
// TODO: support also Mean/Scale image case
OPENVINO_ASSERT(layout::has_channels(context.layout()), "Channels dimension is not specified in layout");
auto channels_index = layout::channels(context.layout());
auto node_shape = node->get_output_partial_shape(0);
auto node_rank = node->get_output_partial_shape(0).rank();
OPENVINO_ASSERT(node_rank.is_static(), "Mean/scale vector operation is not supported for fully dynamic shape");
auto node_rank = node_shape.rank();
auto channels_index = get_and_check_channels_idx(context.layout(), node_shape);
std::vector<std::size_t> v(node_rank.get_length(), 1);
if (channels_index < 0) {
// E.g. channels_index = -1 means last dimension
channels_index = node_rank.get_length() + channels_index;
}
OPENVINO_ASSERT(node_rank.get_length() > channels_index, "Channels dimension is out of bounds");
OPENVINO_ASSERT(node_shape[channels_index].is_dynamic() || node_shape[channels_index] == values_size,
"Number of channels and mean/values size mismatch: Channels = ",
@@ -98,5 +91,60 @@ void PreProcessSteps::PreProcessStepsImpl::add_convert_impl(const ov::element::T
true));
}
void PreProcessSteps::PreProcessStepsImpl::add_resize_impl(ResizeAlgorithm alg, int dst_height, int dst_width) {
using InterpolateMode = op::v4::Interpolate::InterpolateMode;
m_actions.emplace_back(std::make_tuple(
[alg, dst_width, dst_height](const std::vector<std::shared_ptr<Node>>& nodes, PreprocessingContext& ctxt) {
OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't add resize for empty input.");
OPENVINO_ASSERT(nodes.size() == 1,
"Can't resize multi-plane input. Suggesting to convert current image to "
"RGB/BGR color format using 'PreProcessSteps::convert_color'");
auto to_mode = [](ResizeAlgorithm alg) -> InterpolateMode {
switch (alg) {
case ResizeAlgorithm::RESIZE_NEAREST:
return InterpolateMode::NEAREST;
case ResizeAlgorithm::RESIZE_CUBIC:
return InterpolateMode::CUBIC;
case ResizeAlgorithm::RESIZE_LINEAR:
default:
return InterpolateMode::LINEAR;
}
};
auto node = nodes.front();
auto layout = ctxt.layout();
OPENVINO_ASSERT(ov::layout::has_height(layout) && ov::layout::has_width(layout),
"Can't add resize for layout without W/H specified. Use 'set_layout' API to define layout "
"of image data, like `NCHW`");
auto node_rank = node->get_output_partial_shape(0).rank();
OPENVINO_ASSERT(node_rank.is_static(), "Resize operation is not supported for fully dynamic shape");
auto height_idx = static_cast<int64_t>(get_and_check_height_idx(layout, node->get_output_partial_shape(0)));
auto width_idx = static_cast<int64_t>(get_and_check_width_idx(layout, node->get_output_partial_shape(0)));
if (dst_height < 0 || dst_width < 0) {
OPENVINO_ASSERT(ctxt.network_shape().rank().is_static(),
"Resize is not fully specified while target network shape is dynamic");
}
int new_image_width = dst_width < 0 ? static_cast<int>(ctxt.get_network_width_for_resize()) : dst_width;
int new_image_height = dst_height < 0 ? static_cast<int>(ctxt.get_network_height_for_resize()) : dst_height;
auto target_spatial_shape =
op::v0::Constant::create<int64_t>(element::i64, Shape{2}, {new_image_height, new_image_width});
auto scales = op::v0::Constant::create<float>(element::f32, Shape{2}, {1, 1});
// In future consider replacing this to set of new OV operations like `getDimByName(node, "H")`
// This is to allow specifying layout on 'evaluation' stage
auto axes = op::v0::Constant::create<int64_t>(element::i64, Shape{2}, {height_idx, width_idx});
op::v4::Interpolate::InterpolateAttrs attrs(to_mode(alg),
op::v4::Interpolate::ShapeCalcMode::SIZES,
{0, 0},
{0, 0});
auto interp = std::make_shared<op::v4::Interpolate>(node, target_spatial_shape, scales, axes, attrs);
interp->set_friendly_name(nodes[0]->get_friendly_name() + "/resize");
return interp;
},
true));
}
} // namespace preprocess
} // namespace ov
} // namespace ov

76
ngraph/core/src/preprocess/preprocess_steps_impl.hpp
View File

@@ -7,11 +7,54 @@
#include <list>
#include "openvino/core/layout.hpp"
#include "openvino/core/partial_shape.hpp"
#include "openvino/core/preprocess/preprocess_steps.hpp"
namespace ov {
namespace preprocess {
inline size_t get_and_check_width_idx(const Layout& layout, const PartialShape& shape) {
OPENVINO_ASSERT(ov::layout::has_width(layout), "Layout ", layout.to_string(), " doesn't have `width` dimension");
OPENVINO_ASSERT(shape.rank().is_static(), "Can't get shape width index for shape with dynamic rank");
auto idx = ov::layout::width(layout);
if (idx < 0) {
idx = shape.rank().get_length() + idx;
}
OPENVINO_ASSERT(idx >= 0 && shape.rank().get_length() > idx,
"Width dimension is out of bounds ",
std::to_string(idx));
return idx;
}
inline size_t get_and_check_height_idx(const Layout& layout, const PartialShape& shape) {
OPENVINO_ASSERT(ov::layout::has_height(layout), "Layout ", layout.to_string(), " doesn't have `height` dimension");
OPENVINO_ASSERT(shape.rank().is_static(), "Can't get shape height index for shape with dynamic rank");
auto idx = ov::layout::height(layout);
if (idx < 0) {
idx = shape.rank().get_length() + idx;
}
OPENVINO_ASSERT(idx >= 0 && shape.rank().get_length() > idx,
"Height dimension is out of bounds ",
std::to_string(idx));
return idx;
}
inline size_t get_and_check_channels_idx(const Layout& layout, const PartialShape& shape) {
OPENVINO_ASSERT(ov::layout::has_channels(layout),
"Layout ",
layout.to_string(),
" doesn't have `channels` dimension");
OPENVINO_ASSERT(shape.rank().is_static(), "Can't get shape channels index for shape with dynamic rank");
auto idx = ov::layout::channels(layout);
if (idx < 0) {
idx = shape.rank().get_length() + idx;
}
OPENVINO_ASSERT(idx >= 0 && shape.rank().get_length() > idx,
"Channels dimension is out of bounds ",
std::to_string(idx));
return idx;
}
/// \brief Preprocessing context passed to each preprocessing operation.
/// This is internal structure which is not shared to custom operations yet.
class PreprocessingContext {
@@ -26,8 +69,40 @@ public:
return m_layout;
}
const PartialShape& network_shape() const {
return m_network_shape;
}
PartialShape& network_shape() {
return m_network_shape;
}
const Layout& network_layout() const {
return m_network_layout;
}
Layout& network_layout() {
return m_network_layout;
}
size_t get_network_height_for_resize() const {
auto network_height_idx = get_and_check_height_idx(network_layout(), network_shape());
OPENVINO_ASSERT(network_shape()[network_height_idx].is_static(),
"Dynamic resize: Network height dimension shall be static");
return network_shape()[network_height_idx].get_length();
}
size_t get_network_width_for_resize() const {
auto network_width_idx = get_and_check_width_idx(network_layout(), network_shape());
OPENVINO_ASSERT(network_shape()[network_width_idx].is_static(),
"Dynamic resize: Network width dimension shall be static");
return network_shape()[network_width_idx].get_length();
}
private:
Layout m_layout;
PartialShape m_network_shape;
Layout m_network_layout;
};
using InternalPreprocessOp =
@@ -40,6 +115,7 @@ public:
void add_scale_impl(const std::vector<float>& values);
void add_mean_impl(const std::vector<float>& values);
void add_convert_impl(const element::Type& type);
void add_resize_impl(ResizeAlgorithm alg, int dst_height, int dst_width);
const std::list<std::tuple<InternalPreprocessOp, bool>>& actions() const {
return m_actions;

50
ngraph/test/preprocess.cpp
View File

@@ -277,3 +277,53 @@ TEST(pre_post_process, mean_vector_dynamic_channels_shape) {
.build(f));
EXPECT_EQ(f->get_output_element_type(0), element::f32);
}
// Error cases for 'resize'
TEST(pre_post_process, resize_no_network_layout) {
auto f = create_simple_function(element::f32, Shape{1, 3, 224, 224});
EXPECT_THROW(f = PrePostProcessor()
.input(InputInfo()
.tensor(InputTensorInfo().set_layout("NHWC"))
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_CUBIC)))
.build(f),
ov::AssertFailure);
}
TEST(pre_post_process, tensor_spatial_shape_no_layout_dims) {
auto f = create_simple_function(element::f32, Shape{1, 3, 224, 224});
EXPECT_THROW(f = PrePostProcessor()
.input(InputInfo()
.tensor(InputTensorInfo().set_layout("NC?W").set_spatial_static_shape(480, 640))
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_CUBIC)))
.build(f),
ov::AssertFailure);
EXPECT_THROW(f = PrePostProcessor()
.input(InputInfo()
.tensor(InputTensorInfo().set_layout("NCH?").set_spatial_static_shape(480, 640))
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_CUBIC)))
.build(f),
ov::AssertFailure);
}
TEST(pre_post_process, resize_no_tensor_height) {
auto f = create_simple_function(element::f32, Shape{1, 3, 224, 224});
EXPECT_THROW(f = PrePostProcessor()
.input(InputInfo()
.tensor(InputTensorInfo().set_layout("N?WC"))
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_LINEAR))
.network(InputNetworkInfo().set_layout("NHWC")))
.build(f),
ov::AssertFailure);
}
TEST(pre_post_process, resize_no_tensor_width) {
auto f = create_simple_function(element::f32, Shape{1, 3, 224, 224});
EXPECT_THROW(f = PrePostProcessor()
.input(InputInfo()
.tensor(InputTensorInfo().set_layout("NH?C"))
.preprocess(PreProcessSteps().resize(ResizeAlgorithm::RESIZE_LINEAR))
.network(InputNetworkInfo().set_layout("NHWC")))
.build(f),
ov::AssertFailure);
}