IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[OV2.0] PrePostProcessor dump to output stream for debugging purposes (#9580)

Browse Source 
* Calculate model layout based on 'tensor' layout and convert steps

Previously, 'model layout' is set to '...' by default,
thus no shape conversion happened when tensor layout is set to 'NHWC', then there was explicit convert_layout "NCHW"

Now "model layout" is calculated based on tensor layout and conversion steps:
Examples:
 1) Tensor: NHWC, Convert: NCHW. Result: NCHW
 2) Tensor: NHWC, Convert: 0312. Result: NCHW

* Initial move of tensor data calculation

* Moved 'impls' to new file

* Postprocessing + unit tests

* clang-format fix

* Added more details to preprocessing nodes
- Mean/Scale - will print mean/scale values
- Convert type - will print type
- Convert layout - will print destination layout
- Convert color - will print destination color

It is needed to troubleshoot the problems. If error occurs, message will not display last op's target shape/layout/type

* Add python bindings

* update tests

* Added memory type to dump if set

* Code style fix

* unity build fix

* Dump tensor if only memory type is set

* Added debug print

* Fix Param->Result case

Previously, layout was set by preprocessing set to old parameter as well
This is incorrect because in case of exception layout info will not be reverted

In this case old Result pointed to old Parameter and was able to preserve runtime info

After fixing of this, case Param->Result was broken if revalidation is not triggerred

Fix is to detect 'Result' as a consumer of some parameter and force revalidation in this case

* Revert occasionally committed line

* And one more line
This commit is contained in:
Mikhail Nosov 2022-01-12 22:00:32 +03:00 committed by GitHub
parent e78ada1178
commit 896532ace2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 1414 additions and 817 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/bindings/python/src/pyopenvino/graph/preprocess/pre_post_process.cpp
View File

@ -423,4 +423,14 @@ void regclass_graph_PrePostProcessor(py::module m) {
}, },
py::arg("output_index")); py::arg("output_index"));
proc.def("build", &ov::preprocess::PrePostProcessor::build); proc.def("build", &ov::preprocess::PrePostProcessor::build);
proc.def("__str__", [](const ov::preprocess::PrePostProcessor& self) -> std::string {
std::stringstream ss;
ss << self;
return ss.str();
});
proc.def("__repr__", [](const ov::preprocess::PrePostProcessor& self) -> std::string {
return "<PrePostProcessor: " + py::cast(self).attr("__str__")().cast<std::string>() + ">";
});
} }

30
src/bindings/python/tests/test_ngraph/test_preprocess.py
View File

@ -431,3 +431,33 @@ def test_ngraph_preprocess_model():
output = computation(input_data, input_data) output = computation(input_data, input_data)
assert np.equal(output, expected_output).all() assert np.equal(output, expected_output).all()
def test_ngraph_preprocess_dump():
shape = [1, 3, 224, 224]
parameter_a = ops.parameter(shape, dtype=np.float32, name="RGB_input")
model = parameter_a
function = Model(model, [parameter_a], "TestFunction")
p = PrePostProcessor(function)
p.input().tensor()\
.set_layout(ov.Layout("NHWC"))\
.set_element_type(Type.u8)\
.set_spatial_dynamic_shape()
p.input().preprocess()\
.convert_element_type(Type.f32)\
.reverse_channels()\
.mean([1, 2, 3])\
.scale([4, 5, 6])\
.resize(ResizeAlgorithm.RESIZE_LINEAR)
p.input().model().set_layout(ov.Layout("NCHW"))
p_str = str(p)
print(p)
assert "Pre-processing steps (5):" in p_str
assert "convert type (f32):" in p_str
assert "reverse channels:" in p_str
assert "mean (1,2,3):" in p_str
assert "scale (4,5,6):" in p_str
assert "resize to model width/height:" in p_str
assert "Implicit pre-processing steps (1):" in p_str
assert "convert layout " + ov.Layout("NCHW").to_string() in p_str

14
src/core/include/openvino/core/preprocess/pre_post_process.hpp
View File

@ -94,7 +94,21 @@ public:
/// ///
/// \return Function with added pre/post-processing operations /// \return Function with added pre/post-processing operations
std::shared_ptr<Model> build(); std::shared_ptr<Model> build();
private:
friend OPENVINO_API std::ostream& operator<<(std::ostream& str, const PrePostProcessor& prePostProcessor);
void dump(std::ostream&) const;
}; };
/// \brief Inserts a human-readable representation of a PrePostProcessors into an output stream. The output to the
/// stream is in "informal" notation and can be used for debugging purposes
///
/// \param str The output stream targeted for insertion.
///
/// \param prePostProcessor The shape to be inserted into output stream.
///
/// \return A reference to same output stream after insertion.
OPENVINO_API std::ostream& operator<<(std::ostream& str, const PrePostProcessor& prePostProcessor);
} // namespace preprocess } // namespace preprocess
} // namespace ov } // namespace ov

531
src/core/src/preprocess/pre_post_process.cpp
View File

@ -7,232 +7,12 @@
#include "color_utils.hpp" #include "color_utils.hpp"
#include "function_guard.hpp" #include "function_guard.hpp"
#include "layout_utils.hpp" #include "layout_utils.hpp"
#include "ngraph/opsets/opset1.hpp"
#include "openvino/core/model.hpp" #include "openvino/core/model.hpp"
#include "preprocess_steps_impl.hpp" #include "preprocess_impls.hpp"
namespace ov { namespace ov {
namespace preprocess { namespace preprocess {
class TensorInfoImplBase {
public:
TensorInfoImplBase() = default;
void set_element_type(const element::Type& type) {
m_type = type;
m_type_set = true;
}
bool is_element_type_set() const {
return m_type_set;
}
const element::Type& get_element_type() const {
return m_type;
}
void set_layout(const Layout& layout) {
m_layout = layout;
m_layout_set = true;
}
bool is_layout_set() const {
return m_layout_set;
}
const Layout& get_layout() const {
return m_layout;
}
protected:
element::Type m_type = element::dynamic;
bool m_type_set = false;
Layout m_layout = Layout();
bool m_layout_set = false;
};
/// \brief InputTensorInfoImpl - internal data structure
class InputTensorInfo::InputTensorInfoImpl : public TensorInfoImplBase {
public:
InputTensorInfoImpl() = default;
bool is_spatial_shape_set() const {
return m_spatial_shape_set;
}
int get_spatial_width() const {
return m_spatial_width;
}
int get_spatial_height() const {
return m_spatial_height;
}
bool is_spatial_shape_dynamic() const {
return m_spatial_shape_set && m_spatial_width == -1 && m_spatial_height == -1;
}
void set_spatial_dynamic_shape() {
OPENVINO_ASSERT(!m_shape_set, "'set_spatial_dynamic_shape' and 'set_shape' shall not be used together");
m_spatial_shape_set = true;
m_spatial_width = -1;
m_spatial_height = -1;
}
void set_spatial_static_shape(size_t height, size_t width) & {
OPENVINO_ASSERT(!m_shape_set, "'set_spatial_static_shape' and 'set_shape' shall not be used together");
m_spatial_shape_set = true;
m_spatial_height = static_cast<int>(height);
m_spatial_width = static_cast<int>(width);
}
const ColorFormat& get_color_format() const {
return m_color_format;
}
void set_color_format(ColorFormat format, const std::vector<std::string>& sub_names) {
auto info = ColorFormatInfo::get(format);
if (info->planes_count() == 1) {
OPENVINO_ASSERT(sub_names.empty(),
"Plane names are not allowed for single plane color format '",
color_format_name(format),
"'");
} else if (!sub_names.empty()) {
OPENVINO_ASSERT(sub_names.size() == info->planes_count(),
"Number of sub-names (",
sub_names.size(),
") shall match with number of planes for '",
color_format_name(format),
"' color format (",
info->planes_count(),
")");
}
m_planes_sub_names = sub_names;
m_color_format = format;
}
const std::vector<std::string>& planes_sub_names() const {
return m_planes_sub_names;
}
void set_memory_type(const std::string& mem_type) {
m_memory_type_set = true;
m_memory_type = mem_type;
}
const std::string& get_memory_type() const {
return m_memory_type;
}
bool is_memory_type_set() const {
return m_memory_type_set;
}
void set_shape(const PartialShape& shape) {
OPENVINO_ASSERT(
!m_spatial_shape_set,
"'set_spatial_static_shape', 'set_spatial_dynamic_shape', 'set_shape' shall not be used together");
m_shape = shape;
m_shape_set = true;
}
bool is_shape_set() const {
return m_shape_set;
}
const PartialShape& get_shape() const {
return m_shape;
}
private:
ColorFormat m_color_format = ColorFormat::UNDEFINED;
std::vector<std::string> m_planes_sub_names;
element::Type m_type = element::dynamic;
bool m_type_set = false;
Layout m_layout = Layout();
bool m_layout_set = false;
int m_spatial_width = -1;
int m_spatial_height = -1;
bool m_spatial_shape_set = false;
std::string m_memory_type = {};
bool m_memory_type_set = false;
PartialShape m_shape = {};
bool m_shape_set = false;
};
class OutputTensorInfo::OutputTensorInfoImpl : public TensorInfoImplBase {};
/// \brief ModelInfoImpl - internal data structure
class ModelInfoImpl {
public:
ModelInfoImpl() = default;
void set_layout(const Layout& layout) {
m_layout = layout;
m_layout_set = true;
}
bool is_layout_set() const {
return m_layout_set;
}
const Layout& get_layout() const {
return m_layout;
}
private:
Layout m_layout = Layout();
bool m_layout_set = false;
};
class InputModelInfo::InputModelInfoImpl : public ModelInfoImpl {};
class OutputModelInfo::OutputModelInfoImpl : public ModelInfoImpl {};
/// \brief InputInfoImpl - internal data structure
struct InputInfo::InputInfoImpl {
InputInfoImpl() = default;
std::unique_ptr<InputTensorInfo::InputTensorInfoImpl>& get_tensor_data() {
return m_tensor_info.m_impl;
}
std::unique_ptr<PreProcessSteps::PreProcessStepsImpl>& get_preprocess() {
return m_preprocess.m_impl;
}
std::unique_ptr<InputModelInfo::InputModelInfoImpl>& get_model() {
return m_model_data.m_impl;
}
InputTensorInfo m_tensor_info;
PreProcessSteps m_preprocess;
InputModelInfo m_model_data;
std::shared_ptr<op::v0::Parameter> m_resolved_param;
};
/// \brief OutputInfoImpl - internal data structure
struct OutputInfo::OutputInfoImpl {
OutputInfoImpl() = default;
std::unique_ptr<OutputTensorInfo::OutputTensorInfoImpl>& get_tensor_data() {
return m_tensor_info.m_impl;
}
std::unique_ptr<PostProcessSteps::PostProcessStepsImpl>& get_postprocess() {
return m_postprocess.m_impl;
}
std::unique_ptr<OutputModelInfo::OutputModelInfoImpl>& get_model_data() {
return m_model_info.m_impl;
}
OutputTensorInfo m_tensor_info;
PostProcessSteps m_postprocess;
OutputModelInfo m_model_info;
ov::Output<ov::Node> m_output_node;
};
//-------------- InputInfo ------------------ //-------------- InputInfo ------------------
InputInfo::InputInfo() : m_impl(std::unique_ptr<InputInfoImpl>(new InputInfoImpl)) {} InputInfo::InputInfo() : m_impl(std::unique_ptr<InputInfoImpl>(new InputInfoImpl)) {}
InputInfo::InputInfo(InputInfo&& other) noexcept = default; InputInfo::InputInfo(InputInfo&& other) noexcept = default;
@ -359,196 +139,37 @@ OutputInfo& PrePostProcessor::output(const std::string& tensor_name) {
return m_impl->find_output(tensor_name); return m_impl->find_output(tensor_name);
} }
std::ostream& operator<<(std::ostream& str, const PrePostProcessor& prePostProcessor) {
try {
prePostProcessor.dump(str);
} catch (ov::AssertFailure& ex) {
str << std::endl << "Error occurred: " << ex.what();
}
return str;
}
void PrePostProcessor::dump(std::ostream& str) const {
auto model = m_impl->m_function;
std::tuple<std::unordered_set<std::string>, bool> existing_names{std::unordered_set<std::string>{}, false};
for (const auto& input_info : m_impl->m_inputs) {
input_info.m_impl->dump(str, model, existing_names);
}
for (const auto& output_info : m_impl->m_outputs) {
output_info.m_impl->dump(str);
}
}
std::shared_ptr<Model> PrePostProcessor::build() { std::shared_ptr<Model> PrePostProcessor::build() {
auto function = m_impl->m_function; auto function = m_impl->m_function;
FunctionGuard guard(function);
std::tuple<std::unordered_set<std::string>, bool> existing_names{std::unordered_set<std::string>{}, false}; std::tuple<std::unordered_set<std::string>, bool> existing_names{std::unordered_set<std::string>{}, false};
bool tensor_data_updated = false; FunctionGuard guard(function);
for (const auto& input_info : m_impl->m_inputs) { bool need_validate = false;
auto& input = input_info.m_impl;
// Set parameter layout from 'model' information
if (input->get_model()->is_layout_set()) {
// Overwrite existing model's layout here (fix 74065)
input->m_resolved_param->set_layout(input->get_model()->get_layout());
}
}
auto results = function->get_results(); auto results = function->get_results();
auto parameters_list = std::list<std::shared_ptr<op::v0::Parameter>>(function->get_parameters().begin(), auto parameters_list = std::list<std::shared_ptr<opset8::Parameter>>(function->get_parameters().begin(),
function->get_parameters().end()); function->get_parameters().end());
for (const auto& input_info : m_impl->m_inputs) { for (const auto& input_info : m_impl->m_inputs) {
const auto& input = input_info.m_impl; need_validate |= input_info.m_impl->build(function, existing_names, parameters_list);
auto param = input->m_resolved_param;
auto consumers = param->output(0).get_target_inputs();
if (!input->get_tensor_data()->is_element_type_set()) {
input->get_tensor_data()->set_element_type(param->get_element_type());
}
auto color_info = ColorFormatInfo::get(input->get_tensor_data()->get_color_format());
if (!input->get_tensor_data()->is_layout_set()) {
if (!color_info->default_layout().empty()) {
input->get_tensor_data()->set_layout(color_info->default_layout());
}
}
auto net_shape = param->get_partial_shape();
auto new_param_shape = net_shape;
auto model_layout = param->get_layout();
if (model_layout.empty() && input->get_tensor_data()->is_layout_set()) {
model_layout = input->get_preprocess()->propagate_layout(input->get_tensor_data()->get_layout());
}
if (input->get_tensor_data()->is_layout_set() && !model_layout.empty() &&
model_layout != input->get_tensor_data()->get_layout()) {
auto sq_layout = Layout();
// Find if some squeeze is needed between model and tensor
// E.g. model=NCHW, tensor=HWC
std::tie(new_param_shape, sq_layout) =
layout::utils::find_squeeze(model_layout, net_shape, input->get_tensor_data()->get_layout());
// Find transpose between model and tensor layouts and update tensor shape
auto net_to_tensor =
layout::utils::find_permutation(sq_layout, new_param_shape, input->get_tensor_data()->get_layout());
if (!net_to_tensor.empty() && new_param_shape.rank().is_static()) {
std::vector<ov::Dimension> dims(new_param_shape.size());
std::transform(net_to_tensor.begin(), net_to_tensor.end(), dims.begin(), [&](int64_t v) {
return new_param_shape[v];
});
new_param_shape = PartialShape(dims);
}
} else {
Layout new_layout;
std::tie(new_param_shape, new_layout) =
input->get_preprocess()->calculate_param_shape(new_param_shape, model_layout);
if (!input->get_tensor_data()->is_layout_set()) {
// Reusing param's layout according to converted calculated layout
input->get_tensor_data()->set_layout(new_layout);
}
}
if (input->get_tensor_data()->is_shape_set()) {
new_param_shape = input->get_tensor_data()->get_shape();
} else if (input->get_tensor_data()->is_spatial_shape_set()) {
auto height_idx = get_and_check_height_idx(input->get_tensor_data()->get_layout(), new_param_shape);
auto width_idx = get_and_check_width_idx(input->get_tensor_data()->get_layout(), new_param_shape);
if (input->get_tensor_data()->is_spatial_shape_dynamic()) {
// Use dynamic spatial dimensions
new_param_shape[height_idx] = Dimension::dynamic();
new_param_shape[width_idx] = Dimension::dynamic();
} else {
// Use static spatial dimensions
new_param_shape[height_idx] = input->get_tensor_data()->get_spatial_height();
new_param_shape[width_idx] = input->get_tensor_data()->get_spatial_width();
}
}
std::vector<Output<Node>> nodes;
std::vector<std::shared_ptr<op::v0::Parameter>> new_params;
// Create separate parameter for each plane. Shape is based on color format
for (size_t plane = 0; plane < color_info->planes_count(); plane++) {
auto plane_shape = color_info->shape(plane, new_param_shape);
auto plane_param =
std::make_shared<op::v0::Parameter>(input->get_tensor_data()->get_element_type(), plane_shape);
if (plane < input->get_tensor_data()->planes_sub_names().size()) {
std::unordered_set<std::string> plane_tensor_names;
std::string sub_name;
sub_name = std::string("/") + input->get_tensor_data()->planes_sub_names()[plane];
if (!std::get<1>(existing_names)) {
existing_names = std::make_tuple(get_function_tensor_names(function), true);
}
for (const auto& tensor_name : param->get_default_output().get_tensor().get_names()) {
auto new_name = tensor_name + sub_name;
OPENVINO_ASSERT(
std::get<0>(existing_names).count(new_name) == 0,
"Error while trying to create plane input with name '",
new_name,
"' - name already exists in model. Please specify another sub-name for set_color_format");
plane_tensor_names.insert(new_name);
}
plane_param->get_default_output().get_tensor().set_names(plane_tensor_names);
plane_param->set_friendly_name(param->get_friendly_name() + sub_name);
} else if (color_info->planes_count() == 1) {
plane_param->get_default_output().get_tensor().set_names(
param->get_default_output().get_tensor().get_names());
plane_param->set_friendly_name(param->get_friendly_name());
}
// Fill runtime info
plane_param->get_rt_info() = param->get_rt_info();
plane_param->output(0).get_rt_info() = param->output(0).get_rt_info();
if (!input->get_tensor_data()->get_layout().empty()) {
plane_param->set_layout(input->get_tensor_data()->get_layout());
}
if (input->get_tensor_data()->is_memory_type_set()) {
if (input->get_tensor_data()->get_memory_type().empty()) {
plane_param->output(0).get_rt_info().erase(TensorInfoMemoryType::get_type_info_static());
} else {
plane_param->output(0).get_rt_info()[TensorInfoMemoryType::get_type_info_static()] =
TensorInfoMemoryType(input->get_tensor_data()->get_memory_type());
}
}
new_params.push_back(plane_param);
nodes.emplace_back(plane_param);
}
PreprocessingContext context(input->get_tensor_data()->get_layout());
context.color_format() = input->get_tensor_data()->get_color_format();
context.target_layout() = model_layout;
context.model_shape() = param->get_partial_shape();
context.target_element_type() = param->get_element_type();
// 2. Apply preprocessing
for (const auto& action : input->get_preprocess()->actions()) {
auto action_result = action(nodes, function, context);
nodes = std::get<0>(action_result);
tensor_data_updated |= std::get<1>(action_result);
}
OPENVINO_ASSERT(nodes.size() == 1,
"Multiple plane input is not allowed as model input. Consider using of convert_color "
"preprocessing operation. Current format is '",
color_format_name(context.color_format()),
"'");
OPENVINO_ASSERT(is_rgb_family(context.color_format()) || context.color_format() == ColorFormat::UNDEFINED,
"model shall have RGB/BGR color format. Consider add 'convert_color' preprocessing operation "
"to convert current color format '",
color_format_name(context.color_format()),
"'to RGB/BGR");
// Implicit: Convert element type + layout to user's tensor implicitly
PreStepsList implicit_steps;
implicit_steps.add_convert_impl(param->get_element_type());
if (!context.target_layout().empty()) {
implicit_steps.add_convert_layout_impl(context.target_layout());
}
for (const auto& action : implicit_steps.actions()) {
auto action_result = action(nodes, function, context);
nodes = std::get<0>(action_result);
}
auto node = nodes[0];
if (node.get_partial_shape() != param->get_partial_shape()) {
tensor_data_updated = true; // Trigger revalidation if input parameter shape is changed
}
// Check final shape
OPENVINO_ASSERT(node.get_partial_shape().compatible(param->get_partial_shape()),
"Resulting shape '",
node.get_partial_shape(),
"' after preprocessing is not aligned with original parameter's shape: ",
param->get_partial_shape(),
", input parameter: ",
param->get_friendly_name());
// Replace parameter
for (auto consumer : consumers) {
consumer.replace_source_output(node);
}
{
auto param_it = std::find(parameters_list.begin(), parameters_list.end(), param);
OPENVINO_ASSERT(param_it != parameters_list.end(),
"Parameter to replace has been replaced by previous steps of preprocessing. Use only one "
"InputInfo for one input parameter");
// Insert list of new parameters to the place of original parameter
param_it = parameters_list.erase(param_it);
parameters_list.insert(param_it, new_params.begin(), new_params.end());
}
} }
// Add parameters with right order // Add parameters with right order
@ -560,85 +181,13 @@ std::shared_ptr<Model> PrePostProcessor::build() {
function->add_parameters(parameters_vec); function->add_parameters(parameters_vec);
} }
// Validate nodes after preprocessing if needed (no need to repeat it after post-processing) // Validate nodes after preprocessing if needed (no need to repeat it after post-processing)
if (tensor_data_updated) { if (need_validate) {
function->validate_nodes_and_infer_types(); function->validate_nodes_and_infer_types();
} }
// Post processing // Post processing
for (const auto& output_info : m_impl->m_outputs) { for (const auto& output_info : m_impl->m_outputs) {
const auto& output = output_info.m_impl; output_info.m_impl->build(results);
std::shared_ptr<op::v0::Result> result;
Output<Node> node = output->m_output_node;
auto start_out_node_names = node.get_tensor().get_names();
node.get_tensor().set_names({});
result = std::dynamic_pointer_cast<op::v0::Result>(node.get_node_shared_ptr());
// Set result layout from 'model' information
if (output->get_model_data()->is_layout_set()) {
// Overwrite existing model's layout here (fix 74065)
result->set_layout(output->get_model_data()->get_layout());
}
auto parent = result->get_input_source_output(0);
PostprocessingContext context(result->get_layout());
if (output->get_tensor_data()->is_layout_set()) {
context.target_layout() = output->get_tensor_data()->get_layout();
}
if (output->get_tensor_data()->is_element_type_set()) {
context.target_element_type() = output->get_tensor_data()->get_element_type();
}
// Apply post-processing
node = result->get_input_source_output(0);
bool post_processing_applied = false;
for (const auto& action : output->get_postprocess()->actions()) {
auto action_result = action({node}, context);
node = std::get<0>(action_result);
post_processing_applied = true;
}
// Implicit: Convert element type + layout to user's tensor implicitly
PostStepsList implicit_steps;
if (node.get_element_type() != output->get_tensor_data()->get_element_type() &&
output->get_tensor_data()->is_element_type_set() && node.get_element_type() != element::dynamic) {
implicit_steps.add_convert_impl(output->get_tensor_data()->get_element_type());
}
if (!context.target_layout().empty() && context.target_layout() != context.layout()) {
implicit_steps.add_convert_layout_impl(context.target_layout());
}
for (const auto& action : implicit_steps.actions()) {
auto action_result = action({node}, context);
node = std::get<0>(action_result);
post_processing_applied = true;
}
node.get_node_shared_ptr()->set_friendly_name(
result->get_input_source_output(0).get_node_shared_ptr()->get_friendly_name());
// Reset friendly name of input node to avoid names collision
// when there is at a new node inserted by post-processing steps
// If no new nodes are inserted by post-processing, then we need to preserve friendly name of input
// as it's required for old API correct work
if (post_processing_applied)
result->get_input_source_output(0).get_node_shared_ptr()->set_friendly_name("");
// Create result
auto new_result = std::make_shared<ov::op::v0::Result>(node);
new_result->set_friendly_name(result->get_friendly_name());
node.get_tensor().set_names(start_out_node_names);
// Preserve runtime info of original result
new_result->get_rt_info() = result->get_rt_info();
new_result->input(0).get_rt_info() = result->input(0).get_rt_info();
new_result->output(0).get_rt_info() = result->output(0).get_rt_info();
// Update layout
if (!context.layout().empty()) {
new_result->set_layout(context.layout());
}
for (auto& old_result : results) {
if (result == old_result) {
old_result = new_result;
break;
}
}
} }
// Add results with right order // Add results with right order
while (!function->get_results().empty()) while (!function->get_results().empty())
@ -758,14 +307,16 @@ PreProcessSteps& PreProcessSteps::convert_color(const ov::preprocess::ColorForma
PreProcessSteps& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb) { PreProcessSteps& PreProcessSteps::custom(const CustomPreprocessOp& preprocess_cb) {
// 'true' indicates that custom preprocessing step will trigger validate_and_infer_types // 'true' indicates that custom preprocessing step will trigger validate_and_infer_types
m_impl->actions().emplace_back([preprocess_cb](const std::vector<Output<Node>>& nodes, m_impl->actions().emplace_back(
const std::shared_ptr<ov::Model>&, [preprocess_cb](const std::vector<Output<Node>>& nodes,
PreprocessingContext&) { const std::shared_ptr<ov::Model>&,
OPENVINO_ASSERT(nodes.size() == 1, PreprocessingContext&) {
"Can't apply custom preprocessing step for multi-plane input. Suggesting to convert " OPENVINO_ASSERT(nodes.size() == 1,
"current image to RGB/BGR color format using 'convert_color'"); "Can't apply custom preprocessing step for multi-plane input. Suggesting to convert "
return std::make_tuple(std::vector<Output<Node>>{preprocess_cb(nodes[0])}, true); "current image to RGB/BGR color format using 'convert_color'");
}); return std::make_tuple(std::vector<Output<Node>>{preprocess_cb(nodes[0])}, true);
},
"custom");
return *this; return *this;
} }
@ -819,9 +370,11 @@ PostProcessSteps& PostProcessSteps::convert_layout(const std::vector<uint64_t>&
PostProcessSteps& PostProcessSteps::custom(const CustomPostprocessOp& postprocess_cb) { PostProcessSteps& PostProcessSteps::custom(const CustomPostprocessOp& postprocess_cb) {
// 'true' indicates that custom postprocessing step will trigger validate_and_infer_types // 'true' indicates that custom postprocessing step will trigger validate_and_infer_types
m_impl->actions().emplace_back([postprocess_cb](const Output<ov::Node>& node, PostprocessingContext&) { m_impl->actions().emplace_back(
return std::make_tuple(postprocess_cb(node), true); [postprocess_cb](const Output<ov::Node>& node, PostprocessingContext&) {
}); return std::make_tuple(postprocess_cb(node), true);
},
"custom");
return *this; return *this;
} }

467
src/core/src/preprocess/preprocess_impls.cpp Normal file
View File

@ -0,0 +1,467 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "preprocess_impls.hpp"
#include "layout_utils.hpp"
namespace ov {
namespace preprocess {
namespace {
void dump_tensor(std::ostream& str,
const PartialShape& shape,
const Layout& layout,
const element::Type& type,
const ColorFormat& color = ColorFormat::UNDEFINED) {
str << shape << ", ";
if (layout.empty()) {
str << "<no layout>";
} else {
str << layout.to_string();
}
str << ", " << type;
if (color != ColorFormat::UNDEFINED) {
str << ", " << color_format_name(color);
}
}
} // namespace
InputInfo::InputInfoImpl::InputInfoData InputInfo::InputInfoImpl::create_new_params(
std::tuple<std::unordered_set<std::string>, bool>& existing_names,
const std::shared_ptr<Model>& model) const {
InputInfoData res;
res.m_param = m_resolved_param;
auto tensor_elem_type = get_tensor_data()->is_element_type_set() ? get_tensor_data()->get_element_type()
: res.m_param->get_element_type();
res.m_tensor_layout = get_tensor_data()->get_layout();
auto color_info = ColorFormatInfo::get(get_tensor_data()->get_color_format());
if (!get_tensor_data()->is_layout_set()) {
if (!color_info->default_layout().empty()) {
res.m_tensor_layout = color_info->default_layout();
}
}
auto model_shape = res.m_param->get_partial_shape();
auto new_param_shape = model_shape;
if (get_tensor_data()->is_shape_set()) {
new_param_shape = get_tensor_data()->get_shape();
}
res.m_model_layout = get_model()->is_layout_set() ? get_model()->get_layout() : res.m_param->get_layout();
if (res.m_model_layout.empty() && get_tensor_data()->is_layout_set()) {
res.m_model_layout = get_preprocess()->propagate_layout(res.m_tensor_layout);
}
if (!res.m_tensor_layout.empty() && !res.m_model_layout.empty() && res.m_model_layout != res.m_tensor_layout) {
auto sq_layout = Layout();
// Find if some squeeze is needed between model and tensor
// E.g. model=NCHW, tensor=HWC
std::tie(new_param_shape, sq_layout) =
layout::utils::find_squeeze(res.m_model_layout, model_shape, res.m_tensor_layout);
// Find transpose between model and tensor layouts and update tensor shape
auto net_to_tensor = layout::utils::find_permutation(sq_layout, new_param_shape, res.m_tensor_layout);
if (!net_to_tensor.empty() && new_param_shape.rank().is_static()) {
std::vector<ov::Dimension> dims(new_param_shape.size());
std::transform(net_to_tensor.begin(), net_to_tensor.end(), dims.begin(), [&](int64_t v) {
return new_param_shape[v];
});
new_param_shape = PartialShape(dims);
}
} else {
Layout new_layout;
std::tie(new_param_shape, new_layout) =
get_preprocess()->calculate_param_shape(new_param_shape, res.m_model_layout);
if (res.m_tensor_layout.empty()) {
// Reusing param's layout according to converted calculated layout
res.m_tensor_layout = new_layout;
}
}
if (get_tensor_data()->is_shape_set()) {
new_param_shape = get_tensor_data()->get_shape();
} else if (get_tensor_data()->is_spatial_shape_set()) {
auto height_idx = get_and_check_height_idx(res.m_tensor_layout, new_param_shape);
auto width_idx = get_and_check_width_idx(res.m_tensor_layout, new_param_shape);
if (get_tensor_data()->is_spatial_shape_dynamic()) {
// Use dynamic spatial dimensions
new_param_shape[height_idx] = Dimension::dynamic();
new_param_shape[width_idx] = Dimension::dynamic();
} else {
// Use static spatial dimensions
new_param_shape[height_idx] = get_tensor_data()->get_spatial_height();
new_param_shape[width_idx] = get_tensor_data()->get_spatial_width();
}
}
// Create separate parameter for each plane. Shape is based on color format
for (size_t plane = 0; plane < color_info->planes_count(); plane++) {
auto plane_shape = color_info->shape(plane, new_param_shape);
auto plane_param = std::make_shared<opset8::Parameter>(tensor_elem_type, plane_shape);
if (plane < get_tensor_data()->planes_sub_names().size()) {
std::unordered_set<std::string> plane_tensor_names;
std::string sub_name;
sub_name = std::string("/") + get_tensor_data()->planes_sub_names()[plane];
if (!std::get<1>(existing_names)) {
existing_names = std::make_tuple(get_function_tensor_names(model), true);
}
for (const auto& tensor_name : res.m_param->get_default_output().get_tensor().get_names()) {
auto new_name = tensor_name + sub_name;
OPENVINO_ASSERT(
std::get<0>(existing_names).count(new_name) == 0,
"Error while trying to create plane input with name '",
new_name,
"' - name already exists in model. Please specify another sub-name for set_color_format");
plane_tensor_names.insert(new_name);
}
plane_param->get_default_output().get_tensor().set_names(plane_tensor_names);
plane_param->set_friendly_name(res.m_param->get_friendly_name() + sub_name);
} else if (color_info->planes_count() == 1) {
plane_param->get_default_output().get_tensor().set_names(
res.m_param->get_default_output().get_tensor().get_names());
plane_param->set_friendly_name(res.m_param->get_friendly_name());
}
// Fill runtime info
plane_param->get_rt_info() = res.m_param->get_rt_info();
plane_param->output(0).get_rt_info() = res.m_param->output(0).get_rt_info();
if (!res.m_tensor_layout.empty()) {
plane_param->set_layout(res.m_tensor_layout);
}
if (get_tensor_data()->is_memory_type_set()) {
if (get_tensor_data()->get_memory_type().empty()) {
plane_param->output(0).get_rt_info().erase(TensorInfoMemoryType::get_type_info_static());
} else {
plane_param->output(0).get_rt_info()[TensorInfoMemoryType::get_type_info_static()] =
TensorInfoMemoryType(get_tensor_data()->get_memory_type());
}
}
res.m_new_params.push_back(plane_param);
}
return res;
}
PreStepsList InputInfo::InputInfoImpl::create_implicit_steps(const PreprocessingContext& context, element::Type type) {
PreStepsList implicit_steps;
if (type != context.target_element_type()) {
implicit_steps.add_convert_impl(context.target_element_type());
}
if (!context.target_layout().empty() && context.target_layout() != context.layout()) {
implicit_steps.add_convert_layout_impl(context.target_layout());
}
return implicit_steps;
}
bool InputInfo::InputInfoImpl::build(const std::shared_ptr<Model>& model,
std::tuple<std::unordered_set<std::string>, bool>& existing_names,
std::list<std::shared_ptr<opset8::Parameter>>& parameters_list) {
auto data = create_new_params(existing_names, model);
auto consumers = data.m_param->output(0).get_target_inputs();
bool need_validate = false;
PreprocessingContext context(data.m_tensor_layout);
context.color_format() = get_tensor_data()->get_color_format();
context.target_layout() = data.m_model_layout;
context.model_shape() = data.m_param->get_partial_shape();
context.target_element_type() = data.m_param->get_element_type();
// 2. Apply preprocessing
auto nodes = data.as_nodes();
for (const auto& action : get_preprocess()->actions()) {
auto action_result = action.m_op(nodes, model, context);
nodes = std::get<0>(action_result);
need_validate |= std::get<1>(action_result);
}
OPENVINO_ASSERT(nodes.size() == 1,
"Multiple plane input is not allowed as model input. Consider using of convert_color "
"preprocessing operation. Current format is '",
color_format_name(context.color_format()),
"'");
OPENVINO_ASSERT(is_rgb_family(context.color_format()) || context.color_format() == ColorFormat::UNDEFINED,
"model shall have RGB/BGR color format. Consider add 'convert_color' preprocessing operation "
"to convert current color format '",
color_format_name(context.color_format()),
"'to RGB/BGR");
// Implicit: Convert element type + layout to user's tensor implicitly
auto implicit_steps = create_implicit_steps(context, nodes[0].get_element_type());
for (const auto& action : implicit_steps.actions()) {
auto action_result = action.m_op(nodes, model, context);
nodes = std::get<0>(action_result);
}
auto node = nodes[0];
if (node.get_partial_shape() != context.model_shape()) {
need_validate = true; // Trigger revalidation if input parameter shape is changed
}
// Check final shape
OPENVINO_ASSERT(node.get_partial_shape().compatible(context.model_shape()),
"Resulting shape '",
node.get_partial_shape(),
"' after preprocessing is not aligned with original parameter's shape: ",
context.model_shape(),
", input parameter: ",
data.m_param->get_friendly_name());
// Replace parameter
for (auto consumer : consumers) {
if (dynamic_cast<ov::opset8::Result*>(consumer.get_node())) {
// Some result points to old parameter (Param->Result case), need to trigger revalidation
need_validate = true;
}
consumer.replace_source_output(node);
}
{
auto param_it = std::find(parameters_list.begin(), parameters_list.end(), data.m_param);
OPENVINO_ASSERT(param_it != parameters_list.end(),
"Parameter to replace has been replaced by previous steps of preprocessing. Use only one "
"InputInfo for one input parameter");
// Insert list of new parameters to the place of original parameter
param_it = parameters_list.erase(param_it);
parameters_list.insert(param_it, data.m_new_params.begin(), data.m_new_params.end());
}
return need_validate;
}
void InputInfo::InputInfoImpl::dump(std::ostream& str,
const std::shared_ptr<Model>& model,
std::tuple<std::unordered_set<std::string>, bool>& existing_names) const {
auto data = create_new_params(existing_names, model);
auto nodes = data.as_nodes();
PreprocessingContext context(data.m_tensor_layout);
context.color_format() = get_tensor_data()->get_color_format();
context.target_layout() = data.m_model_layout;
context.model_shape() = data.m_param->get_partial_shape();
context.target_element_type() = data.m_param->get_element_type();
bool need_dump = nodes.size() > 1 || nodes[0].get_partial_shape() != context.model_shape() ||
data.m_param->get_layout() != context.target_layout() ||
nodes[0].get_element_type() != context.target_element_type() ||
get_tensor_data()->is_memory_type_set() || !get_preprocess()->actions().empty();
if (!need_dump) {
return;
}
// Dump tensor and model shapes if any preprocessing is needed
str << "Input ";
if (!data.m_param->output(0).get_names().empty()) {
str << "\"" << data.m_param->output(0).get_any_name() << "\"";
}
if (context.color_format() != ColorFormat::UNDEFINED) {
str << " (color " << color_format_name(context.color_format()) << ")";
}
if (get_tensor_data()->is_memory_type_set()) {
str << " memory type=" << get_tensor_data()->get_memory_type();
}
str << ":" << std::endl;
if (nodes.size() == 1) {
str << " User's input tensor: ";
dump_tensor(str, nodes[0].get_partial_shape(), context.layout(), nodes[0].get_element_type());
str << std::endl;
} else {
str << " " << nodes.size() << " user's tensors expected for each plane:" << std::endl;
for (size_t i = 0; i < nodes.size(); i++) {
str << " " << i << ": ";
if (!nodes[i].get_names().empty()) {
str << nodes[i].get_any_name() << " ";
}
dump_tensor(str, nodes[i].get_partial_shape(), context.layout(), nodes[i].get_element_type());
str << std::endl;
}
}
str << " Model's expected tensor: ";
dump_tensor(str, context.model_shape(), context.target_layout(), context.target_element_type());
str << std::endl;
// Apply and dump preprocessing operations
if (!get_preprocess()->actions().empty()) {
str << " Pre-processing steps (" << get_preprocess()->actions().size() << "):" << std::endl;
}
for (const auto& action : get_preprocess()->actions()) {
str << " " << action.m_name << ": (";
dump_tensor(str,
nodes[0].get_partial_shape(),
context.layout(),
nodes[0].get_element_type(),
context.color_format());
auto action_result = action.m_op(nodes, model, context);
nodes = std::get<0>(action_result);
str << ") -> (";
dump_tensor(str,
nodes[0].get_partial_shape(),
context.layout(),
nodes[0].get_element_type(),
context.color_format());
str << ")" << std::endl;
}
// Implicit: Convert element type + layout to user's tensor implicitly
auto implicit_steps = create_implicit_steps(context, nodes[0].get_element_type());
if (!implicit_steps.actions().empty()) {
str << " Implicit pre-processing steps (" << implicit_steps.actions().size() << "):" << std::endl;
}
for (const auto& action : implicit_steps.actions()) {
str << " " << action.m_name << ": (";
dump_tensor(str,
nodes[0].get_partial_shape(),
context.layout(),
nodes[0].get_element_type(),
context.color_format());
auto action_result = action.m_op(nodes, model, context);
nodes = std::get<0>(action_result);
str << ") -> (";
dump_tensor(str,
nodes[0].get_partial_shape(),
context.layout(),
nodes[0].get_element_type(),
context.color_format());
str << ")" << std::endl;
}
}
//----------- OutputInfoImpl ----------
void OutputInfo::OutputInfoImpl::build(ov::ResultVector& results) {
std::shared_ptr<opset8::Result> result;
auto node = m_output_node;
auto start_out_node_names = node.get_tensor().get_names();
node.get_tensor().set_names({});
result = std::dynamic_pointer_cast<opset8::Result>(node.get_node_shared_ptr());
// Set result layout from 'model' information
if (get_model_data()->is_layout_set()) {
// Overwrite existing model's layout here (fix 74065)
result->set_layout(get_model_data()->get_layout());
}
PostprocessingContext context(result->get_layout());
if (get_tensor_data()->is_layout_set()) {
context.target_layout() = get_tensor_data()->get_layout();
}
if (get_tensor_data()->is_element_type_set()) {
context.target_element_type() = get_tensor_data()->get_element_type();
}
// Apply post-processing
node = result->get_input_source_output(0);
bool post_processing_applied = false;
for (const auto& action : get_postprocess()->actions()) {
auto action_result = action.m_op({node}, context);
node = std::get<0>(action_result);
post_processing_applied = true;
}
// Implicit: Convert element type + layout to user's tensor implicitly
PostStepsList implicit_steps;
if (node.get_element_type() != get_tensor_data()->get_element_type() && get_tensor_data()->is_element_type_set() &&
node.get_element_type() != element::dynamic) {
implicit_steps.add_convert_impl(get_tensor_data()->get_element_type());
}
if (!context.target_layout().empty() && context.target_layout() != context.layout()) {
implicit_steps.add_convert_layout_impl(context.target_layout());
}
for (const auto& action : implicit_steps.actions()) {
auto action_result = action.m_op({node}, context);
node = std::get<0>(action_result);
post_processing_applied = true;
}
node.get_node_shared_ptr()->set_friendly_name(
result->get_input_source_output(0).get_node_shared_ptr()->get_friendly_name());
// Reset friendly name of input node to avoid names collision
// when there is at a new node inserted by post-processing steps
// If no new nodes are inserted by post-processing, then we need to preserve friendly name of input
// as it's required for old API correct work
if (post_processing_applied)
result->get_input_source_output(0).get_node_shared_ptr()->set_friendly_name("");
// Create result
auto new_result = std::make_shared<opset8::Result>(node);
new_result->set_friendly_name(result->get_friendly_name());
node.get_tensor().set_names(start_out_node_names);
// Preserve runtime info of original result
new_result->get_rt_info() = result->get_rt_info();
new_result->input(0).get_rt_info() = result->input(0).get_rt_info();
new_result->output(0).get_rt_info() = result->output(0).get_rt_info();
// Update layout
if (!context.layout().empty()) {
new_result->set_layout(context.layout());
}
for (auto& old_result : results) {
if (result == old_result) {
old_result = new_result;
break;
}
}
}
void OutputInfo::OutputInfoImpl::dump(std::ostream& str) const {
std::shared_ptr<opset8::Result> result;
auto node = m_output_node;
auto start_out_node_names = node.get_tensor().get_names();
result = std::dynamic_pointer_cast<opset8::Result>(node.get_node_shared_ptr());
auto model_layout = get_model_data()->is_layout_set() ? get_model_data()->get_layout() : result->get_layout();
PostprocessingContext context(model_layout);
if (get_tensor_data()->is_layout_set()) {
context.target_layout() = get_tensor_data()->get_layout();
}
if (get_tensor_data()->is_element_type_set()) {
context.target_element_type() = get_tensor_data()->get_element_type();
}
bool need_dump =
(model_layout != context.target_layout() && get_tensor_data()->is_layout_set()) ||
(node.get_element_type() != context.target_element_type() && get_tensor_data()->is_element_type_set()) ||
!get_postprocess()->actions().empty();
if (!need_dump) {
return;
}
str << "Output ";
if (!start_out_node_names.empty()) {
str << "\"" << *start_out_node_names.begin() << "\"";
}
str << ":" << std::endl;
str << " Model's data tensor: ";
dump_tensor(str, node.get_partial_shape(), model_layout, node.get_element_type());
str << std::endl;
if (!get_postprocess()->actions().empty()) {
str << " Post-processing steps (" << get_postprocess()->actions().size() << "):" << std::endl;
}
// Apply post-processing
node = result->get_input_source_output(0);
for (const auto& action : get_postprocess()->actions()) {
str << " " << action.m_name << ": (";
dump_tensor(str, node.get_partial_shape(), context.layout(), node.get_element_type());
auto action_result = action.m_op({node}, context);
node = std::get<0>(action_result);
str << ") -> (";
dump_tensor(str, node.get_partial_shape(), context.layout(), node.get_element_type());
str << ")" << std::endl;
}
// Implicit: Convert element type + layout to user's tensor implicitly
PostStepsList implicit_steps;
if (node.get_element_type() != get_tensor_data()->get_element_type() && get_tensor_data()->is_element_type_set() &&
node.get_element_type() != element::dynamic) {
implicit_steps.add_convert_impl(get_tensor_data()->get_element_type());
}
if (!context.target_layout().empty() && context.target_layout() != context.layout()) {
implicit_steps.add_convert_layout_impl(context.target_layout());
}
if (!implicit_steps.actions().empty()) {
str << " Post-processing implicit steps (" << implicit_steps.actions().size() << "):" << std::endl;
}
for (const auto& action : implicit_steps.actions()) {
str << " " << action.m_name << ": (";
dump_tensor(str, node.get_partial_shape(), context.layout(), node.get_element_type());
auto action_result = action.m_op({node}, context);
node = std::get<0>(action_result);
str << ") -> (";
dump_tensor(str, node.get_partial_shape(), context.layout(), node.get_element_type());
str << ")" << std::endl;
}
str << " User's output tensor: ";
dump_tensor(str, node.get_partial_shape(), context.layout(), node.get_element_type());
}
} // namespace preprocess
} // namespace ov

290
src/core/src/preprocess/preprocess_impls.hpp Normal file
View File

@ -0,0 +1,290 @@
// Copyright (C) 2018-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <openvino/core/preprocess/input_info.hpp>
#include <openvino/core/preprocess/output_info.hpp>
#include <openvino/opsets/opset8.hpp>
#include "color_utils.hpp"
#include "preprocess_steps_impl.hpp"
namespace ov {
namespace preprocess {
/// \brief ModelInfoImpl - internal data structure
class ModelInfoImpl {
public:
ModelInfoImpl() = default;
void set_layout(const Layout& layout) {
m_layout = layout;
m_layout_set = true;
}
bool is_layout_set() const {
return m_layout_set;
}
const Layout& get_layout() const {
return m_layout;
}
private:
Layout m_layout = Layout();
bool m_layout_set = false;
};
class InputModelInfo::InputModelInfoImpl : public ModelInfoImpl {};
class OutputModelInfo::OutputModelInfoImpl : public ModelInfoImpl {};
/// \brief OutputInfoImpl - internal data structure
struct OutputInfo::OutputInfoImpl {
OutputInfoImpl() = default;
std::unique_ptr<OutputTensorInfo::OutputTensorInfoImpl>& get_tensor_data() {
return m_tensor_info.m_impl;
}
const std::unique_ptr<OutputTensorInfo::OutputTensorInfoImpl>& get_tensor_data() const {
return m_tensor_info.m_impl;
}
std::unique_ptr<PostProcessSteps::PostProcessStepsImpl>& get_postprocess() {
return m_postprocess.m_impl;
}
const std::unique_ptr<PostProcessSteps::PostProcessStepsImpl>& get_postprocess() const {
return m_postprocess.m_impl;
}
std::unique_ptr<OutputModelInfo::OutputModelInfoImpl>& get_model_data() {
return m_model_info.m_impl;
}
const std::unique_ptr<OutputModelInfo::OutputModelInfoImpl>& get_model_data() const {
return m_model_info.m_impl;
}
void build(ov::ResultVector& results);
void dump(std::ostream& str) const;
OutputTensorInfo m_tensor_info;
PostProcessSteps m_postprocess;
OutputModelInfo m_model_info;
ov::Output<ov::Node> m_output_node;
};
class TensorInfoImplBase {
public:
TensorInfoImplBase() = default;
void set_element_type(const element::Type& type) {
m_type = type;
m_type_set = true;
}
bool is_element_type_set() const {
return m_type_set;
}
const element::Type& get_element_type() const {
return m_type;
}
void set_layout(const Layout& layout) {
m_layout = layout;
m_layout_set = true;
}
bool is_layout_set() const {
return m_layout_set;
}
const Layout& get_layout() const {
return m_layout;
}
protected:
element::Type m_type = element::dynamic;
bool m_type_set = false;
Layout m_layout = Layout();
bool m_layout_set = false;
};
class OutputTensorInfo::OutputTensorInfoImpl : public TensorInfoImplBase {};
/// \brief InputTensorInfoImpl - internal data structure
class InputTensorInfo::InputTensorInfoImpl : public TensorInfoImplBase {
public:
InputTensorInfoImpl() = default;
bool is_spatial_shape_set() const {
return m_spatial_shape_set;
}
int get_spatial_width() const {
return m_spatial_width;
}
int get_spatial_height() const {
return m_spatial_height;
}
bool is_spatial_shape_dynamic() const {
return m_spatial_shape_set && m_spatial_width == -1 && m_spatial_height == -1;
}
void set_spatial_dynamic_shape() {
OPENVINO_ASSERT(!m_shape_set, "'set_spatial_dynamic_shape' and 'set_shape' shall not be used together");
m_spatial_shape_set = true;
m_spatial_width = -1;
m_spatial_height = -1;
}
void set_spatial_static_shape(size_t height, size_t width) & {
OPENVINO_ASSERT(!m_shape_set, "'set_spatial_static_shape' and 'set_shape' shall not be used together");
m_spatial_shape_set = true;
m_spatial_height = static_cast<int>(height);
m_spatial_width = static_cast<int>(width);
}
const ColorFormat& get_color_format() const {
return m_color_format;
}
void set_color_format(ColorFormat format, const std::vector<std::string>& sub_names) {
auto info = ColorFormatInfo::get(format);
if (info->planes_count() == 1) {
OPENVINO_ASSERT(sub_names.empty(),
"Plane names are not allowed for single plane color format '",
color_format_name(format),
"'");
} else if (!sub_names.empty()) {
OPENVINO_ASSERT(sub_names.size() == info->planes_count(),
"Number of sub-names (",
sub_names.size(),
") shall match with number of planes for '",
color_format_name(format),
"' color format (",
info->planes_count(),
")");
}
m_planes_sub_names = sub_names;
m_color_format = format;
}
const std::vector<std::string>& planes_sub_names() const {
return m_planes_sub_names;
}
void set_memory_type(const std::string& mem_type) {
m_memory_type_set = true;
m_memory_type = mem_type;
}
const std::string& get_memory_type() const {
return m_memory_type;
}
bool is_memory_type_set() const {
return m_memory_type_set;
}
void set_shape(const PartialShape& shape) {
OPENVINO_ASSERT(
!m_spatial_shape_set,
"'set_spatial_static_shape', 'set_spatial_dynamic_shape', 'set_shape' shall not be used together");
m_shape = shape;
m_shape_set = true;
}
bool is_shape_set() const {
return m_shape_set;
}
const PartialShape& get_shape() const {
return m_shape;
}
private:
ColorFormat m_color_format = ColorFormat::UNDEFINED;
std::vector<std::string> m_planes_sub_names;
element::Type m_type = element::dynamic;
bool m_type_set = false;
Layout m_layout = Layout();
bool m_layout_set = false;
int m_spatial_width = -1;
int m_spatial_height = -1;
bool m_spatial_shape_set = false;
std::string m_memory_type = {};
bool m_memory_type_set = false;
PartialShape m_shape = {};
bool m_shape_set = false;
};
/// \brief InputInfoImpl - internal data structure
struct InputInfo::InputInfoImpl {
struct InputInfoData {
std::vector<std::shared_ptr<opset8::Parameter>> m_new_params;
std::shared_ptr<opset8::Parameter> m_param;
Layout m_model_layout;
Layout m_tensor_layout;
std::vector<Output<Node>> as_nodes() const {
std::vector<Output<Node>> res;
std::transform(m_new_params.begin(),
m_new_params.end(),
std::back_inserter(res),
[](const std::shared_ptr<opset8::Parameter>& param) {
return param;
});
return res;
}
};
InputInfoImpl() = default;
std::unique_ptr<InputTensorInfo::InputTensorInfoImpl>& get_tensor_data() {
return m_tensor_info.m_impl;
}
const std::unique_ptr<InputTensorInfo::InputTensorInfoImpl>& get_tensor_data() const {
return m_tensor_info.m_impl;
}
std::unique_ptr<PreProcessSteps::PreProcessStepsImpl>& get_preprocess() {
return m_preprocess.m_impl;
}
const std::unique_ptr<PreProcessSteps::PreProcessStepsImpl>& get_preprocess() const {
return m_preprocess.m_impl;
}
const std::unique_ptr<InputModelInfo::InputModelInfoImpl>& get_model() const {
return m_model_data.m_impl;
}
InputInfoData create_new_params(std::tuple<std::unordered_set<std::string>, bool>& existing_names,
const std::shared_ptr<Model>& model) const;
static PreStepsList create_implicit_steps(const PreprocessingContext& context, element::Type type);
bool build(const std::shared_ptr<Model>& model,
std::tuple<std::unordered_set<std::string>, bool>& existing_names,
std::list<std::shared_ptr<opset8::Parameter>>& parameters_list);
void dump(std::ostream& str,
const std::shared_ptr<Model>& model,
std::tuple<std::unordered_set<std::string>, bool>& existing_names) const;
InputTensorInfo m_tensor_info;
PreProcessSteps m_preprocess;
InputModelInfo m_model_data;
std::shared_ptr<op::v0::Parameter> m_resolved_param;
};
} // namespace preprocess
} // namespace ov

696
src/core/src/preprocess/preprocess_steps_impl.cpp
View File

@ -37,140 +37,169 @@ static Shape construct_mean_scale_shape(const Output<Node>& node,
return {v}; return {v};
} }
template <typename T>
static std::string vector_to_string(const std::vector<T>& values) {
if (values.empty()) {
return {};
}
std::stringstream s;
s << "(" << values[0];
for (size_t i = 1; i < values.size(); i++) {
s << "," << values[i];
}
s << ")";
return s.str();
}
void PreStepsList::add_scale_impl(const std::vector<float>& values) { void PreStepsList::add_scale_impl(const std::vector<float>& values) {
m_actions.emplace_back([values](const std::vector<Output<Node>>& nodes, m_actions.emplace_back(
const std::shared_ptr<ov::Model>& function, [values](const std::vector<Output<Node>>& nodes,
PreprocessingContext& context) -> std::tuple<std::vector<Output<Node>>, bool> { const std::shared_ptr<ov::Model>& function,
OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't apply scale preprocessing for empty input."); PreprocessingContext& context) -> std::tuple<std::vector<Output<Node>>, bool> {
OPENVINO_ASSERT(nodes.size() == 1, OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't apply scale preprocessing for empty input.");
"Can't apply scale preprocessing for multi-plane input. Suggesting to convert current " OPENVINO_ASSERT(nodes.size() == 1,
"image to RGB/BGR color format using 'convert_color'"); "Can't apply scale preprocessing for multi-plane input. Suggesting to convert current "
Shape shape; "image to RGB/BGR color format using 'convert_color'");
if (values.size() == 1) { Shape shape;
shape = Shape{1}; if (values.size() == 1) {
} else { shape = Shape{1};
shape = construct_mean_scale_shape(nodes[0], values.size(), context); } else {
} shape = construct_mean_scale_shape(nodes[0], values.size(), context);
auto element_type = nodes[0].get_element_type(); }
OPENVINO_ASSERT(element_type.is_real(), auto element_type = nodes[0].get_element_type();
"Scale preprocessing can be applied to 'float' inputs. Consider using of " OPENVINO_ASSERT(element_type.is_real(),
"'convert_element_type' before scaling. Current type is: ", "Scale preprocessing can be applied to 'float' inputs. Consider using of "
element_type); "'convert_element_type' before scaling. Current type is: ",
element_type);
auto constant = op::v0::Constant::create(element_type, shape, values); auto constant = op::v0::Constant::create(element_type, shape, values);
auto new_op = std::make_shared<op::v1::Divide>(nodes[0], constant); auto new_op = std::make_shared<op::v1::Divide>(nodes[0], constant);
set_is_preprocessing_node(new_op); set_is_preprocessing_node(new_op);
return std::make_tuple(std::vector<Output<Node>>{new_op}, false); return std::make_tuple(std::vector<Output<Node>>{new_op}, false);
}); },
"scale " + vector_to_string(values));
} }
void PreStepsList::add_mean_impl(const std::vector<float>& values) { void PreStepsList::add_mean_impl(const std::vector<float>& values) {
m_actions.emplace_back([values](const std::vector<Output<Node>>& nodes, m_actions.emplace_back(
const std::shared_ptr<ov::Model>& function, [values](const std::vector<Output<Node>>& nodes,
PreprocessingContext& context) { const std::shared_ptr<ov::Model>& function,
OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't apply mean preprocessing for empty input."); PreprocessingContext& context) {
OPENVINO_ASSERT(nodes.size() == 1, OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't apply mean preprocessing for empty input.");
"Can't apply scale preprocessing for multi-plane input. Suggesting to convert current " OPENVINO_ASSERT(nodes.size() == 1,
"image to RGB/BGR color format using 'convert_color'"); "Can't apply scale preprocessing for multi-plane input. Suggesting to convert current "
Shape shape; "image to RGB/BGR color format using 'convert_color'");
if (values.size() == 1) { Shape shape;
shape = Shape{1}; if (values.size() == 1) {
} else { shape = Shape{1};
shape = construct_mean_scale_shape(nodes[0], values.size(), context); } else {
} shape = construct_mean_scale_shape(nodes[0], values.size(), context);
auto element_type = nodes[0].get_element_type(); }
OPENVINO_ASSERT(element_type.is_real(), auto element_type = nodes[0].get_element_type();
"Mean preprocessing can be applied to 'float' inputs. Consider using of 'convert_element_type' " OPENVINO_ASSERT(
"before scaling. Current type is: ", element_type.is_real(),
element_type); "Mean preprocessing can be applied to 'float' inputs. Consider using of 'convert_element_type' "
"before scaling. Current type is: ",
element_type);
auto constant = op::v0::Constant::create(element_type, shape, values); auto constant = op::v0::Constant::create(element_type, shape, values);
auto new_op = std::make_shared<op::v1::Subtract>(nodes[0], constant); auto new_op = std::make_shared<op::v1::Subtract>(nodes[0], constant);
set_is_preprocessing_node(new_op); set_is_preprocessing_node(new_op);
return std::make_tuple(std::vector<Output<Node>>{new_op}, false); return std::make_tuple(std::vector<Output<Node>>{new_op}, false);
}); },
"mean " + vector_to_string(values));
} }
void PreStepsList::add_convert_impl(const element::Type& type) { void PreStepsList::add_convert_impl(const element::Type& type) {
m_actions.emplace_back([type](const std::vector<Output<Node>>& nodes, m_actions.emplace_back(
const std::shared_ptr<Model>& function, [type](const std::vector<Output<Node>>& nodes,
PreprocessingContext& ctxt) { const std::shared_ptr<Model>& function,
OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't set element type for empty input."); PreprocessingContext& ctxt) {
std::vector<Output<Node>> res; OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't set element type for empty input.");
element::Type t = type; std::vector<Output<Node>> res;
if (t == element::Type{}) { element::Type t = type;
t = ctxt.target_element_type(); if (t == element::Type{}) {
} t = ctxt.target_element_type();
for (const auto& node : nodes) {
OPENVINO_ASSERT(node.get_element_type().is_static(),
"Can't insert 'convert_element_type' for dynamic source tensor type.");
if (t != node.get_element_type()) {
auto convert = std::make_shared<op::v0::Convert>(node, t);
res.emplace_back(convert);
} else {
res.emplace_back(node);
} }
} for (const auto& node : nodes) {
// return false to avoid excess function revalidations as conversion of types OPENVINO_ASSERT(node.get_element_type().is_static(),
// doesn't require shape or type propagation. "Can't insert 'convert_element_type' for dynamic source tensor type.");
return std::make_tuple(res, false); if (t != node.get_element_type()) {
}); auto convert = std::make_shared<op::v0::Convert>(node, t);
res.emplace_back(convert);
} else {
res.emplace_back(node);
}
}
// return false to avoid excess function revalidations as conversion of types
// doesn't require shape or type propagation.
return std::make_tuple(res, false);
},
"convert type (" + type.get_type_name() + ")");
} }
void PreStepsList::add_resize_impl(ResizeAlgorithm alg, int dst_height, int dst_width) { void PreStepsList::add_resize_impl(ResizeAlgorithm alg, int dst_height, int dst_width) {
using InterpolateMode = op::v4::Interpolate::InterpolateMode; using InterpolateMode = op::v4::Interpolate::InterpolateMode;
m_actions.emplace_back([alg, dst_width, dst_height](const std::vector<Output<Node>>& nodes, std::string name;
const std::shared_ptr<Model>& function, if (dst_width > 0 && dst_height > 0) {
PreprocessingContext& ctxt) { name = "resize to (" + std::to_string(dst_height) + ", " + std::to_string(dst_width) + ")";
OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't add resize for empty input."); } else {
OPENVINO_ASSERT(nodes.size() == 1, name = "resize to model width/height";
"Can't resize multi-plane input. Suggesting to convert current image to " }
"RGB/BGR color format using 'PreProcessSteps::convert_color'"); m_actions.emplace_back(
auto to_mode = [](ResizeAlgorithm alg) -> InterpolateMode { [alg, dst_width, dst_height](const std::vector<Output<Node>>& nodes,
switch (alg) { const std::shared_ptr<Model>& function,
case ResizeAlgorithm::RESIZE_NEAREST: PreprocessingContext& ctxt) {
return InterpolateMode::NEAREST; OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't add resize for empty input.");
case ResizeAlgorithm::RESIZE_CUBIC: OPENVINO_ASSERT(nodes.size() == 1,
return InterpolateMode::CUBIC; "Can't resize multi-plane input. Suggesting to convert current image to "
case ResizeAlgorithm::RESIZE_LINEAR: "RGB/BGR color format using 'PreProcessSteps::convert_color'");
default: auto to_mode = [](ResizeAlgorithm alg) -> InterpolateMode {
return InterpolateMode::LINEAR; switch (alg) {
case ResizeAlgorithm::RESIZE_NEAREST:
return InterpolateMode::NEAREST;
case ResizeAlgorithm::RESIZE_CUBIC:
return InterpolateMode::CUBIC;
case ResizeAlgorithm::RESIZE_LINEAR:
default:
return InterpolateMode::LINEAR;
}
};
auto node = nodes.front();
auto layout = ctxt.layout();
OPENVINO_ASSERT(ov::layout::has_height(layout) && ov::layout::has_width(layout),
"Can't add resize for layout without W/H specified. Use 'set_layout' API to define layout "
"of image data, like `NCHW`");
auto node_rank = node.get_partial_shape().rank();
OPENVINO_ASSERT(node_rank.is_static(), "Resize operation is not supported for fully dynamic shape");
auto height_idx = static_cast<int64_t>(get_and_check_height_idx(layout, node.get_partial_shape()));
auto width_idx = static_cast<int64_t>(get_and_check_width_idx(layout, node.get_partial_shape()));
if (dst_height < 0 || dst_width < 0) {
OPENVINO_ASSERT(ctxt.model_shape().rank().is_static(),
"Resize is not fully specified while target model shape is dynamic");
} }
}; int new_image_width = dst_width < 0 ? static_cast<int>(ctxt.get_model_width_for_resize()) : dst_width;
auto node = nodes.front(); int new_image_height = dst_height < 0 ? static_cast<int>(ctxt.get_model_height_for_resize()) : dst_height;
auto layout = ctxt.layout();
OPENVINO_ASSERT(ov::layout::has_height(layout) && ov::layout::has_width(layout),
"Can't add resize for layout without W/H specified. Use 'set_layout' API to define layout "
"of image data, like `NCHW`");
auto node_rank = node.get_partial_shape().rank();
OPENVINO_ASSERT(node_rank.is_static(), "Resize operation is not supported for fully dynamic shape");
auto height_idx = static_cast<int64_t>(get_and_check_height_idx(layout, node.get_partial_shape())); auto target_spatial_shape =
auto width_idx = static_cast<int64_t>(get_and_check_width_idx(layout, node.get_partial_shape())); op::v0::Constant::create<int64_t>(element::i64, Shape{2}, {new_image_height, new_image_width});
if (dst_height < 0 || dst_width < 0) { auto scales = op::v0::Constant::create<float>(element::f32, Shape{2}, {1, 1});
OPENVINO_ASSERT(ctxt.model_shape().rank().is_static(), // In future consider replacing this to set of new OV operations like `getDimByName(node, "H")`
"Resize is not fully specified while target model shape is dynamic"); // This is to allow specifying layout on 'evaluation' stage
} auto axes = op::v0::Constant::create<int64_t>(element::i64, Shape{2}, {height_idx, width_idx});
int new_image_width = dst_width < 0 ? static_cast<int>(ctxt.get_model_width_for_resize()) : dst_width;
int new_image_height = dst_height < 0 ? static_cast<int>(ctxt.get_model_height_for_resize()) : dst_height;
auto target_spatial_shape = op::v4::Interpolate::InterpolateAttrs attrs(to_mode(alg),
op::v0::Constant::create<int64_t>(element::i64, Shape{2}, {new_image_height, new_image_width}); op::v4::Interpolate::ShapeCalcMode::SIZES,
auto scales = op::v0::Constant::create<float>(element::f32, Shape{2}, {1, 1}); {0, 0},
// In future consider replacing this to set of new OV operations like `getDimByName(node, "H")` {0, 0});
// This is to allow specifying layout on 'evaluation' stage
auto axes = op::v0::Constant::create<int64_t>(element::i64, Shape{2}, {height_idx, width_idx});
op::v4::Interpolate::InterpolateAttrs attrs(to_mode(alg), auto interp = std::make_shared<op::v4::Interpolate>(node, target_spatial_shape, scales, axes, attrs);
op::v4::Interpolate::ShapeCalcMode::SIZES, return std::make_tuple(std::vector<Output<Node>>{interp}, true);
{0, 0}, },
{0, 0}); name);
auto interp = std::make_shared<op::v4::Interpolate>(node, target_spatial_shape, scales, axes, attrs);
return std::make_tuple(std::vector<Output<Node>>{interp}, true);
});
} }
Layout PreStepsList::propagate_layout(const Layout& tensor_layout) const { Layout PreStepsList::propagate_layout(const Layout& tensor_layout) const {
@ -185,46 +214,50 @@ void PreStepsList::add_convert_layout_impl(const Layout& layout) {
m_forward_layout_converts.clear(); m_forward_layout_converts.clear();
m_last_explicit_layout = layout; m_last_explicit_layout = layout;
m_last_explicit_layout_set = true; m_last_explicit_layout_set = true;
m_actions.emplace_back([layout](const std::vector<Output<Node>>& nodes, m_actions.emplace_back(
const std::shared_ptr<Model>& function, [layout](const std::vector<Output<Node>>& nodes,
PreprocessingContext& context) { const std::shared_ptr<Model>& function,
OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't convert layout for empty input."); PreprocessingContext& context) {
OPENVINO_ASSERT(nodes.size() == 1, OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't convert layout for empty input.");
"Can't convert layout for multi-plane input. Suggesting to convert current image to " OPENVINO_ASSERT(nodes.size() == 1,
"RGB/BGR color format using 'convert_color'"); "Can't convert layout for multi-plane input. Suggesting to convert current image to "
Layout dst_layout = layout.empty() ? context.target_layout() : layout; "RGB/BGR color format using 'convert_color'");
auto node = nodes[0]; Layout dst_layout = layout.empty() ? context.target_layout() : layout;
auto shape = node.get_partial_shape(); auto node = nodes[0];
size_t add_cnt; auto shape = node.get_partial_shape();
Layout unsqueeze_layout; size_t add_cnt;
std::tie(shape, unsqueeze_layout, add_cnt) = layout::utils::find_unsqueeze(context.layout(), shape, dst_layout); Layout unsqueeze_layout;
if (add_cnt) { std::tie(shape, unsqueeze_layout, add_cnt) =
std::vector<size_t> dims; layout::utils::find_unsqueeze(context.layout(), shape, dst_layout);
dims.push_back(add_cnt); if (add_cnt) {
Shape const_shape(dims); std::vector<size_t> dims;
std::vector<int64_t> vals(add_cnt); dims.push_back(add_cnt);
for (auto i = 0; i < add_cnt; i++) { Shape const_shape(dims);
vals[i] = i; std::vector<int64_t> vals(add_cnt);
for (auto i = 0; i < add_cnt; i++) {
vals[i] = i;
}
auto axes = op::v0::Constant::create<int64_t>(element::i64, const_shape, vals);
// Add unsqueeze on top
node = std::make_shared<opset8::Unsqueeze>(node, axes);
} }
auto axes = op::v0::Constant::create<int64_t>(element::i64, const_shape, vals); auto permutation = layout::utils::find_permutation(unsqueeze_layout, shape, dst_layout);
// Add unsqueeze on top if (permutation.empty()) {
node = std::make_shared<opset8::Unsqueeze>(node, axes); // No transpose is needed, just update layout
} if (!layout.empty()) {
auto permutation = layout::utils::find_permutation(unsqueeze_layout, shape, dst_layout); context.layout() = layout;
if (permutation.empty()) { }
// No transpose is needed, just update layout return std::make_tuple(nodes, false);
if (!layout.empty()) {
context.layout() = layout;
} }
return std::make_tuple(nodes, false); auto perm_constant =
} op::v0::Constant::create<int64_t>(element::i64, Shape{permutation.size()}, permutation);
auto perm_constant = op::v0::Constant::create<int64_t>(element::i64, Shape{permutation.size()}, permutation); auto transpose = std::make_shared<op::v1::Transpose>(node, perm_constant);
auto transpose = std::make_shared<op::v1::Transpose>(node, perm_constant); context.layout() = dst_layout; // Update context's current layout
context.layout() = dst_layout; // Update context's current layout // return false to avoid excess function revalidations as layout conversion
// return false to avoid excess function revalidations as layout conversion // doesn't require shape or type propagation.
// doesn't require shape or type propagation. return std::make_tuple(std::vector<Output<Node>>{transpose}, false);
return std::make_tuple(std::vector<Output<Node>>{transpose}, false); },
}); "convert layout " + layout.to_string());
} }
void PreStepsList::add_convert_layout_impl(const std::vector<uint64_t>& dims) { void PreStepsList::add_convert_layout_impl(const std::vector<uint64_t>& dims) {
@ -233,21 +266,23 @@ void PreStepsList::add_convert_layout_impl(const std::vector<uint64_t>& dims) {
} }
m_layout_converts.emplace_front(dims); m_layout_converts.emplace_front(dims);
m_forward_layout_converts.emplace_back(dims); m_forward_layout_converts.emplace_back(dims);
m_actions.emplace_back([dims](const std::vector<Output<Node>>& nodes, m_actions.emplace_back(
const std::shared_ptr<Model>& function, [dims](const std::vector<Output<Node>>& nodes,
PreprocessingContext& context) { const std::shared_ptr<Model>& function,
OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't convert layout for empty input."); PreprocessingContext& context) {
OPENVINO_ASSERT(nodes.size() == 1, OPENVINO_ASSERT(!nodes.empty(), "Internal error: Can't convert layout for empty input.");
"Can't convert layout for multi-plane input. Suggesting to convert current image to " OPENVINO_ASSERT(nodes.size() == 1,
"RGB/BGR color format using 'convert_color'"); "Can't convert layout for multi-plane input. Suggesting to convert current image to "
auto new_layout = layout::utils::apply_permutation(context.layout(), dims); "RGB/BGR color format using 'convert_color'");
auto perm_constant = op::v0::Constant::create<uint64_t>(element::u64, Shape{dims.size()}, dims); auto new_layout = layout::utils::apply_permutation(context.layout(), dims);
auto transpose = std::make_shared<op::v1::Transpose>(nodes[0], perm_constant); auto perm_constant = op::v0::Constant::create<uint64_t>(element::u64, Shape{dims.size()}, dims);
context.layout() = std::move(new_layout); // Update context's current layout auto transpose = std::make_shared<op::v1::Transpose>(nodes[0], perm_constant);
// return false to avoid excess function revalidations as layout conversion context.layout() = std::move(new_layout); // Update context's current layout
// doesn't require shape or type propagation. // return false to avoid excess function revalidations as layout conversion
return std::make_tuple(std::vector<Output<Node>>{transpose}, false); // doesn't require shape or type propagation.
}); return std::make_tuple(std::vector<Output<Node>>{transpose}, false);
},
"convert layout " + vector_to_string(dims));
} }
std::tuple<PartialShape, Layout> PreStepsList::calculate_param_shape(const PartialShape& model_shape, std::tuple<PartialShape, Layout> PreStepsList::calculate_param_shape(const PartialShape& model_shape,
@ -276,134 +311,144 @@ std::tuple<PartialShape, Layout> PreStepsList::calculate_param_shape(const Parti
} }
void PreStepsList::add_convert_color_impl(const ColorFormat& dst_format) { void PreStepsList::add_convert_color_impl(const ColorFormat& dst_format) {
m_actions.emplace_back([dst_format](const std::vector<Output<Node>>& nodes, m_actions.emplace_back(
const std::shared_ptr<Model>& function, [dst_format](const std::vector<Output<Node>>& nodes,
PreprocessingContext& context) { const std::shared_ptr<Model>& function,
if (context.color_format() == dst_format) { PreprocessingContext& context) {
return std::make_tuple(nodes, false); if (context.color_format() == dst_format) {
} return std::make_tuple(nodes, false);
if (context.color_format() == ColorFormat::NV12_SINGLE_PLANE) {
OPENVINO_ASSERT(nodes.size() == 1, "Internal error: single plane NV12 image can't have multiple inputs");
std::shared_ptr<Node> convert;
switch (dst_format) {
case ColorFormat::RGB:
convert = std::make_shared<op::v8::NV12toRGB>(nodes[0]);
break;
case ColorFormat::BGR:
convert = std::make_shared<op::v8::NV12toBGR>(nodes[0]);
break;
default:
OPENVINO_ASSERT(false,
"Unsupported conversion from NV12 to '",
color_format_name(dst_format),
"' format:");
} }
context.color_format() = dst_format; if (context.color_format() == ColorFormat::NV12_SINGLE_PLANE) {
return std::make_tuple(std::vector<Output<Node>>{convert}, true); OPENVINO_ASSERT(nodes.size() == 1,
} else if (context.color_format() == ColorFormat::NV12_TWO_PLANES) { "Internal error: single plane NV12 image can't have multiple inputs");
OPENVINO_ASSERT(nodes.size() == 2, "Internal error: two-plane NV12 image must have exactly two inputs"); std::shared_ptr<Node> convert;
std::shared_ptr<Node> convert; switch (dst_format) {
switch (dst_format) { case ColorFormat::RGB:
case ColorFormat::RGB: convert = std::make_shared<op::v8::NV12toRGB>(nodes[0]);
convert = std::make_shared<op::v8::NV12toRGB>(nodes[0], nodes[1]); break;
break; case ColorFormat::BGR:
case ColorFormat::BGR: convert = std::make_shared<op::v8::NV12toBGR>(nodes[0]);
convert = std::make_shared<op::v8::NV12toBGR>(nodes[0], nodes[1]); break;
break; default:
default: OPENVINO_ASSERT(false,
OPENVINO_ASSERT(false, "Unsupported conversion from NV12 to '",
"Unsupported conversion from NV12 to '", color_format_name(dst_format),
color_format_name(dst_format), "' format:");
"' format:"); }
context.color_format() = dst_format;
return std::make_tuple(std::vector<Output<Node>>{convert}, true);
} else if (context.color_format() == ColorFormat::NV12_TWO_PLANES) {
OPENVINO_ASSERT(nodes.size() == 2, "Internal error: two-plane NV12 image must have exactly two inputs");
std::shared_ptr<Node> convert;
switch (dst_format) {
case ColorFormat::RGB:
convert = std::make_shared<op::v8::NV12toRGB>(nodes[0], nodes[1]);
break;
case ColorFormat::BGR:
convert = std::make_shared<op::v8::NV12toBGR>(nodes[0], nodes[1]);
break;
default:
OPENVINO_ASSERT(false,
"Unsupported conversion from NV12 to '",
color_format_name(dst_format),
"' format:");
}
context.color_format() = dst_format;
return std::make_tuple(std::vector<Output<Node>>{convert}, true);
} else if (context.color_format() == ColorFormat::I420_SINGLE_PLANE) {
OPENVINO_ASSERT(nodes.size() == 1,
"Internal error: single plane I420 image can't have multiple inputs");
std::shared_ptr<Node> convert;
switch (dst_format) {
case ColorFormat::RGB:
convert = std::make_shared<op::v8::I420toRGB>(nodes[0]);
break;
case ColorFormat::BGR:
convert = std::make_shared<op::v8::I420toBGR>(nodes[0]);
break;
default:
OPENVINO_ASSERT(false,
"Unsupported conversion from I420 to '",
color_format_name(dst_format),
"' format:");
}
context.color_format() = dst_format;
return std::make_tuple(std::vector<Output<Node>>{convert}, true);
} else if (context.color_format() == ColorFormat::I420_THREE_PLANES) {
OPENVINO_ASSERT(nodes.size() == 3,
"Internal error: three-plane I420 image must have exactly three inputs");
std::shared_ptr<Node> convert;
switch (dst_format) {
case ColorFormat::RGB:
convert = std::make_shared<op::v8::I420toRGB>(nodes[0], nodes[1], nodes[2]);
break;
case ColorFormat::BGR:
convert = std::make_shared<op::v8::I420toBGR>(nodes[0], nodes[1], nodes[2]);
break;
default:
OPENVINO_ASSERT(false,
"Unsupported conversion from I420 to '",
color_format_name(dst_format),
"' format:");
}
context.color_format() = dst_format;
return std::make_tuple(std::vector<Output<Node>>{convert}, true);
} }
context.color_format() = dst_format; if ((context.color_format() == ColorFormat::RGB || context.color_format() == ColorFormat::BGR) &&
return std::make_tuple(std::vector<Output<Node>>{convert}, true); (dst_format == ColorFormat::RGB || dst_format == ColorFormat::BGR)) {
} else if (context.color_format() == ColorFormat::I420_SINGLE_PLANE) { auto res = reverse_channels(nodes, function, context);
OPENVINO_ASSERT(nodes.size() == 1, "Internal error: single plane I420 image can't have multiple inputs");
std::shared_ptr<Node> convert;
switch (dst_format) {
case ColorFormat::RGB:
convert = std::make_shared<op::v8::I420toRGB>(nodes[0]);
break;
case ColorFormat::BGR:
convert = std::make_shared<op::v8::I420toBGR>(nodes[0]);
break;
default:
OPENVINO_ASSERT(false,
"Unsupported conversion from I420 to '",
color_format_name(dst_format),
"' format:");
}
context.color_format() = dst_format;
return std::make_tuple(std::vector<Output<Node>>{convert}, true);
} else if (context.color_format() == ColorFormat::I420_THREE_PLANES) {
OPENVINO_ASSERT(nodes.size() == 3, "Internal error: three-plane I420 image must have exactly three inputs");
std::shared_ptr<Node> convert;
switch (dst_format) {
case ColorFormat::RGB:
convert = std::make_shared<op::v8::I420toRGB>(nodes[0], nodes[1], nodes[2]);
break;
case ColorFormat::BGR:
convert = std::make_shared<op::v8::I420toBGR>(nodes[0], nodes[1], nodes[2]);
break;
default:
OPENVINO_ASSERT(false,
"Unsupported conversion from I420 to '",
color_format_name(dst_format),
"' format:");
}
context.color_format() = dst_format;
return std::make_tuple(std::vector<Output<Node>>{convert}, true);
}
if ((context.color_format() == ColorFormat::RGB || context.color_format() == ColorFormat::BGR) &&
(dst_format == ColorFormat::RGB || dst_format == ColorFormat::BGR)) {
auto res = reverse_channels(nodes, function, context);
context.color_format() = dst_format;
return res;
}
if (context.color_format() == ColorFormat::RGBX) {
if (dst_format == ColorFormat::RGB) {
auto res = cut_last_channel(nodes, function, context);
context.color_format() = dst_format; context.color_format() = dst_format;
return res; return res;
} else if (dst_format == ColorFormat::BGR) {
auto cut = cut_last_channel(nodes, function, context);
auto reverse = reverse_channels(std::get<0>(cut), function, context);
bool updated = std::get<1>(cut) | std::get<1>(reverse);
context.color_format() = dst_format;
return std::make_tuple(std::get<0>(reverse), updated);
} }
} if (context.color_format() == ColorFormat::RGBX) {
if (context.color_format() == ColorFormat::BGRX) { if (dst_format == ColorFormat::RGB) {
if (dst_format == ColorFormat::BGR) { auto res = cut_last_channel(nodes, function, context);
auto res = cut_last_channel(nodes, function, context); context.color_format() = dst_format;
context.color_format() = dst_format; return res;
return res; } else if (dst_format == ColorFormat::BGR) {
} else if (dst_format == ColorFormat::RGB) { auto cut = cut_last_channel(nodes, function, context);
auto cut = cut_last_channel(nodes, function, context); auto reverse = reverse_channels(std::get<0>(cut), function, context);
auto reverse = reverse_channels(std::get<0>(cut), function, context); bool updated = std::get<1>(cut) | std::get<1>(reverse);
bool updated = std::get<1>(cut) | std::get<1>(reverse); context.color_format() = dst_format;
context.color_format() = dst_format; return std::make_tuple(std::get<0>(reverse), updated);
return std::make_tuple(std::get<0>(reverse), updated); }
} }
} if (context.color_format() == ColorFormat::BGRX) {
OPENVINO_ASSERT(false, if (dst_format == ColorFormat::BGR) {
"Source color format '", auto res = cut_last_channel(nodes, function, context);
color_format_name(context.color_format()), context.color_format() = dst_format;
"' is not convertible to any other"); return res;
}); } else if (dst_format == ColorFormat::RGB) {
auto cut = cut_last_channel(nodes, function, context);
auto reverse = reverse_channels(std::get<0>(cut), function, context);
bool updated = std::get<1>(cut) | std::get<1>(reverse);
context.color_format() = dst_format;
return std::make_tuple(std::get<0>(reverse), updated);
}
}
OPENVINO_ASSERT(false,
"Source color format '",
color_format_name(context.color_format()),
"' is not convertible to '",
color_format_name(dst_format),
"'");
},
"convert color (" + color_format_name(dst_format) + ")");
} }
void PreStepsList::add_reverse_channels() { void PreStepsList::add_reverse_channels() {
m_actions.emplace_back([](const std::vector<Output<Node>>& nodes, m_actions.emplace_back(
const std::shared_ptr<Model>& function, [](const std::vector<Output<Node>>& nodes,
PreprocessingContext& context) { const std::shared_ptr<Model>& function,
auto resp = reverse_channels(nodes, function, context); PreprocessingContext& context) {
auto outputs = std::get<0>(resp); auto resp = reverse_channels(nodes, function, context);
OPENVINO_ASSERT(outputs.size() == 1, "Internal error: reverse_channels returned unexpected number of outputs"); auto outputs = std::get<0>(resp);
set_is_preprocessing_node(outputs.at(0).get_node_shared_ptr()); OPENVINO_ASSERT(outputs.size() == 1,
return resp; "Internal error: reverse_channels returned unexpected number of outputs");
}); set_is_preprocessing_node(outputs.at(0).get_node_shared_ptr());
return resp;
},
"reverse channels");
} }
std::tuple<std::vector<Output<Node>>, bool> PreStepsList::reverse_channels(const std::vector<Output<Node>>& nodes, std::tuple<std::vector<Output<Node>>, bool> PreStepsList::reverse_channels(const std::vector<Output<Node>>& nodes,
@ -477,52 +522,59 @@ std::tuple<std::vector<Output<Node>>, bool> PreStepsList::cut_last_channel(const
//------------- Post processing ------ //------------- Post processing ------
void PostStepsList::add_convert_impl(const element::Type& type) { void PostStepsList::add_convert_impl(const element::Type& type) {
m_actions.emplace_back([type](const Output<Node>& node, PostprocessingContext& ctxt) { m_actions.emplace_back(
element::Type t = type; [type](const Output<Node>& node, PostprocessingContext& ctxt) {
if (t == element::Type{}) { element::Type t = type;
t = ctxt.target_element_type(); if (t == element::Type{}) {
} t = ctxt.target_element_type();
if (t == node.get_element_type()) { }
return std::make_tuple(node, false); if (t == node.get_element_type()) {
} return std::make_tuple(node, false);
OPENVINO_ASSERT( }
!t.is_dynamic() && t != element::undefined, OPENVINO_ASSERT(
"Can't convert to dynamic/unknown element type, consider using of InputTensorInfo::set_element_type"); !t.is_dynamic() && t != element::undefined,
auto convert = std::make_shared<op::v0::Convert>(node, t); "Can't convert to dynamic/unknown element type, consider using of InputTensorInfo::set_element_type");
return std::make_tuple(Output<Node>(convert), true); auto convert = std::make_shared<op::v0::Convert>(node, t);
}); return std::make_tuple(Output<Node>(convert), true);
},
"convert type (" + type.get_type_name() + ")");
} }
void PostStepsList::add_convert_layout_impl(const Layout& layout) { void PostStepsList::add_convert_layout_impl(const Layout& layout) {
m_actions.emplace_back([layout](const Output<Node>& node, PostprocessingContext& context) { m_actions.emplace_back(
Layout dst_layout = layout.empty() ? context.target_layout() : layout; [layout](const Output<Node>& node, PostprocessingContext& context) {
auto permutation = layout::utils::find_permutation(context.layout(), node.get_partial_shape(), dst_layout); Layout dst_layout = layout.empty() ? context.target_layout() : layout;
if (permutation.empty()) { auto permutation = layout::utils::find_permutation(context.layout(), node.get_partial_shape(), dst_layout);
// No transpose is needed, just update layout if (permutation.empty()) {
if (!layout.empty()) { // No transpose is needed, just update layout
context.layout() = layout; if (!layout.empty()) {
context.layout() = layout;
}
return std::make_tuple(node, false);
} }
return std::make_tuple(node, false); auto perm_constant =
} op::v0::Constant::create<int64_t>(element::i64, Shape{permutation.size()}, permutation);
auto perm_constant = op::v0::Constant::create<int64_t>(element::i64, Shape{permutation.size()}, permutation); auto transpose = std::make_shared<op::v1::Transpose>(node, perm_constant);
auto transpose = std::make_shared<op::v1::Transpose>(node, perm_constant); context.layout() = dst_layout; // Update context's current layout
context.layout() = dst_layout; // Update context's current layout return std::make_tuple(Output<Node>(transpose), true);
return std::make_tuple(Output<Node>(transpose), true); },
}); "convert layout " + layout.to_string());
} }
void PostStepsList::add_convert_layout_impl(const std::vector<uint64_t>& dims) { void PostStepsList::add_convert_layout_impl(const std::vector<uint64_t>& dims) {
if (dims.empty()) { if (dims.empty()) {
return; return;
} }
m_actions.emplace_back([dims](const Output<Node>& node, PostprocessingContext& context) { m_actions.emplace_back(
auto perm_constant = op::v0::Constant::create<uint64_t>(element::u64, Shape{dims.size()}, dims); [dims](const Output<Node>& node, PostprocessingContext& context) {
auto new_layout = layout::utils::apply_permutation(context.layout(), dims); auto perm_constant = op::v0::Constant::create<uint64_t>(element::u64, Shape{dims.size()}, dims);
auto transpose = std::make_shared<op::v1::Transpose>(node, perm_constant); auto new_layout = layout::utils::apply_permutation(context.layout(), dims);
auto res = std::make_tuple(Output<Node>(transpose), true); auto transpose = std::make_shared<op::v1::Transpose>(node, perm_constant);
context.layout() = std::move(new_layout); // Update context's current layout auto res = std::make_tuple(Output<Node>(transpose), true);
return res; context.layout() = std::move(new_layout); // Update context's current layout
}); return res;
},
"convert layout " + vector_to_string(dims));
} }
} // namespace preprocess } // namespace preprocess
} // namespace ov } // namespace ov

28
src/core/src/preprocess/preprocess_steps_impl.hpp
View File

@ -146,6 +146,14 @@ using InternalPreprocessOp =
const std::shared_ptr<Model>& function, const std::shared_ptr<Model>& function,
PreprocessingContext& context)>; PreprocessingContext& context)>;
struct InternalPreprocessAction {
InternalPreprocessAction(InternalPreprocessOp op, std::string name)
: m_op(std::move(op)),
m_name(std::move(name)) {}
InternalPreprocessOp m_op;
std::string m_name;
};
/// \brief PreProcessStepsImpl - internal data structure /// \brief PreProcessStepsImpl - internal data structure
class PreStepsList { class PreStepsList {
public: public:
@ -160,10 +168,10 @@ public:
std::tuple<PartialShape, Layout> calculate_param_shape(const PartialShape& model_shape, std::tuple<PartialShape, Layout> calculate_param_shape(const PartialShape& model_shape,
const Layout& model_layout) const; const Layout& model_layout) const;
const std::list<InternalPreprocessOp>& actions() const { const std::list<InternalPreprocessAction>& actions() const {
return m_actions; return m_actions;
} }
std::list<InternalPreprocessOp>& actions() { std::list<InternalPreprocessAction>& actions() {
return m_actions; return m_actions;
} }
@ -179,7 +187,7 @@ private:
PreprocessingContext& context); PreprocessingContext& context);
private: private:
std::list<InternalPreprocessOp> m_actions; std::list<InternalPreprocessAction> m_actions;
std::list<std::vector<uint64_t>> m_layout_converts; std::list<std::vector<uint64_t>> m_layout_converts;
std::list<std::vector<uint64_t>> m_forward_layout_converts; std::list<std::vector<uint64_t>> m_forward_layout_converts;
Layout m_last_explicit_layout; Layout m_last_explicit_layout;
@ -197,6 +205,14 @@ public:
using InternalPostprocessOp = std::function<std::tuple<ov::Output<ov::Node>, bool>(const ov::Output<ov::Node>& node, using InternalPostprocessOp = std::function<std::tuple<ov::Output<ov::Node>, bool>(const ov::Output<ov::Node>& node,
PostprocessingContext& context)>; PostprocessingContext& context)>;
struct InternalPostprocessAction {
InternalPostprocessAction(InternalPostprocessOp op, std::string name)
: m_op(std::move(op)),
m_name(std::move(name)) {}
InternalPostprocessOp m_op;
std::string m_name;
};
/// \brief PostProcessStepsImpl - internal data structure /// \brief PostProcessStepsImpl - internal data structure
class PostStepsList { class PostStepsList {
public: public:
@ -204,15 +220,15 @@ public:
void add_convert_layout_impl(const Layout& layout); void add_convert_layout_impl(const Layout& layout);
void add_convert_layout_impl(const std::vector<uint64_t>& dims); void add_convert_layout_impl(const std::vector<uint64_t>& dims);
const std::list<InternalPostprocessOp>& actions() const { const std::list<InternalPostprocessAction>& actions() const {
return m_actions; return m_actions;
} }
std::list<InternalPostprocessOp>& actions() { std::list<InternalPostprocessAction>& actions() {
return m_actions; return m_actions;
} }
private: private:
std::list<InternalPostprocessOp> m_actions; std::list<InternalPostprocessAction> m_actions;
}; };
class PostProcessSteps::PostProcessStepsImpl : public PostStepsList {}; class PostProcessSteps::PostProcessStepsImpl : public PostStepsList {};

165
src/core/tests/preprocess.cpp
View File

@ -24,6 +24,16 @@ static std::shared_ptr<Model> create_simple_function(element::Type type, const P
return std::make_shared<Model>(ResultVector{res}, ParameterVector{data1}); return std::make_shared<Model>(ResultVector{res}, ParameterVector{data1});
} }
static std::shared_ptr<Model> create_trivial(element::Type type, const PartialShape& shape) {
auto data1 = std::make_shared<op::v0::Parameter>(type, shape);
data1->set_friendly_name("input1");
data1->get_output_tensor(0).set_names({"tensor_input1"});
auto res = std::make_shared<op::v0::Result>(data1);
res->set_friendly_name("Result1");
res->get_output_tensor(0).set_names({"tensor_output1"});
return std::make_shared<Model>(ResultVector{res}, ParameterVector{data1});
}
template <int N> template <int N>
static std::shared_ptr<Model> create_n_inputs(element::Type type, const PartialShape& shape) { static std::shared_ptr<Model> create_n_inputs(element::Type type, const PartialShape& shape) {
ResultVector res; ResultVector res;
@ -1694,3 +1704,158 @@ TEST(pre_post_process, exception_safety) {
EXPECT_EQ(f->output(1).get_node_shared_ptr()->get_friendly_name(), out_name1); EXPECT_EQ(f->output(1).get_node_shared_ptr()->get_friendly_name(), out_name1);
EXPECT_EQ(f->output(1).get_tensor().get_names(), out_tensor_names1); EXPECT_EQ(f->output(1).get_tensor().get_names(), out_tensor_names1);
} }
TEST(pre_post_process, layout_on_trivial) {
auto f = create_trivial(element::f32, Shape{1, 440});
auto p = PrePostProcessor(f);
p.input().tensor().set_layout("NC").set_element_type(element::f32);
p.input().model().set_layout("NC");
p.output().tensor().set_element_type(element::f32);
EXPECT_EQ(f->input().get_partial_shape(), (PartialShape{1, 440}));
f = p.build();
EXPECT_EQ(layout::get_layout(f->input()), "NC") << layout::get_layout(f->input()).to_string();
EXPECT_EQ(f->input().get_partial_shape(), (PartialShape{1, 440}));
ov::set_batch(f, 2);
EXPECT_EQ(f->input().get_partial_shape(), (PartialShape{2, 440}));
}
TEST(pre_post_process, dump_empty) {
auto f = create_simple_function(element::f32, Shape{1, 3, 2, 2});
auto p = PrePostProcessor(f);
std::stringstream str;
str << p;
EXPECT_EQ(str.str(), std::string());
}
TEST(pre_post_process, dump_non_empty) {
auto f = create_simple_function(element::f32, Shape{1, 3, 2, 2});
auto p = PrePostProcessor(f);
p.input().tensor().set_memory_type("some_memory_type");
std::stringstream str;
str << p;
EXPECT_NE(str.str(), std::string());
}
TEST(pre_post_process, dump_preprocess) {
auto shape = PartialShape{1, 3, 2, 2};
std::stringstream shape_stream;
shape_stream << shape;
auto shape_str = shape_stream.str();
auto f = create_simple_function(element::f32, shape);
auto p = PrePostProcessor(f);
p.input()
.tensor()
.set_element_type(element::u8)
.set_layout("NHWC")
.set_spatial_dynamic_shape()
.set_memory_type("test_memory_type");
p.input()
.preprocess()
.convert_element_type(element::f32)
.mean(1.f)
.scale(2.f)
.convert_layout({3, 2, 1, 0})
.resize(ResizeAlgorithm::RESIZE_LINEAR, 480, 640)
.resize(ResizeAlgorithm::RESIZE_LINEAR)
.custom([](const Output<Node>& node) {
return node;
});
p.input().model().set_layout("NCHW");
std::stringstream stream;
stream << p;
auto dump = stream.str();
EXPECT_TRUE(dump.find("Input") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("input1") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("memory type=test_memory_type") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("Pre-processing steps (7):") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("mean") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("scale") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("convert type") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("convert layout (3,2,1,0):") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("resize to (480, 640):") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("resize to model width/height:") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("custom:") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("Implicit pre-processing steps (1):") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("convert layout") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("Model's expected tensor: " + shape_str + ", " + Layout("NCHW").to_string()) !=
std::string::npos)
<< dump;
EXPECT_TRUE(dump.find("output1") == std::string::npos) << dump;
}
TEST(pre_post_process, dump_preprocess_multiplane) {
auto shape_to_string = [](const PartialShape& shape) {
std::stringstream shape_stream;
shape_stream << shape;
return shape_stream.str();
};
auto shape = PartialShape{1, 3, 20, 20};
auto shape_str = shape_to_string(shape);
auto f = create_simple_function(element::f32, shape);
auto p = PrePostProcessor(f);
p.input().tensor().set_element_type(element::u8).set_color_format(ColorFormat::NV12_TWO_PLANES);
p.input().preprocess().convert_element_type(element::f32).convert_color(ColorFormat::RGB);
p.input().model().set_layout("NCHW");
std::stringstream stream;
stream << p;
auto dump = stream.str();
EXPECT_TRUE(dump.find("Input") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("input1") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("memory type=") == std::string::npos) << dump;
EXPECT_TRUE(dump.find("NV12") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("RGB") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("Implicit pre-processing steps (1):") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("Pre-processing steps (2):") != std::string::npos) << dump;
EXPECT_TRUE(dump.find(shape_to_string(PartialShape{1, 20, 20, 1})) != std::string::npos) << dump;
EXPECT_TRUE(dump.find(shape_to_string(PartialShape{1, 10, 10, 2})) != std::string::npos) << dump;
EXPECT_TRUE(dump.find("convert type") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("convert color") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("convert layout") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("Model's expected tensor: " + shape_str + ", " + Layout("NCHW").to_string()) !=
std::string::npos)
<< dump;
EXPECT_TRUE(dump.find("output1") == std::string::npos) << dump;
}
TEST(pre_post_process, dump_postprocess) {
auto shape = PartialShape{1, 3, 2, 2};
std::stringstream shape_stream;
shape_stream << shape;
auto shape_str = shape_stream.str();
auto f = create_simple_function(element::f32, shape);
auto p = PrePostProcessor(f);
p.output().model().set_layout("NCHW");
p.output()
.postprocess()
.convert_element_type(element::i32)
.convert_layout({3, 2, 1, 0})
.custom([](const Output<Node>& node) {
return node;
});
p.output().tensor().set_element_type(element::u8).set_layout("NHWC");
std::stringstream stream;
stream << p;
auto dump = stream.str();
EXPECT_TRUE(dump.find("Output") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("output1") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("Post-processing steps (3):") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("Post-processing implicit steps (2):") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("convert type") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("convert layout (3,2,1,0):") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("convert layout " + Layout("NHWC").to_string()) != std::string::npos) << dump;
EXPECT_TRUE(dump.find("custom:") != std::string::npos) << dump;
EXPECT_TRUE(dump.find("Model's data tensor: " + shape_str + ", " + Layout("NCHW").to_string()) != std::string::npos)
<< dump;
EXPECT_TRUE(dump.find("input1") == std::string::npos) << dump;
}
TEST(pre_post_process, dump_error) {
auto f = create_simple_function(element::f32, Shape{2, 2});
auto p = PrePostProcessor(f);
p.input().tensor().set_layout("NC");
p.input().model().set_layout("HW");
std::stringstream stream;
stream << p;
auto dump = stream.str();
EXPECT_TRUE(dump.find("Error occurred:") != std::string::npos) << dump;
}