From 6ef59ce3e40388df1d89961be6a9093725fe9d14 Mon Sep 17 00:00:00 2001 From: Mikhail Nosov Date: Tue, 7 Dec 2021 14:31:55 +0300 Subject: [PATCH] [OV2.0] Model Optimizer: mean/scale/reverse_input_channels/layout for new frontends (#8751) * Preprocessing API - base classes Includes API definition for trivial mean/scale operations (which don't require layout) Mean/scale with 'layout' support will be done under separate task together with Layout Current test code coverage: 100% * Python bindings for base preprocessing API * remove pre_post_process directory from ngraph/core * remove files from ngraph/python dir * move pyngraph pre_post_process files from ngraph/python to runtime * remove pre_post_process test from CMakeList * move include to the header * update include path for pre_post_process * style fix * bind InputTensorInfo::set_layout * cleaned test_preprocess * fix test expected output * remove duplicate test * update description of set_element_type * fix style * move preprocess from pyngraph to pyopenvino/graph * update test_preprocess imports and remove unnecessary test * remove duplicate import * update custom method * update test * update test * create decorator that changes Node into Output * create function that cast Node to Output * update test_preprocess to use decorator for custom function * change _cast_to_output -> _from_node * move frontend folder to pyopenvino * rename includes and add compile options * include frontend to pyopenvino * move __init__.py * move tests * remove mock from tests_compatibility * rename import module * Fix code style cpp * refactor a few lines * style fix * update few lines in mo * add tests fro scale and mean with vector input * style fix * add docstring for custom_preprocess_function * bind InputInfo network method * style fix * Add pyopenvino to dependencies * bind OutputInfo * fix description of preprocess submodule * fix style * update copyright year * Fix mock * update docstring * bind OutputTensorInfo * bind OutputNetworkInfo and InputNetworkInfo * bind ColorFormat and ResizeAlgorithm * clean imports * fix typo * add PostProcessSteps to init * bind PreProcessSteps * create additional tests * Fix mo test * remove module local * fix code style * update comment * fix return type * update docs * fix code style * change ngraph.Type to ov.Type * fix typo * move _from_node to node_output.hpp * add read_model from buffer * update imports * add new line * remove bad quotes * update imports * style fix * add new line * rename functin args * remove Type import * update tests * style fix * test clean * remove blank line * update PrePostProcessor init and build methods * create test with model update tests with new PrePostProcessor init and build * # Conflicts: # inference-engine/ie_bridges/python/src/openvino/offline_transformations/offline_transformations_api.pyx # inference-engine/ie_bridges/python/src/openvino/offline_transformations/offline_transformations_api_impl.cpp # inference-engine/ie_bridges/python/src/openvino/offline_transformations/offline_transformations_api_impl.hpp # inference-engine/ie_bridges/python/src/openvino/offline_transformations/offline_transformations_api_impl_defs.pxd # inference-engine/tests/ie_test_utils/common_test_utils/ngraph_test_utils.cpp # inference-engine/tests/ie_test_utils/common_test_utils/ngraph_test_utils.hpp # model-optimizer/mo/moc_frontend/serialize.py # thirdparty/gflags/gflags # thirdparty/gtest/gtest * Stash * move preprocess module from openvino.impl to openvino * fix building * fix code style * try to move MO to use new api * Intermediate commit * try to move MO to use new api * Test pybind11 custom holder for Preprocessing types (InputInfo and PreProcessingSteps) * Initial code for source_target layout handling for preprocessing Initial implementation of reverse input channels * Use input's tensor names instead of friendly names * Skeleton for guessing layouts and clearing it after preprocessing * updated package_BOM.txt * Use reference_wrapper for preprocess bindings * Update tests * Layout::find_permutation - support of dynamic layouts Covered case for 'trivial convert' where no permutation is needed It is needed for Model Optimizer for logic which will guess model's layout, like "?c??" * Stash * add bindings to I420_SINGLE_PLANE and I420_THREE_PLANES * remove init from all classes except PrePostProcessor and add RGBX and BGRX to ColorFormat enum * Guess layout so that existing mean/scale tests passed * update test name * Draft to guess layout for 'reverse_input_channels' * More unit tests (error cases) * pylint & flake8 * pylint - ignore import error * Stash * Moved preprocessing to 'back' folder * More tests * Update package_BOM * Support layout_values with no names Support layout set for 'outputs' Tests * Export more enum names from nrgaph * Basic --layout parsing * removed debug prints * Further updates after rebase * Update imports * Removed part from 8829 * Fix imports in test code * Minor cosmetics * Don't guess 'C' if layout is already set by model Expose 'Layout::empty' method * Style fix * Apply review comments Restricted 'heuristics' C++: Added 'fp16', 'fp64' support to mean/scale * Applied review comments * Added some dynamic test cases * Move call of 'apply_preprocessing' to 'serialize.py' * Unnecessary change * Added more comments to code Co-authored-by: pszmel Co-authored-by: Alexey Lebedev Co-authored-by: bszmelcz Co-authored-by: Anastasia Kuporosova Co-authored-by: y Co-authored-by: Vafin, Maxim --- model-optimizer/automation/package_BOM.txt | 1 + model-optimizer/mo/back/preprocessing.py | 401 ++++++++++++ model-optimizer/mo/moc_frontend/serialize.py | 5 + .../mo/back/moc_preprocessing_test_actual.py | 617 ++++++++++++++++++ .../unit_tests/mo/frontend_ngraph_test.py | 9 + .../python/src/openvino/runtime/__init__.py | 1 + .../python/src/pyopenvino/graph/layout.cpp | 1 + .../src/preprocess/preprocess_steps_impl.cpp | 18 +- src/core/tests/preprocess.cpp | 14 +- 9 files changed, 1061 insertions(+), 6 deletions(-) create mode 100644 model-optimizer/mo/back/preprocessing.py create mode 100644 model-optimizer/unit_tests/mo/back/moc_preprocessing_test_actual.py diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index 82070da5ece..1b36cb13f99 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -807,6 +807,7 @@ mo/back/__init__.py mo/back/ie_ir_ver_2/__init__.py mo/back/ie_ir_ver_2/emitter.py mo/back/offline_transformations.py +mo/back/preprocessing.py mo/back/replacement.py mo/front/__init__.py mo/front/caffe/__init__.py diff --git a/model-optimizer/mo/back/preprocessing.py b/model-optimizer/mo/back/preprocessing.py new file mode 100644 index 00000000000..861da882169 --- /dev/null +++ b/model-optimizer/mo/back/preprocessing.py @@ -0,0 +1,401 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import logging as log + +from mo.utils.error import Error +from mo.utils.utils import refer_to_faq_msg + +import numpy as np + +from openvino.preprocess import PrePostProcessor # pylint: disable=no-name-in-module,import-error +# pylint: disable=no-name-in-module,import-error +from openvino.runtime import Function, Layout, PartialShape, layout_helpers + + +def update_mean_scale_to_dict(input_nodes: list, mean_scale_val, scale): + """ + Internal function. Updates mean/scale values from array to dictionary + :param: input_nodes Inputs of model + :param: mean_scale_val Parsed 'mean_scale_val' object from command line arguments + :param: scale Global scale factor for all inputs from --scale command line arguments + """ + if not isinstance(mean_scale_val, dict): + if len(mean_scale_val) != len(input_nodes): + raise Error('Numbers of inputs and mean/scale values do not match. ' + refer_to_faq_msg(61)) + + data = np.copy(mean_scale_val) + mean_scale_val = {} + for idx, node in enumerate(input_nodes): + names_list = list(node.get_tensor().get_names()) + if not names_list: + continue + node_name = names_list[0] + mean_scale_val.update( + { + node_name: { + 'mean': data[idx][0], + 'scale': data[idx][1] + } + } + ) + + if scale: + for node in input_nodes: + names_list = list(node.get_tensor().get_names()) + if not names_list: + continue + node_name = names_list[0] + old_val = mean_scale_val[node_name] if node_name in mean_scale_val else None + mean_scale_val.update( + { + node_name: { + 'mean': old_val['mean'] if old_val and 'mean' in old_val else None, + 'scale': scale + } + } + ) + return mean_scale_val + + +def check_keys_valid(ov_function: Function, keys: list, search_outputs: bool): + """ + Internal function: checks if keys from cmd line arguments correspond to ov_function's inputs/outputs + Throws if some key is not found + Throws if some different keys point to the same actual input/output + """ + nodes_used = {} + nodes = ov_function.inputs + if search_outputs: + nodes += ov_function.outputs + + for name in keys: + node_found = False + for ov_node in nodes: + if name in ov_node.get_tensor().get_names(): + if ov_node in nodes_used: + raise Error('Key for {} and {} point to same model input/output.' + .format(name, nodes_used[ov_node])) + nodes_used[ov_node] = name + node_found = True + break + + if not node_found: + if not search_outputs: + raise Error('Input with name {} wasn\'t found! {}'.format(name, refer_to_faq_msg(83))) + else: + raise Error('Input/Output with name {} wasn\'t found! {}'.format(name, refer_to_faq_msg(83))) + + +def update_layout_is_input_flag(ov_function: Function, layout_values: dict): + """ + Internal function: updates layout_values with flag whether each layout belongs to input or to output + """ + for name, layout_value in layout_values.items(): + layout_value['is_input'] = False + for ov_input in ov_function.inputs: + if name in ov_input.get_tensor().get_names(): + layout_value['is_input'] = True + break + return layout_values + + +def find_channels_dimension(shape: PartialShape, num_channels: int, name: str, layout_values): + """ + Internal function. Finds dimension index matching with expected channels number + Raises error if there is no candidates or number of candidates is > 1 + :param: shape Parameter's partial shape + :param: num_channels Number of channels to find in shape + :param: name Parameter's name, used for Error-handling purposes + :param: layout_values Existing source/target layout items specified by user + :return: updated layout items with guessed layouts + """ + if shape.rank.is_dynamic: + raise Error('Can\'t determine channels dimension for dynamic shape for parameter {}.' + .format(name)) + + dim_idx_found = -1 + for dim_idx in range(shape.rank.get_length()): + dim = shape.get_dimension(dim_idx) + if dim.is_static and dim.get_length() == num_channels: + if dim_idx_found >= 0: + raise Error('Can\'t determine channels dimension for {}. ' + 'Input shape is {}, needed channels {}. ' + 'Conflicting dimensions: {} and {}. Please specify layout manually.' + .format(name, shape, num_channels, dim_idx_found, dim_idx)) + dim_idx_found = dim_idx + if dim_idx_found < 0: + raise Error('Can\'t determine channels dimension for {}. ' + 'Input shape is {}, needed channels {}' + .format(name, shape, num_channels)) + + # Restrict guessed channels index to particular position depending on tensor shape(3d, 4d, 5d) + if shape.rank.get_length() == 3: + # CHW or HWC, possible channels index is 0 or 2 + if dim_idx_found != 0 and dim_idx_found != 2: + raise Error('Can\'t determine channels dimension for 3D input {} (CHW or HWC) with shape {}. ' + 'Please specify layout containing \'C\' channels manually.'.format(name, shape)) + elif shape.rank.get_length() == 4: + # NCHW or NHWC, possible channels index is 1 or 3 + if dim_idx_found != 1 and dim_idx_found != 3: + raise Error('Can\'t determine channels dimension for 4D input {} (NCHW or NHWC) with shape {}. ' + 'Please specify layout containing \'C\' channels manually.'.format(name, shape)) + elif shape.rank.get_length() == 5: + # NCDHW or NDHWC, possible channels index is 1 or 4 + if dim_idx_found != 1 and dim_idx_found != 4: + raise Error('Can\'t determine channels dimension for 5D input {} (NCDHW or NDHWC) with shape {}. ' + 'Please specify layout containing \'C\' channels manually.'.format(name, shape)) + else: + raise Error('Can\'t determine channels dimension for {}D input {} with shape {}.' + 'Please specify layout containing \'C\' channels manually.' + .format(shape.rank.get_length(), name, shape)) + + layout_str = "?" * shape.rank.get_length() + layout_str = layout_str[:dim_idx_found] + 'C' + layout_str[dim_idx_found+1:] + layout_values[name] = { + 'source_layout': layout_str, + 'target_layout': None, + 'source_guessed': True, + 'is_input': True + } + return layout_values + + +def guess_source_layouts_by_mean_scale(ov_function: Function, layout_values, mean_scale_values: dict): + """ + Internal function. Try to guess source layout for input by its shape and/or framework + :param: ov_function Original model + :param: layout_values Existing source/target layout items specified by user + :param: mean_scale_values Dictionary with mean/scale values defined for each argument + :return: updated layout items with guessed layouts + """ + for ms_name, mean_scale in mean_scale_values.items(): + num_channels_mean = len(mean_scale['mean']) if mean_scale['mean'] is not None else 0 + num_channels_scale = len(mean_scale['scale']) if hasattr(mean_scale['scale'], '__len__') else 0 + + if num_channels_mean > 1 and \ + num_channels_scale > 1 and \ + num_channels_mean is not num_channels_scale: + raise Error('Mean/Scale values for {} have different sizes: {} {}' + .format(ms_name, num_channels_mean, num_channels_scale)) + + need_guess_channels = num_channels_mean > 1 or num_channels_scale > 1 + if not need_guess_channels: # Mean/scale is complex and needs 'channels' specified in layout + continue + + num_channels = num_channels_mean if num_channels_mean > 1 else num_channels_scale + + for i in range(0, len(ov_function.inputs)): + ov_input = ov_function.input(i) + + if not ov_function.get_parameters()[i].layout.empty: + continue + + if ms_name not in ov_input.get_tensor().get_names(): + continue + + layout_item = None + for name in ov_input.get_tensor().get_names(): + if name in layout_values: + layout_item = layout_values[name] + break + + if layout_item is not None: + # User specified some layout, skip guessing + continue + + # Guess layout is applicable only when number of channels is '3' + if num_channels != 3: + raise Error('Can\'t determine channels dimension for {}. ' + 'When number of mean/scale values is {} (not 3), ' + 'please specify layout for input manually'.format(ms_name, num_channels)) + + layout_values = find_channels_dimension(shape=ov_input.get_partial_shape(), + num_channels=num_channels, + name=ms_name, + layout_values=layout_values) + return layout_values + + +def check_suitable_for_reverse(layout: Layout, ov_input): + """ + Internal function. Checks if input with layout is suitable for reversing channels + :param: layout Existing source/target layout items specified by user + :param: ov_input Model's input + :return: True if reverse channels can be applied to input + """ + if not layout_helpers.has_channels(layout): + return False + if ov_input.get_partial_shape().rank.is_dynamic: + return False + + c_idx = layout_helpers.channels_idx(layout) + rank = ov_input.get_partial_shape().rank.get_length() + if c_idx < 0: + c_idx += rank + if c_idx >= rank: + raise Error('Layout {} for input {} is inconsistent with shape {}'.format( + layout, ov_input.get_tensor().get_any_name(), ov_input.get_partial_shape())) + c_num = ov_input.get_partial_shape()[c_idx] + return c_num.is_dynamic or c_num.get_length() == 3 + + +def guess_source_layouts_for_reverse_channels(ov_function: Function, layout_values): + """ + Internal function. Try to guess source layout for input by finding dimension with size=3 (RGB/BGR) + Additionally checks existing layouts and detects suitable inputs for reversing of input channels + :param: ov_function Original model + :param: layout_values Existing source/target layout items specified by user + :return: array with suitable parameters for reversing of input channels + """ + all_params = [] + suitable_params = [] + for i in range(0, len(ov_function.inputs)): + ov_input = ov_function.input(i) + param_info = [ov_input.get_tensor().get_any_name(), ov_input.get_partial_shape()] + all_params.append(param_info) + + if not ov_function.get_parameters()[i].layout.empty: + if check_suitable_for_reverse(ov_function.get_parameters()[i].layout, ov_input): + suitable_params.append(param_info) + continue + + layout_item = None + first_name = ov_input.get_tensor().get_any_name() + for name in ov_input.get_tensor().get_names(): + if name in layout_values: + layout_item = layout_values[name] + break + + if layout_item is not None: + if layout_item.get('target_layout'): + if check_suitable_for_reverse(Layout(layout_item['target_layout']), ov_input): + suitable_params.append(param_info) + elif layout_item.get('source_layout'): + if check_suitable_for_reverse(Layout(layout_item['source_layout']), ov_input): + suitable_params.append(param_info) + continue + + try: + layout_values = find_channels_dimension(shape=ov_input.get_partial_shape(), + num_channels=3, + name=first_name, + layout_values=layout_values) + except Error as e: + log.debug('Reverse input channels guess did not succeed {}'.format(e)) + else: + layout = layout_values[first_name].get('source_layout') + if layout and check_suitable_for_reverse(Layout(layout), ov_input): + suitable_params.append(param_info) + + if len(suitable_params) < len(all_params): + log.error('Network has {} inputs overall, but only {} of them are suitable for input channels reversing.\n' + 'Suitable for input channel reversing inputs are 4-dimensional with 3 channels\nAll inputs: {}\n' + 'Suitable inputs {}'.format(len(all_params), len(suitable_params), all_params, suitable_params), + extra={'is_warning': True}) + return suitable_params + + +def apply_preprocessing(ov_function: Function, argv: argparse.Namespace): + """ + Applies pre-processing of model inputs by adding appropriate operations + On return, 'ov_function' object will be updated + Expected 'argv.mean_scale_values' formats examples: + a) Dict: {'inputName': {'mean': [1., 2., 3.], 'scale': [2., 4., 8.]}} + b) List: list(np.array([(np.array([1., 2., 3.]), np.array([2., 4., 6.])), + (np.array([7., 8., 9.]), np.array([5., 6., 7.]))) + Expected 'argv.layout_values' format examples: + a) Specific layouts for inputs and outputs + { 'input1': { + 'source_layout': 'nchw', + 'target_layout': 'nhwc' + }, + 'output2': { + 'source_layout': 'nhwc' + } + } + b) Layout for single input: {'': {'source_layout': 'nchw'}} + :param: ov_function OV function for applying mean/scale pre-processing + :param: argv Parsed command line arguments + """ + prep = PrePostProcessor(ov_function) + + if 'mean_scale_values' in argv and argv.mean_scale_values: + mean_scale_values = argv.mean_scale_values + else: + mean_scale_values = {} + + mean_scale_values = update_mean_scale_to_dict(input_nodes=ov_function.inputs, + mean_scale_val=mean_scale_values, + scale=argv.scale) + # On return, mean_scale_values is a dictionary with input names as key and mean/scale pair as value + # {'inputName': {'mean': [1., 2., 3.], 'scale': [2.]}} + + layout_values = {} + if 'layout_values' in argv and argv.layout_values: + layout_values = argv.layout_values + + if '' in layout_values: + if len(ov_function.inputs) > 1: + input_names = [list(ov_input.get_tensor().get_names())[0] for ov_input in ov_function.inputs] + raise Error('Layout without name can be specified for models with only one input, ' + 'but provided model has {} inputs: \'{}\'. ' + 'Please specify explicitly input/output name for --layout option' + .format(len(input_names), input_names)) + layout_values = { + list(ov_function.input().get_tensor().get_names())[0]: { + 'source_layout': layout_values[''].get('source_layout'), + 'target_layout': layout_values[''].get('target_layout') + } + } + check_keys_valid(ov_function=ov_function, keys=mean_scale_values.keys(), search_outputs=False) + check_keys_valid(ov_function=ov_function, keys=layout_values.keys(), search_outputs=True) + + layout_values = update_layout_is_input_flag(ov_function, layout_values) + layout_values = guess_source_layouts_by_mean_scale(ov_function, layout_values, mean_scale_values) + need_reverse = 'reverse_input_channels' in argv and argv.reverse_input_channels + suitable_params_ric = [] + if need_reverse: + suitable_params_ric = guess_source_layouts_for_reverse_channels(ov_function=ov_function, + layout_values=layout_values) + + for node_name, layout_value in layout_values.items(): + if layout_value.get('source_layout'): + if layout_value.get('is_input'): + prep.input(node_name).network().set_layout(Layout(layout_value['source_layout'])) + else: + prep.output(node_name).network().set_layout(Layout(layout_value['source_layout'])) + if layout_value.get('target_layout'): + if layout_value.get('is_input'): + prep.input(node_name).tensor().set_layout(Layout(layout_value['target_layout'])) + else: + prep.output(node_name).tensor().set_layout(Layout(layout_value['target_layout'])) + + for node_name, node_mean_scale_values in mean_scale_values.items(): + # Apply mean first, then scale + if node_mean_scale_values['mean'] is not None: + prep.input(node_name).preprocess().mean(node_mean_scale_values['mean']) + if node_mean_scale_values['scale'] is not None: + prep.input(node_name).preprocess().scale(node_mean_scale_values['scale']) + log.debug('Mean/Scale pre-processing applied to {}'.format(node_name)) + + # Apply reverse_input_channels + if need_reverse: + for name, _ in suitable_params_ric: + prep.input(name).preprocess().reverse_channels() + log.debug('reverse_input_channels pre-processing applied to {}'.format(name)) + + # Apply pre-processing builder to a function + ov_function = prep.build() + + # Remove guessed layout values from ov_function (these values shall not be serialized to IR + for node_name, layout_value in layout_values.items(): + if layout_value.get('source_guessed') and \ + not layout_value.get('target_layout'): + # search for parameter object + for idx, ov_input in enumerate(ov_function.inputs): + if node_name in ov_input.get_tensor().get_names(): + log.debug('Clearing guessed layout {} for {}' + .format(layout_value['source_layout'], node_name)) + ov_function.get_parameters()[idx].layout = Layout() diff --git a/model-optimizer/mo/moc_frontend/serialize.py b/model-optimizer/mo/moc_frontend/serialize.py index 75104318e41..f2eedcd5dba 100644 --- a/model-optimizer/mo/moc_frontend/serialize.py +++ b/model-optimizer/mo/moc_frontend/serialize.py @@ -5,6 +5,7 @@ import argparse import os from mo.pipeline.common import get_ir_version from mo.back.ie_ir_ver_2.emitter import append_ir_info +from mo.back.preprocessing import apply_preprocessing from mo.utils.cli_parser import get_meta_info, parse_transform from openvino.runtime import Function # pylint: disable=no-name-in-module,import-error @@ -13,6 +14,10 @@ from openvino.runtime import Function # pylint: disable=no-name-in-modul def moc_emit_ir(ngraph_function: Function, argv: argparse.Namespace): output_dir = argv.output_dir if argv.output_dir != '.' else os.getcwd() + # Apply preprocessing (mean/scale/reverse_channels/convert_layout/etc) + apply_preprocessing(ov_function=ngraph_function, argv=argv) + + # Apply transformations from mo.back.offline_transformations import apply_user_transformations, apply_moc_transformations apply_user_transformations(ngraph_function, parse_transform(argv.transform)) apply_moc_transformations(ngraph_function) diff --git a/model-optimizer/unit_tests/mo/back/moc_preprocessing_test_actual.py b/model-optimizer/unit_tests/mo/back/moc_preprocessing_test_actual.py new file mode 100644 index 00000000000..6c22c95537c --- /dev/null +++ b/model-optimizer/unit_tests/mo/back/moc_preprocessing_test_actual.py @@ -0,0 +1,617 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import unittest +from argparse import Namespace + +from mo.utils.error import Error + +import numpy as np + + +try: + # pylint: disable=no-name-in-module,import-error + from mo.back.preprocessing import apply_preprocessing + + # pylint: disable=no-name-in-module,import-error + import openvino.runtime.opset8 as ops + from openvino.runtime import Function, Layout, PartialShape + +except Exception: + print("No OpenVINO API available," + "ensure to set correct PYTHONPATH when running these tests") + raise + + +def create_function2(shape1=[2, 2], shape2=[2, 2], dtype1=np.float32, dtype2=np.float32): + input1 = ops.parameter(shape1, dtype=dtype1, name="input1") + input1.get_output_tensor(0).set_names({'input1', 'input1a'}) + relu1 = ops.relu(input1) + res1 = ops.result(relu1, "res1") + res1.get_output_tensor(0).set_names({'res1', 'res1a'}) + input2 = ops.parameter(shape2, dtype=dtype2, name="input2") + input2.get_output_tensor(0).set_names({'input2', 'input2a'}) + relu2 = ops.relu(input2) + res2 = ops.result(relu2, "res2") + res2.get_output_tensor(0).set_names({'res2', 'res2a'}) + function = Function(results=[res1, res2], parameters=[input1, input2], name="TestFunction") + return function + + +def create_function1(shape1=[2, 2]): + input1 = ops.parameter(shape1, dtype=np.float32, name="input1") + input1.get_output_tensor(0).set_names({'input1', 'input1a'}) + relu1 = ops.relu(input1) + res1 = ops.result(relu1, "res1") + res1.get_output_tensor(0).set_names({'res1', 'res1a'}) + function = Function(results=[res1], parameters=[input1], name="TestFunction") + return function + + +def process_function(ov_function: Function, argv: Namespace): + apply_preprocessing(ov_function=ov_function, argv=argv) + + +class TestPreprocessingMOC(unittest.TestCase): + def setUp(self): + pass + + def check_scale_constant(self, node, expected, shape=None): + const_node = node.input(1).get_source_output().get_node() + self.assertEqual(const_node.get_type_name(), 'Constant') + if node.get_type_name() == 'Divide': + self.assertTrue(np.allclose(const_node.get_vector(), expected)) + else: + self.assertTrue(np.allclose(const_node.get_vector(), 1. / expected)) + if shape: + assert const_node.shape == PartialShape(shape) + + def check_mean_constant(self, node, expected, shape=None): + const_node = node.input(1).get_source_output().get_node() + self.assertEqual(const_node.get_type_name(), 'Constant') + if node.get_type_name() == 'Subtract': + self.assertTrue(np.allclose(const_node.get_vector(), expected)) + else: + self.assertTrue(np.allclose(const_node.get_vector(), -expected.toList())) + if shape: + self.assertEqual(const_node.shape, PartialShape(shape)) + + def test_scale_single_value(self): + argv = Namespace(mean_scale_values=None, scale=2.0) + function = create_function2() + process_function(ov_function=function, argv=argv) + + for param in function.get_parameters(): + op_node = list(param.output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Divide' or op_node.get_type_name() == 'Multiply') + self.check_scale_constant(op_node, [2.0]) + + def test_scale_single_value_fp64(self): + argv = Namespace(mean_scale_values=None, scale=2.0) + function = create_function2(dtype1=np.float64) + process_function(ov_function=function, argv=argv) + + for ov_input in function.inputs: + op_node = list(ov_input.get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Divide' or op_node.get_type_name() == 'Multiply') + self.check_scale_constant(op_node, [2.0]) + + def test_scale_single_value_fp16(self): + argv = Namespace(mean_scale_values=None, scale=2.0) + function = create_function2(dtype1=np.float16) + process_function(ov_function=function, argv=argv) + + for ov_input in function.inputs: + op_node = list(ov_input.get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Divide' or op_node.get_type_name() == 'Multiply') + + def test_scale_vector(self): + argv = Namespace(mean_scale_values={'input1': {'scale': np.array([4.]), 'mean': None}}, scale=None) + function = create_function2() + process_function(ov_function=function, argv=argv) + op_node = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Divide' or op_node.get_type_name() == 'Multiply') + self.check_scale_constant(op_node, [4.0], shape=None) + # Verify that input2 is not affected + op_node = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertEqual(op_node.get_type_name(), 'Relu') + + def test_scale_vector3(self): + argv = Namespace(mean_scale_values={'input1': {'scale': np.array([2., 4., 8.]), 'mean': None}}, scale=None) + function = create_function2(shape1=[1, 3, 224, 224]) + process_function(ov_function=function, argv=argv) + op_node = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Divide' or op_node.get_type_name() == 'Multiply') + self.check_scale_constant(op_node, expected=[2., 4., 8.], shape=[1, 3, 1, 1]) + + # Verify that input2 is not affected + op_node = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertEqual(op_node.get_type_name(), 'Relu') + + # Verify that guessed layout (?C??) is not appeared in input1 + self.assertEqual(function.get_parameters()[0].layout, Layout()) + + def test_scale_vector4_layout(self): + argv = Namespace(mean_scale_values={'input1': {'scale': np.array([2., 4., 8., 9.]), 'mean': None}}, + layout_values={'input1': {'source_layout': 'nhwc'}}, + scale=None) + function = create_function2(shape1=[1, 3, 3, 4]) # Use layout to determine channels dim + + process_function(ov_function=function, argv=argv) + op_node = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Divide' or op_node.get_type_name() == 'Multiply') + self.check_scale_constant(op_node, expected=[2., 4., 8., 9.], shape=[1, 1, 1, 4]) + + # Verify that input2 is not affected + op_node = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertEqual(op_node.get_type_name(), 'Relu') + + # Verify that layout (NHWC) is appeared in input1 + self.assertEqual(function.get_parameters()[0].layout, Layout('nhwc')) + + def test_mean_single(self): + argv = Namespace(mean_scale_values={'input1': {'mean': np.array([4.]), 'scale': None}}, scale=None) + function = create_function2() + process_function(ov_function=function, argv=argv) + op_node = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Subtract' or op_node.get_type_name() == 'Add') + self.check_mean_constant(op_node, [4.0], shape=None) + # Verify that input2 is not affected + op_node = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertEqual(op_node.get_type_name(), 'Relu') + + def test_mean_single_fp64(self): + argv = Namespace(mean_scale_values={'input1': {'mean': np.array([4.]), 'scale': None}}, scale=None) + function = create_function2(dtype1=np.float64) + process_function(ov_function=function, argv=argv) + op_node = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Subtract' or op_node.get_type_name() == 'Add') + self.check_mean_constant(op_node, [4.0], shape=None) + # Verify that input2 is not affected + op_node = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertEqual(op_node.get_type_name(), 'Relu') + + def test_mean_single_fp16(self): + argv = Namespace(mean_scale_values={'input1': {'mean': np.array([4.]), 'scale': None}}, scale=None) + function = create_function2(dtype1=np.float16) + process_function(ov_function=function, argv=argv) + op_node = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Subtract' or op_node.get_type_name() == 'Add') + # Verify that input2 is not affected + op_node = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertEqual(op_node.get_type_name(), 'Relu') + + def test_mean_vector3(self): + argv = Namespace(mean_scale_values={'input2': {'mean': np.array([2., 4., 8.]), 'scale': None}}, scale=None) + function = create_function2(shape2=[1, 3, 224, 224]) + process_function(ov_function=function, argv=argv) + op_node = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Subtract' or op_node.get_type_name() == 'Add') + self.check_mean_constant(op_node, expected=[2., 4., 8.], shape=[1, 3, 1, 1]) + + # Verify that input1 is not affected + op_node = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertEqual(op_node.get_type_name(), 'Relu') + + # Verify that guessed layout (?C??) is not appeared in input2 + self.assertEqual(function.get_parameters()[1].layout, Layout()) + + def test_mean_scale(self): + argv = Namespace(mean_scale_values={'input2a': {'mean': np.array([1., 2., 3.]), + 'scale': np.array([2., 4., 8.])}}, + scale=None) + function = create_function2(shape2=[1, 3, 224, 224]) + process_function(ov_function=function, argv=argv) + # Verify that first is 'subtract mean', then 'scale' + op_node = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Subtract' or op_node.get_type_name() == 'Add') + self.check_mean_constant(op_node, expected=[1., 2., 3.], shape=[1, 3, 1, 1]) + + op_node = list(op_node.output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Divide' or op_node.get_type_name() == 'Multiply') + self.check_scale_constant(op_node, expected=[2., 4., 8.], shape=[1, 3, 1, 1]) + + # Verify that input1 is not affected + op_node = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertEqual(op_node.get_type_name(), 'Relu') + + # Verify that guessed layout (?C??) is not appeared in input2 + self.assertEqual(function.get_parameters()[1].layout, Layout()) + + def test_mean_scale_with_layout(self): + argv = Namespace(mean_scale_values={'input2a': {'mean': np.array([1., 2., 3., 4.]), + 'scale': np.array([2., 4., 8., 9.])}}, + scale=None) + function = create_function2(shape2=[1, 3, 3, 4]) + function.get_parameters()[1].layout = Layout("NHWC") + process_function(ov_function=function, argv=argv) + # Verify that first is 'subtract mean', then 'scale' + op_node = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Subtract' or op_node.get_type_name() == 'Add') + self.check_mean_constant(op_node, expected=[1., 2., 3., 4.], shape=[1, 1, 1, 4]) + + op_node = list(op_node.output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Divide' or op_node.get_type_name() == 'Multiply') + self.check_scale_constant(op_node, expected=[2., 4., 8., 9.], shape=[1, 1, 1, 4]) + + # Verify that input1 is not affected + op_node = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertEqual(op_node.get_type_name(), 'Relu') + + # Verify that layout presents in function after preprocessing + self.assertEqual(function.get_parameters()[1].layout, Layout("NHWC")) + + def test_mean_scale_with_layout_dynamic(self): + argv = Namespace(mean_scale_values={'input2a': {'mean': np.array([1., 2., 3., 4.]), + 'scale': np.array([2., 4., 8., 9.])}}, + scale=None) + function = create_function2(shape2=[-1, -1, -1, -1]) + function.get_parameters()[1].layout = Layout("NHWC") + process_function(ov_function=function, argv=argv) + # Verify that first is 'subtract mean', then 'scale' + op_node = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Subtract' or op_node.get_type_name() == 'Add') + self.check_mean_constant(op_node, expected=[1., 2., 3., 4.], shape=[1, 1, 1, 4]) + + op_node = list(op_node.output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Divide' or op_node.get_type_name() == 'Multiply') + self.check_scale_constant(op_node, expected=[2., 4., 8., 9.], shape=[1, 1, 1, 4]) + + # Verify that input1 is not affected + op_node = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertEqual(op_node.get_type_name(), 'Relu') + + # Verify that layout presents in function after preprocessing + self.assertEqual(function.get_parameters()[1].layout, Layout("NHWC")) + + def test_no_param_name(self): + argv = Namespace(mean_scale_values=list(np.array([(np.array([1., 2., 3.]), np.array([2., 4., 6.])), + (np.array([7., 8., 9.]), None)], + dtype='object')), scale=None) + function = create_function2(shape1=[1, 3, 224, 224], shape2=[1, 224, 224, 3]) + process_function(ov_function=function, argv=argv) + + op_node = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Subtract' or op_node.get_type_name() == 'Add') + self.check_mean_constant(op_node, expected=[1., 2., 3.], shape=[1, 3, 1, 1]) + + op_node = list(op_node.output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Divide' or op_node.get_type_name() == 'Multiply') + self.check_scale_constant(op_node, expected=[2., 4., 6.], shape=[1, 3, 1, 1]) + + op_node = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Subtract' or op_node.get_type_name() == 'Add') + self.check_mean_constant(op_node, expected=[7., 8., 9.], shape=[1, 1, 1, 3]) + + # Verify that guessed layouts are not appeared in inputs + self.assertEqual(function.get_parameters()[0].layout, Layout()) + self.assertEqual(function.get_parameters()[1].layout, Layout()) + + def test_no_param_name_single_value(self): + argv = Namespace(mean_scale_values=list(np.array([(np.array([1.]), None), + (np.array([2., 3., 4.]), np.array([5.]))], + dtype='object')), scale=None) + function = create_function2(shape1=[1, 3, 224, 224], shape2=[1, 224, 224, 3]) + process_function(ov_function=function, argv=argv) + + op_node = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Subtract' or op_node.get_type_name() == 'Add') + self.check_mean_constant(op_node, expected=[1.], shape=None) + + op_node = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Subtract' or op_node.get_type_name() == 'Add') + self.check_mean_constant(op_node, expected=[2., 3., 4.], shape=[1, 1, 1, 3]) + + op_node = list(op_node.output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Divide' or op_node.get_type_name() == 'Multiply') + self.check_scale_constant(op_node, expected=[5.], shape=None) + + # Two inputs, but 'mean_scale_value' has only one array + def test_error_no_param_name_number_not_match(self): + argv = Namespace(mean_scale_values=[(np.array([2., 3.]), np.array([4.]))], scale=None) + function = create_function2(shape1=[1, 3, 224, 224], shape2=[1, 2, 224, 224]) + with self.assertRaisesRegex(Error, '.*question.*61.*'): + process_function(ov_function=function, argv=argv) + + def test_mean_scale_error_no_node_name_found(self): + argv = Namespace(mean_scale_values={'not_found': {'scale': np.array([1.]), 'mean': np.array([1.])}}, + scale=None) + function = create_function2(shape1=[1, 3, 224, 224], shape2=[1, 2, 224, 224]) + with self.assertRaisesRegex(Error, '.*question.*83.*'): + process_function(ov_function=function, argv=argv) + + def test_layout_error_no_node_name_found(self): + argv = Namespace(layout_values={'not_found': {'source_layout': 'nhwc'}}, + scale=None) + function = create_function2(shape1=[1, 3, 224, 224], shape2=[1, 2, 224, 224]) + with self.assertRaisesRegex(Error, '.*question.*83.*'): + process_function(ov_function=function, argv=argv) + + def test_error_dimension_mismatch(self): + argv = Namespace(mean_scale_values={'input1': {'scale': np.array([1., 2., 3., 4.]), 'mean': None}}, + scale=None) + function = create_function2(shape1=[1, 3, 224, 224]) + with self.assertRaises(Exception): + process_function(ov_function=function, argv=argv) + + def test_error_dimension_not_clear(self): + argv = Namespace(mean_scale_values={'input1': {'scale': np.array([1., 2., 3.]), 'mean': None}}, + scale=None) + function = create_function2(shape1=[1, 3, 3, 3]) # Not clear to which 3 should scale be applied + with self.assertRaises(Exception): + process_function(ov_function=function, argv=argv) + + def test_error_dimension_mismatch_with_scale(self): + argv = Namespace(mean_scale_values={'input1': {'scale': np.array([1., 2., 3., 4.]), + 'mean': np.array([1., 2., 3.])}}, + scale=None) + function = create_function2(shape1=[1, 3, 4, 224]) + with self.assertRaises(Exception): + process_function(ov_function=function, argv=argv) + + def test_error_guess_c_wrong_position_3d(self): + argv = Namespace(mean_scale_values={'input1': {'scale': np.array([1., 2., 3.]), + 'mean': np.array([1., 2., 3.])}}, + scale=None) + function = create_function2(shape1=[2, 3, 4]) + with self.assertRaises(Exception): + process_function(ov_function=function, argv=argv) + + def test_error_guess_c_wrong_position_4d(self): + argv = Namespace(mean_scale_values={'input1': {'scale': np.array([1., 2., 3.]), + 'mean': np.array([1., 2., 3.])}}, + scale=None) + function = create_function2(shape1=[1, 2, 3, 4]) + with self.assertRaises(Exception): + process_function(ov_function=function, argv=argv) + + def test_error_guess_c_wrong_position_5d(self): + argv = Namespace(mean_scale_values={'input1': {'scale': np.array([1., 2., 3.]), + 'mean': np.array([1., 2., 3.])}}, + scale=None) + function = create_function2(shape1=[1, 2, 3, 4, 5]) + with self.assertRaises(Exception): + process_function(ov_function=function, argv=argv) + + def test_error_guess_c_wrong_position_6d(self): + argv = Namespace(mean_scale_values={'input1': {'scale': np.array([1., 2., 3.]), + 'mean': np.array([1., 2., 3.])}}, + scale=None) + function = create_function2(shape1=[1, 2, 4, 5, 6, 3]) + with self.assertRaises(Exception): + process_function(ov_function=function, argv=argv) + + def test_error_2_names_to_same_input(self): + argv = Namespace(mean_scale_values={'input1': {'scale': np.array([1., 2., 3.])}, + 'input1a': {'scale': np.array([1., 2., 3.])}}, + scale=None) + function = create_function2(shape1=[1, 3, 224, 224]) + with self.assertRaises(Exception): + process_function(ov_function=function, argv=argv) + + def test_error_2_names_to_same_input_single_value(self): + argv = Namespace(mean_scale_values={'input1': {'scale': np.array([2.])}, + 'input1a': {'scale': np.array([3.])}}, + scale=None) + function = create_function2(shape1=[1, 3, 224, 224]) + with self.assertRaises(Exception): + process_function(ov_function=function, argv=argv) + + def test_reverse_input_channels(self): + argv = Namespace(reverse_input_channels=True, mean_scale_values=None, scale=None) + function = create_function2(shape1=[1, 224, 224, 3], shape2=[1, 3, 224, 224]) + process_function(ov_function=function, + argv=argv) + # Verify that some operations are inserted. + # In future, consider using mock PrePostProcessor to verify that 'reverse_channels' was called + op_node0 = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node0.get_type_name() != 'Relu') + op_node1 = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node1.get_type_name() != 'Relu') + + # Verify that guessed layouts are not appeared in input1,input2 + self.assertEqual(function.get_parameters()[0].layout, Layout()) + self.assertEqual(function.get_parameters()[1].layout, Layout()) + + def test_reverse_input_channels_func_layout(self): + argv = Namespace(reverse_input_channels=True, mean_scale_values=None, scale=None) + function = create_function2(shape1=[1, 3, 3, 3], shape2=[1, 3, 3, 3]) + function.get_parameters()[0].layout = Layout("NCHW") + function.get_parameters()[1].layout = Layout("NHWC") + process_function(ov_function=function, + argv=argv) + # Verify that some operations are inserted. + # In future, consider using mock PrePostProcessor to verify that 'reverse_channels' was called + op_node0 = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node0.get_type_name() != 'Relu') + op_node1 = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node1.get_type_name() != 'Relu') + + # Verify that guessed layouts are not appeared in input1,input2 + self.assertEqual(function.get_parameters()[0].layout, Layout("NCHW")) + self.assertEqual(function.get_parameters()[1].layout, Layout("NHWC")) + + def test_reverse_input_channels_layout(self): + argv = Namespace(reverse_input_channels=True, mean_scale_values=None, scale=None, + layout_values={'input1a': { 'source_layout': 'nhwc' }, + 'input2a': { 'source_layout': 'nchw' } + }) + function = create_function2(shape1=[1, 224, 224, 4], shape2=[1, 4, 224, 224]) + process_function(ov_function=function, argv=argv) + # In future, consider using mock PrePostProcessor to verify that 'reverse_channels' was not called + # Verify that reverse_channels are not applied. + op_node0 = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node0.get_type_name() == 'Relu') + op_node1 = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node1.get_type_name() == 'Relu') + + def test_reverse_input_channels_3d(self): + argv = Namespace(reverse_input_channels=True, mean_scale_values=None, scale=None, + layout_values=None) + function = create_function2(shape1=[224, 224, 3], shape2=[3, 224, 224]) + process_function(ov_function=function, argv=argv) + # Verify that reverse_channels are applied. + op_node0 = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node0.get_type_name() != 'Relu') + op_node1 = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node1.get_type_name() != 'Relu') + + def test_reverse_input_channels_6d(self): + argv = Namespace(reverse_input_channels=True, mean_scale_values=None, scale=None, + layout_values=None) + function = create_function2(shape1=[4, 4, 4, 4, 4, 3], shape2=[4, 3, 4, 4, 4, 4]) + process_function(ov_function=function, argv=argv) + # Verify that reverse_channels are NOT applied. + op_node0 = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node0.get_type_name() == 'Relu') + op_node1 = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node1.get_type_name() == 'Relu') + + def test_reverse_input_channels_dynamic(self): + argv = Namespace(reverse_input_channels=True, mean_scale_values=None, scale=None, + layout_values=None) + function = create_function2(shape1=[1, -1, 5, 5], shape2=[-1, -1, -1, -1]) + process_function(ov_function=function, argv=argv) + # Verify that reverse_channels are NOT applied. + op_node0 = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node0.get_type_name() == 'Relu') + op_node1 = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node1.get_type_name() == 'Relu') + + def test_reverse_input_channels_dynamic_layout(self): + argv = Namespace(reverse_input_channels=True, mean_scale_values=None, scale=None, + layout_values={'input1a': { 'source_layout': 'nchw' }, + 'input2a': { 'source_layout': 'nhwc' } + }) + function = create_function2(shape1=[1, -1, 5, 5], shape2=[-1, -1, -1, -1]) + process_function(ov_function=function, argv=argv) + # Verify that reverse_channels are applied. + op_node0 = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node0.get_type_name() != 'Relu') + op_node1 = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node1.get_type_name() != 'Relu') + + def test_reverse_input_channels_2_channels(self): + argv = Namespace(reverse_input_channels=True, + mean_scale_values=None, + scale=None) + function = create_function2(shape1=[1, 224, 224, 2], shape2=[1, 3, 224, 224]) + process_function(ov_function=function, argv=argv) + # Verify that some operations are inserted to input2. + op_node0 = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node0.get_type_name() == 'Relu') + op_node1 = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node1.get_type_name() != 'Relu') + + # Verify that guessed layouts are not appeared in input1,input2 + self.assertEqual(function.get_parameters()[0].layout, Layout()) + self.assertEqual(function.get_parameters()[1].layout, Layout()) + + # When input name for layout is empty for model with one input - it is applied to this input + def test_scale_vector3_layout_empty_input_name(self): + argv = Namespace(mean_scale_values=list(np.array([(None, np.array([2., 4., 8.]))], + dtype='object')), + layout_values={'': {'source_layout': 'nchw'}}, + scale=None) + function = create_function1(shape1=[1, 3, 3, 3]) # Use layout to determine channels dim + + process_function(ov_function=function, argv=argv) + op_node = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node.get_type_name() == 'Divide' or op_node.get_type_name() == 'Multiply') + self.check_scale_constant(op_node, expected=[2., 4., 8.], shape=[1, 3, 1, 1]) + + # Verify that layout (nchw) is appeared in input1 + self.assertEqual(function.get_parameters()[0].layout, Layout('nchw')) + + def test_layout_output(self): + argv = Namespace(mean_scale_values=None, + layout_values={ + 'res1': { + 'source_layout': 'nchw', + 'target_layout': 'nhwc' + }, + 'res2a': { + 'source_layout': 'ncdhw' + } + }, + scale=None) + function = create_function2(shape1=[1, 3, 3, 3], shape2=[1, 3, 3, 3, 3]) + + process_function(ov_function=function, argv=argv) + op_node = function.get_results()[0].input(0).get_source_output().get_node() + self.assertEqual(op_node.get_type_name(), 'Transpose') + + self.assertEqual(function.get_results()[0].layout, Layout('nhwc')) + self.assertEqual(function.get_results()[1].layout, Layout('ncdhw')) + + def test_error_layout_empty_input_name_2_inputs(self): + argv = Namespace(mean_scale_values=None, + layout_values={'': {'source_layout': 'nchw'}}, + scale=None) + function = create_function2(shape1=[1, 3, 3, 3]) + + # Verify user friendly error message contains number of inputs and their names + with self.assertRaisesRegex(Error, '.*2.*inputs.*input1.*input2.*'): + process_function(ov_function=function, argv=argv) + + def test_reverse_channels_bad_layout(self): + argv = Namespace(reverse_input_channels=True, mean_scale_values=None, scale=None) + function = create_function2(shape1=[1, 224, 224, 3], shape2=[1, 4, 224, 224]) + function.get_parameters()[0].layout = Layout("NDHWC") + with self.assertRaisesRegex(Error, '.*input1.*'): + process_function(ov_function=function, argv=argv) + + def test_guess_layout_reverse_channels_dont_apply_to_4(self): + argv = Namespace(reverse_input_channels=True, mean_scale_values=None, scale=None) + function = create_function2(shape1=[1, 224, 224, 3], shape2=[1, 4, 224, 224]) + process_function(ov_function=function, argv=argv) + + op_node0 = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node0.get_type_name() != 'Relu') + op_node1 = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node1.get_type_name() == 'Relu') + + def test_error_guess_layout_reverse_channels_multi_3(self): + argv = Namespace(reverse_input_channels=True, mean_scale_values=None, scale=None) + function = create_function2(shape1=[1, 224, 224, 3], shape2=[1, 3, 3, 224]) + process_function(ov_function=function, argv=argv) + # Applied to only input1 + op_node0 = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node0.get_type_name() != 'Relu') + op_node1 = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node1.get_type_name() == 'Relu') + + + def test_no_guess_layout_reverse_channels_has_layout_no_c(self): + argv = Namespace(reverse_input_channels=True, mean_scale_values=None, scale=None) + function = create_function2(shape1=[1, 224, 224, 3], shape2=[1, 3, 224, 224]) + function.get_parameters()[0].layout = Layout("NHW?") + function.get_parameters()[1].layout = Layout("N?HW") + process_function(ov_function=function, argv=argv) + # Nothing has applied + op_node0 = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node0.get_type_name() == 'Relu') + op_node1 = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node1.get_type_name() == 'Relu') + + def test_guess_layout_reverse_channels_incorrect_pos(self): + argv = Namespace(reverse_input_channels=True, mean_scale_values=None, scale=None) + function = create_function2(shape1=[1, 4, 224, 224], shape2=[1, 224, 224, 2]) + function.get_parameters()[0].layout = Layout("NCHW") + function.get_parameters()[1].layout = Layout("NHWC") + process_function(ov_function=function, argv=argv) + # Nothing has applied + op_node0 = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node0.get_type_name() == 'Relu') + op_node1 = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node1.get_type_name() == 'Relu') + + def test_no_reverse_channels_even_with_layout(self): + argv = Namespace(reverse_input_channels=True, mean_scale_values=None, scale=None) + function = create_function2(shape1=[3, 4, 224, 224], shape2=[1, 224, 3, 224]) + process_function(ov_function=function, argv=argv) + # Nothing has applied + op_node0 = list(function.get_parameters()[0].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node0.get_type_name() == 'Relu') + op_node1 = list(function.get_parameters()[1].output(0).get_target_inputs())[0].get_node() + self.assertTrue(op_node1.get_type_name() == 'Relu') diff --git a/model-optimizer/unit_tests/mo/frontend_ngraph_test.py b/model-optimizer/unit_tests/mo/frontend_ngraph_test.py index f440284b8f9..43aa1749483 100644 --- a/model-optimizer/unit_tests/mo/frontend_ngraph_test.py +++ b/model-optimizer/unit_tests/mo/frontend_ngraph_test.py @@ -43,6 +43,15 @@ def test_moc_extractor(): assert not status.returncode +def test_moc_preprocessing(): + setup_env() + args = [sys.executable, '-m', 'pytest', + os.path.join(os.path.dirname(__file__), 'back/moc_preprocessing_test_actual.py'), '-s'] + + status = subprocess.run(args, env=os.environ) + assert not status.returncode + + def test_main_test(): setup_env() args = [sys.executable, '-m', 'pytest', diff --git a/src/bindings/python/src/openvino/runtime/__init__.py b/src/bindings/python/src/openvino/runtime/__init__.py index d62a22a9333..4975a289736 100644 --- a/src/bindings/python/src/openvino/runtime/__init__.py +++ b/src/bindings/python/src/openvino/runtime/__init__.py @@ -44,6 +44,7 @@ from openvino.runtime.impl import Function from openvino.runtime.impl import Node from openvino.runtime.impl import PartialShape from openvino.runtime.impl import Layout +from openvino.runtime.impl import layout_helpers from openvino.runtime.ie_api import Core from openvino.runtime.ie_api import ExecutableNetwork diff --git a/src/bindings/python/src/pyopenvino/graph/layout.cpp b/src/bindings/python/src/pyopenvino/graph/layout.cpp index 8ce4fdf77bf..bbf58a04069 100644 --- a/src/bindings/python/src/pyopenvino/graph/layout.cpp +++ b/src/bindings/python/src/pyopenvino/graph/layout.cpp @@ -39,4 +39,5 @@ void regclass_graph_Layout(py::module m) { layout.def("__str__", [](const ov::Layout& self) { return self.to_string(); }); + layout.def_property_readonly("empty", &ov::Layout::empty); } diff --git a/src/core/src/preprocess/preprocess_steps_impl.cpp b/src/core/src/preprocess/preprocess_steps_impl.cpp index f970c61f2a8..19e42bc45dd 100644 --- a/src/core/src/preprocess/preprocess_steps_impl.cpp +++ b/src/core/src/preprocess/preprocess_steps_impl.cpp @@ -43,9 +43,15 @@ void PreStepsList::add_scale_impl(const std::vector& values) { if (values.size() == 1) { shape = Shape{1}; } else { - shape = construct_mean_scale_shape(nodes[0].get_node_shared_ptr(), values.size(), context); + shape = construct_mean_scale_shape(nodes[0], values.size(), context); } - auto constant = op::v0::Constant::create(element::f32, shape, values); + auto element_type = nodes[0].get_element_type(); + OPENVINO_ASSERT(element_type.is_real(), + "Scale preprocessing can be applied to 'float' inputs. Consider using of " + "'convert_element_type' before scaling. Current type is: ", + element_type); + + auto constant = op::v0::Constant::create(element_type, shape, values); auto new_op = std::make_shared(nodes[0], constant); return std::make_tuple(std::vector>{new_op}, false); @@ -66,7 +72,13 @@ void PreStepsList::add_mean_impl(const std::vector& values) { } else { shape = construct_mean_scale_shape(nodes[0], values.size(), context); } - auto constant = op::v0::Constant::create(element::f32, shape, values); + auto element_type = nodes[0].get_element_type(); + OPENVINO_ASSERT(element_type.is_real(), + "Mean preprocessing can be applied to 'float' inputs. Consider using of 'convert_element_type' " + "before scaling. Current type is: ", + element_type); + + auto constant = op::v0::Constant::create(element_type, shape, values); auto new_op = std::make_shared(nodes[0], constant); return std::make_tuple(std::vector>{new_op}, false); diff --git a/src/core/tests/preprocess.cpp b/src/core/tests/preprocess.cpp index 19e407acc77..4ecb889b943 100644 --- a/src/core/tests/preprocess.cpp +++ b/src/core/tests/preprocess.cpp @@ -52,12 +52,20 @@ TEST(pre_post_process, simple_mean_scale) { EXPECT_EQ(f->get_output_element_type(0), element::f32); } -TEST(pre_post_process, simple_mean_scale_getters) { - auto f = create_simple_function(element::f32, Shape{1, 3, 2, 2}); +TEST(pre_post_process, simple_mean_scale_getters_f16) { + auto f = create_simple_function(element::f16, Shape{1, 3, 2, 2}); auto p = PrePostProcessor(f); p.input("tensor_input1").preprocess().mean(1).scale(2); f = p.build(); - EXPECT_EQ(f->get_output_element_type(0), element::f32); + EXPECT_EQ(f->get_output_element_type(0), element::f16); +} + +TEST(pre_post_process, simple_mean_scale_getters_f64) { + auto f = create_simple_function(element::f64, Shape{1, 3, 2, 2}); + auto p = PrePostProcessor(f); + p.input("tensor_input1").preprocess().mean(1).scale(2); + f = p.build(); + EXPECT_EQ(f->get_output_element_type(0), element::f64); } TEST(pre_post_process, convert_element_type_and_scale) {