IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[MO][TF FE] Support delayed batch setting (#16937)

Browse Source 
* [TF FE] Support delayed batch setting

Signed-off-by: Kazantsev, Roman <roman.kazantsev@intel.com>

* Cover BOM list

* Add unit-tests for batch setting with layout

* Apply code-review: check batch size

* Apply code-review: default index for any dimension

---------

Signed-off-by: Kazantsev, Roman <roman.kazantsev@intel.com>
This commit is contained in:
Roman Kazantsev 2023-04-15 02:35:43 +04:00 committed by GitHub
parent 8bdc5bc85f
commit 4ba0ac5476
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 406 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tools/mo/automation/package_BOM.txt
View File

@ -835,6 +835,7 @@ openvino/tools/mo/moc_frontend/__init__.py
openvino/tools/mo/moc_frontend/analysis.py
openvino/tools/mo/moc_frontend/check_config.py
openvino/tools/mo/moc_frontend/extractor.py
openvino/tools/mo/moc_frontend/layout_utils.py
openvino/tools/mo/moc_frontend/pipeline.py
openvino/tools/mo/moc_frontend/pytorch_frontend_utils.py
openvino/tools/mo/moc_frontend/serialize.py

54
tools/mo/openvino/tools/mo/back/preprocessing.py
View File

@ -4,14 +4,14 @@
import argparse
import logging as log
from openvino.tools.mo.utils.error import Error
from openvino.tools.mo.utils.utils import refer_to_faq_msg
import numpy as np
from openvino.preprocess import PrePostProcessor # pylint: disable=no-name-in-module,import-error
from openvino.preprocess import PrePostProcessor # pylint: disable=no-name-in-module,import-error
# pylint: disable=no-name-in-module,import-error
from openvino.runtime import Model, Layout, PartialShape, layout_helpers
from openvino.tools.mo.moc_frontend.layout_utils import update_layout_to_dict
from openvino.tools.mo.utils.error import Error
from openvino.tools.mo.utils.utils import refer_to_faq_msg
def update_mean_scale_to_dict(input_nodes: list, mean_scale_val, scale):
@ -61,32 +61,6 @@ def update_mean_scale_to_dict(input_nodes: list, mean_scale_val, scale):
return mean_scale_val
def update_layout_to_dict(input_nodes: list, layout: [list, dict]):
"""
Internal function. Updates layout values from array to dictionary
:param: input_nodes Inputs of model
:param: layout Parsed 'layout' object from command line arguments
"""
if isinstance(layout, dict):
return layout
if isinstance(layout, list):
if len(layout) != len(input_nodes):
raise Error('Numbers of inputs and mean/scale values do not match. ' + refer_to_faq_msg(61))
layout_dict = {}
for idx, node in enumerate(input_nodes):
names_list = list(node.get_tensor().get_names())
if not names_list:
raise Error("Empty tensor names list for node {}".format(node.name))
node_name = names_list[0]
layout_dict.update(
{
node_name: layout[idx]
}
)
return layout_dict
raise Error("Unknown layout type. Expected dict, list. Got {}".format(type(layout)))
def check_keys_valid(ov_function: Model, dict_to_validate: dict, search_outputs: bool):
"""
Internal function: checks if keys from cmd line arguments correspond to ov_function's inputs/outputs
@ -201,7 +175,7 @@ def find_channels_dimension(shape: PartialShape, num_channels: int, name: str, l
.format(shape.rank.get_length(), name, shape))
layout_str = "?" * shape.rank.get_length()
layout_str = layout_str[:dim_idx_found] + 'C' + layout_str[dim_idx_found+1:]
layout_str = layout_str[:dim_idx_found] + 'C' + layout_str[dim_idx_found + 1:]
layout_values[name] = {
'source_layout': layout_str,
'target_layout': None,
@ -361,7 +335,7 @@ def update_tensor_names_to_first_in_sorted_list(values_dict: dict, ov_function:
for input in ov_function.inputs:
tensor_names = list(input.names)
tensor_names.sort()
if not(name in tensor_names or name == input.node.get_friendly_name()):
if not (name in tensor_names or name == input.node.get_friendly_name()):
continue
if input in used_nodes:
raise Error("Tensor names {} and {} refer to the same node.".format(name, used_nodes[input]))
@ -418,21 +392,9 @@ def apply_preprocessing(ov_function: Model, argv: argparse.Namespace):
layout_values = {}
if 'layout_values' in argv and argv.layout_values:
layout_values = update_layout_to_dict(ov_function.inputs, argv.layout_values)
layout_values = update_layout_to_dict(ov_function.inputs, argv.layout_values,
lambda ov_input: ov_input.get_tensor().get_names())
if '' in layout_values:
if len(ov_function.inputs) > 1:
input_names = [list(ov_input.get_tensor().get_names())[0] for ov_input in ov_function.inputs]
raise Error('Layout without name can be specified for models with only one input, '
'but provided model has {} inputs: \'{}\'. '
'Please specify explicitly input/output name for --layout option'
.format(len(input_names), input_names))
layout_values = {
list(ov_function.input().get_tensor().get_names())[0]: {
'source_layout': layout_values[''].get('source_layout'),
'target_layout': layout_values[''].get('target_layout')
}
}
check_keys_valid(ov_function=ov_function, dict_to_validate=mean_scale_values, search_outputs=False)
check_keys_valid(ov_function=ov_function, dict_to_validate=layout_values, search_outputs=True)

10
tools/mo/openvino/tools/mo/convert.py
View File

@ -5,8 +5,7 @@ import pathlib
from collections import namedtuple
from typing import Any
from openvino.runtime import PartialShape, Shape, Layout
from openvino.runtime import PartialShape, Shape, Layout, Model
from openvino.tools.mo.convert_impl import _convert
from openvino.tools.mo.utils.cli_parser import get_all_cli_parser
from openvino.tools.mo.utils.logger import get_logger_state, restore_logger_state
@ -80,7 +79,7 @@ def convert_model(
remove_memory: bool = False,
**args
):
) -> Model:
"""
Converts the model from original framework to OpenVino Model.
@ -160,7 +159,10 @@ def convert_model(
for a model with two inputs with 4D and 2D shapes. Alternatively, specify
shapes with the --input option.
:param batch:
Input batch size
Set batch size. It applies to 1D or higher dimension inputs.
The default dimension index for the batch is zero.
Use a label 'n' in --layout or --source_layout option to set the batch dimension.
For example, "x(hwnc)" defines the third dimension to be the batch.
:param mean_values:
Mean values to be used for the input image per channel. Mean values can
be set by passing a dictionary, where key is input name and value is mean

73
tools/mo/openvino/tools/mo/moc_frontend/layout_utils.py Normal file
View File

@ -0,0 +1,73 @@
# Copyright (C) 2018-2023 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
from typing import Callable
from openvino.runtime import PartialShape # pylint: disable=no-name-in-module,import-error
from openvino.tools.mo.utils.error import Error
from openvino.tools.mo.utils.utils import refer_to_faq_msg
def update_layout_to_dict(inputs: list, layout: [list, dict], get_names_func: Callable):
"""
The function prepares layout values in the dictionary with items of the format:
{ node_name : {'source_layout': 'NHWC', 'target_layout': 'NCHW'} }
"""
if isinstance(layout, dict):
if '' in layout:
input_names = [list(get_names_func(cur_input))[0] for cur_input in inputs]
if len(input_names) > 1:
raise Error('Layout without name can be specified for models with only one input, '
'but provided model has {} inputs: \'{}\'. '
'Please specify explicitly input/output name for --layout option'
.format(len(input_names), input_names))
layout = {
input_names[0]: {
'source_layout': layout[''].get('source_layout'),
'target_layout': layout[''].get('target_layout')
}
}
return layout
if isinstance(layout, list):
if len(layout) != len(inputs):
raise Error('Numbers of inputs and layout values do not match. ' + refer_to_faq_msg(61))
layout_dict = {}
for idx, cur_input in enumerate(inputs):
names_list = list(get_names_func(cur_input))
assert len(names_list) > 0, "No names for input"
node_name = names_list[0]
layout_dict.update(
{
node_name: layout[idx]
}
)
return layout_dict
raise Error("Unknown layout type. Expected dict, list. Got {}".format(type(layout)))
def get_dimension_index_by_label(input_shape: PartialShape, input_names: list, layout_dict: [dict],
dimension_label: str, default_dim: int):
"""
The function returns index of the dimension pointed in the layout
and a flag indicating if the index is chosen by default.
For example, the index for 'D' dimension in "NHWDC" layout is 3.
"""
if input_shape.rank.is_static and input_shape.rank.get_length() == 0:
# in case a scalar, batch dimension is not defined
return None, False
# search for the corresponding layout
for name, layout_value in layout_dict.items():
if name in input_names:
layout = layout_value.get('source_layout', None)
if layout is None:
return default_dim, True
from openvino.runtime import Layout # pylint: disable=no-name-in-module,import-error
layout_parsed = Layout(layout)
if layout_parsed.has_name(dimension_label):
return layout_parsed.get_index_by_name(dimension_label), False
else:
# if the layout is specified and the required dimension label is not found, the batch is unknown
return None, False
return default_dim, True

94
tools/mo/openvino/tools/mo/moc_frontend/pipeline.py
View File

@ -5,18 +5,20 @@ import argparse
import io
import logging as log
import sys
from copy import copy
from typing import List
import numpy as np
from openvino.frontend import FrontEnd, InputModel, NotImplementedFailure, \
Place # pylint: disable=no-name-in-module,import-error
from openvino.runtime import Dimension, PartialShape, Type # pylint: disable=no-name-in-module,import-error
from openvino.runtime import PartialShape, Type # pylint: disable=no-name-in-module,import-error
from openvino.runtime.utils.types import get_element_type, \
get_numpy_ctype # pylint: disable=no-name-in-module,import-error
from openvino.tools.mo.middle.passes.infer import validate_batch_in_shape
from openvino.tools.mo.moc_frontend.analysis import json_model_analysis_dump
from openvino.tools.mo.moc_frontend.extractor import fe_user_data_repack, convert_params_lists_to_dicts
from openvino.tools.mo.moc_frontend.layout_utils import update_layout_to_dict, get_dimension_index_by_label
from openvino.tools.mo.utils.class_registration import get_enabled_and_disabled_transforms
from openvino.tools.mo.utils.error import Error
@ -197,25 +199,89 @@ def moc_pipeline(argv: argparse.Namespace, moc_front_end: FrontEnd):
def shape_to_array(shape: PartialShape):
return [shape.get_dimension(i) for i in range(shape.rank.get_length())]
# Set batch size
# obtain layout for all inputs
layout_values = {}
if 'layout_values' in argv and argv.layout_values:
layout_values = update_layout_to_dict(model_inputs, argv.layout_values,
lambda input_place: input_place.get_names())
deferred_batch_names = []
# set batch size for inputs with a static rank
# for all other inputs, set it after shape deduction is performed during model conversion
if argv.batch is not None and argv.batch > 0:
log.debug('Setting batch size to {}'.format(argv.batch))
frozen_input_names = list(freeze_placeholder.keys()) if freeze_placeholder else []
for place in model_inputs:
old_partial_shape = input_model.get_partial_shape(place)
old_shape_array = shape_to_array(old_partial_shape) if old_partial_shape.rank.is_static else []
input_partial_shape = input_model.get_partial_shape(place)
input_names = place.get_names()
joined_name = ' '.join(place.get_names())
validate_batch_in_shape(old_shape_array, joined_name)
assert len(input_names) > 0, "One input place has no names"
# Assume batch size is always 1-st dimension in shape
# Keep other dimensions unchanged
new_shape = [old_partial_shape.get_dimension(i)
for i in range(old_partial_shape.rank.get_length())]
new_shape[0] = Dimension(argv.batch)
# if this input is frozen, there is no need to set the batch
is_frozen_input = len([name for name in input_names if name in frozen_input_names]) > 0
if is_frozen_input:
# skip the frozen input
continue
if not input_partial_shape.rank.is_static:
# found input with dynamic rank, so have to repeat the batch setting after the model conversion
deferred_batch_names += input_names
continue
batch_dim, is_default_index = get_dimension_index_by_label(input_partial_shape,
place.get_names(), layout_values, 'N', 0)
if batch_dim is None:
# skip because no batch dimension exists in the input
continue
if is_default_index:
# if the batch index is chosen by default, we need to ensure that its size equals -1, 0 or 1
validate_batch_in_shape(shape_to_array(input_partial_shape), joined_name)
assert batch_dim < input_partial_shape.rank.get_length(), \
"Incorrect layout is specified for {}:" \
" index of the batch dimension is out of range.".format(input_names[0])
new_partial_shape = copy(input_partial_shape)
new_partial_shape[batch_dim] = argv.batch
new_partial_shape = PartialShape(new_shape)
log.debug('Input: {}, Old shape: {}, New shape: {}'.format(
joined_name, old_shape_array, new_shape))
joined_name, input_partial_shape, new_partial_shape))
input_model.set_partial_shape(place, new_partial_shape)
ngraph_function = moc_front_end.convert(input_model)
return ngraph_function
ov_model = moc_front_end.convert(input_model)
if argv.batch is not None and argv.batch > 0 and len(deferred_batch_names) > 0:
# Frontend convert method can include reverse infer functionality that can deduce undefined input shapes
# so try to repeat batch setting again
reshape_dict = {}
log.debug('Deferred batch setting to size {}'.format(argv.batch))
is_batch_clarified = False
for model_input in ov_model.inputs:
input_name = model_input.any_name
input_partial_shape = model_input.get_partial_shape()
if input_name in deferred_batch_names and input_partial_shape.rank.is_static:
# update input shape with the specified batch for input that originally has dynamic rank
batch_dim, is_default_index = get_dimension_index_by_label(input_partial_shape,
model_input.get_names(),
layout_values, 'N', 0)
if batch_dim is None:
continue
if is_default_index:
# if the batch index is chosen by default, we need to ensure that its size equals -1, 0 or 1
validate_batch_in_shape(shape_to_array(input_partial_shape), input_name)
assert batch_dim < input_partial_shape.rank.get_length(), \
"Incorrect layout is specified for {}: " \
"index of the batch dimension is out of range.".format(input_name)
input_partial_shape[batch_dim] = argv.batch
is_batch_clarified = True
reshape_dict.update({input_name: input_partial_shape})
if is_batch_clarified:
# call reshape only if batch dimension for one of the input is clarified
ov_model.reshape(reshape_dict)
return ov_model

96
tools/mo/unit_tests/moc_tf_fe/conversion_with_layout.py Normal file
View File

@ -0,0 +1,96 @@
# Copyright (C) 2018-2023 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import os
import unittest
import numpy as np
from generator import generator, generate
import openvino.runtime.opset11 as opset11
from openvino.runtime import Model
from openvino.runtime import PartialShape, Dimension
from openvino.test_utils import compare_functions
from openvino.tools.mo.convert import convert_model
from openvino.tools.mo.utils.error import Error
@generator
class TestConversionWithBatchAndLayout(unittest.TestCase):
def basic_check(self, model_name: str, batch: int, layout: str, refs_shapes: dict):
path = os.path.dirname(__file__)
input_model = os.path.join(path, "test_models", model_name)
ov_model = convert_model(input_model, batch=batch, layout=layout)
for ov_input in ov_model.inputs:
input_name = ov_input.any_name
assert input_name in refs_shapes, "No reference input shape is found for {}".format(input_name)
input_shape = ov_input.get_partial_shape()
ref_shape = refs_shapes[input_name]
assert input_shape == ref_shape, "Incorrect shape for {} input:" \
" expected shape - {}, actual shape - {}".format(input_name, ref_shape,
input_shape)
def test_basic_model_no_layout(self):
path = os.path.dirname(__file__)
input_model = os.path.join(path, "test_models", "model_fp32.pbtxt")
ov_model = convert_model(input_model)
# compare with the reference graph
param1 = opset11.parameter([2, 2], name="in1", dtype=np.float32)
param2 = opset11.parameter([2, 2], name="in2", dtype=np.float32)
add = opset11.add(param1, param2, name="add")
ref_model = Model(add, [param1, param2])
flag, msg = compare_functions(ov_model, ref_model, compare_tensor_names=False)
assert flag, msg
@generate(
*[
(
"model_fp32.pbtxt", 5, "in1(cn),in2(cn)",
{"in1": PartialShape([2, 5]), "in2": PartialShape([2, 5])},
),
(
"model_fp32.pbtxt", 9, "in1(nc),in2(nc)",
{"in1": PartialShape([9, 2]), "in2": PartialShape([9, 2])},
),
(
"model_fp32.pbtxt", 7, "in1(?c),in2(?c)",
{"in1": PartialShape([2, 2]), "in2": PartialShape([2, 2])},
),
],
)
def test_basic_model_with_layout(self, model_name: str, batch: int, layout: str, refs_shapes: dict):
self.basic_check(model_name, batch, layout, refs_shapes)
@generate(
*[
(
"model_with_convolution_dynamic_rank.pbtxt", 7, "x(n???),kernel(????)",
{"x": PartialShape([7, Dimension.dynamic(), Dimension.dynamic(), Dimension.dynamic()]),
"kernel": PartialShape([2, 2, 3, 1])},
),
(
"model_with_convolution_dynamic_rank.pbtxt", 3, "x(???n),kernel(??n?)",
{"x": PartialShape([Dimension.dynamic(), Dimension.dynamic(), Dimension.dynamic(), 3]),
"kernel": PartialShape([2, 2, 3, 1])},
),
],
)
def test_model_with_convolution_dynamic_rank(self, model_name: str, batch: int, layout: str, refs_shapes: dict):
self.basic_check(model_name, batch, layout, refs_shapes)
@generate(
*[
(
"model_fp32.pbtxt", 17, "",
{},
),
],
)
def test_model_expected_failure(self, model_name: str, batch: int, layout: str, refs_shapes: dict):
# try to override batch size by default index (without specifying layout)
with self.assertRaisesRegex(Error,
"When you use -b \(--batch\) option, Model Optimizer applies its value to the first "
"element of the shape if it is equal to -1, 0 or 1\."):
self.basic_check(model_name, batch, layout, refs_shapes)

124
tools/mo/unit_tests/moc_tf_fe/test_models/model_with_convolution_dynamic_rank.pbtxt Normal file
View File

@ -0,0 +1,124 @@
node {
name: "x"
op: "Placeholder"
attr {
key: "dtype"
value {
type: DT_FLOAT
}
}
attr {
key: "shape"
value {
shape {
unknown_rank: true
}
}
}
}
node {
name: "kernel"
op: "Placeholder"
attr {
key: "dtype"
value {
type: DT_FLOAT
}
}
attr {
key: "shape"
value {
shape {
dim {
size: 2
}
dim {
size: 2
}
dim {
size: 3
}
dim {
size: 1
}
}
}
}
}
node {
name: "Conv2D"
op: "Conv2D"
input: "x"
input: "kernel"
attr {
key: "T"
value {
type: DT_FLOAT
}
}
attr {
key: "data_format"
value {
s: "NHWC"
}
}
attr {
key: "dilations"
value {
list {
i: 1
i: 1
i: 1
i: 1
}
}
}
attr {
key: "explicit_paddings"
value {
list {
}
}
}
attr {
key: "padding"
value {
s: "SAME"
}
}
attr {
key: "strides"
value {
list {
i: 1
i: 1
i: 1
i: 1
}
}
}
attr {
key: "use_cudnn_on_gpu"
value {
b: true
}
}
}
node {
name: "Relu"
op: "Relu"
input: "Conv2D"
attr {
key: "T"
value {
type: DT_FLOAT
}
}
}
node {
name: "init"
op: "NoOp"
}
versions {
producer: 808
}

18
tools/mo/unit_tests/moc_tf_fe/test_models/model_with_convolution_dynamic_rank.py Normal file
View File

@ -0,0 +1,18 @@
# Copyright (C) 2018-2023 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import tensorflow.compat.v1 as tf
tf.reset_default_graph()
with tf.Session() as sess:
x = tf.placeholder(tf.float32, None, 'x')
filter = tf.placeholder(tf.float32, [2, 2, 3, 1], 'kernel')
conv2d = tf.raw_ops.Conv2D(input=x, filter=filter, strides=[1, 1, 1, 1], padding='SAME',
dilations=None)
relu = tf.raw_ops.Relu(features=conv2d)
tf.global_variables_initializer()
tf_net = sess.graph_def
tf.io.write_graph(tf_net, './', 'model_with_convolution_dynamic_rank.pbtxt', True)