IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[MO] [Kaldi] Add TDNN Component (#1870)

Browse Source 
* [MO] [Kaldi] Added TDNN Component

* TdnnComponent replacer graphical comment updated

* Added SpecAugmentTimeMaskComponent

* some refactor of memoryoffset shape_infer

* moved memoryoffset splitting to the middle stage

* some corrections
- set `need_shape_inferenc`=False in split_memoryoffset
- use cycle instead of pattern in tdnn_replacer

* separated splitting of MemoryOffsets in LSTM and TDNN blocks

* set transpose_weights=True in TdnnComponent

* Corrected Supported_Frameworks_Layers

* corrected comments

* separate naming for tdnn and lstm memoryoffset splits

* corrected BOM file

* corrected generaldropout_ext.py and removed 'has_default' for tdnn_component

* corrections after PR review

* renamed LSTM -> recurrent; added setting element_size for paired nodes of tdnn_memoffset and othe minor changes

* Update split_tdnn_memoryoffset.py

* corrected partial infer with new API in elemental.py and split_tdnn_memoryoffset.py
This commit is contained in:
Pavel Esir
2020-10-12 14:10:27 +03:00
committed by GitHub
parent 9f1b4e0854
commit 2110a29b7c
19 changed files with 592 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
docs/MO_DG/prepare_model/Supported_Frameworks_Layers.md
View File

@@ -264,6 +264,9 @@ Standard Kaldi\* Layers:
| Crop | No |
| elementwiseproductcomponent | No |
| fixedaffinecomponent | No |
| fixedbiascomponent | No |
| fixedscalecomponent | No |
| generaldropoutcomponent| Not needed for inference |
| linearcomponent | No |
| logsoftmaxcomponent | No |
| lstmnonlinearitycomponent | No |
@@ -279,12 +282,13 @@ Standard Kaldi\* Layers:
| rectifiedlinearcomponent | No |
| rescale | No |
| sigmoid | No |
| slice | No |
| softmax | No |
| softmaxComponent | No |
| softsign | No |
| specaugmenttimemaskcomponent | Not needed for inference |
| splicecomponent | No |
| tanhcomponent | No |
| tdnncomponent | No |
## ONNX\* Supported Operators

10
model-optimizer/automation/package_BOM.txt
View File

@@ -145,8 +145,9 @@ extensions/front/kaldi/replace_lstm_node_pattern.py
extensions/front/kaldi/replace_lstm_nonlinearity.py
extensions/front/kaldi/set_ports.py
extensions/front/kaldi/sigmoid_ext.py
extensions/front/kaldi/split_memoryoffsets.py
extensions/front/kaldi/split_recurrent_memoryoffset.py
extensions/front/kaldi/tanh_component_ext.py
extensions/front/kaldi/tdnn_component_replacer.py
extensions/front/LayerNorm.py
extensions/front/Log1p.py
extensions/front/LogSoftmax.py
@@ -567,6 +568,7 @@ extensions/middle/SharedWeightsDuplication.py
extensions/middle/SliceConverter.py
extensions/middle/SliceLikeToStridedSlice.py
extensions/middle/sparse_reshape.py
extensions/middle/split_tdnn_memoryoffset.py
extensions/middle/SplitConcatPairToInterpolate.py
extensions/middle/SwapAxesMiddleReplacer.py
extensions/middle/TensorIterator_utils.py
@@ -781,17 +783,20 @@ mo/front/kaldi/extractors/batchnorm_component_ext.py
mo/front/kaldi/extractors/bias_component_ext.py
mo/front/kaldi/extractors/clip_ext.py
mo/front/kaldi/extractors/concat_ext.py
mo/front/kaldi/extractors/const_ext.py
mo/front/kaldi/extractors/convolutional_1d_component_ext.py
mo/front/kaldi/extractors/convolutional_component_ext.py
mo/front/kaldi/extractors/copy_ext.py
mo/front/kaldi/extractors/crop_ext.py
mo/front/kaldi/extractors/elementwise_component_ext.py
mo/front/kaldi/extractors/fixed_affine_component_ext.py
mo/front/kaldi/extractors/generaldropout_ext.py
mo/front/kaldi/extractors/linear_component_ext.py
mo/front/kaldi/extractors/lstm_nonlinearity_ext.py
mo/front/kaldi/extractors/lstm_projected_streams_ext.py
mo/front/kaldi/extractors/max_pooling_ext.py
mo/front/kaldi/extractors/memoryoffset_ext.py
mo/front/kaldi/extractors/mul_ext.py
mo/front/kaldi/extractors/naturalgradient_affine_component_ext.py
mo/front/kaldi/extractors/noop_ext.py
mo/front/kaldi/extractors/normalize_component_ext.py
@@ -800,7 +805,9 @@ mo/front/kaldi/extractors/rectified_linear_component_ext.py
mo/front/kaldi/extractors/rescale_ext.py
mo/front/kaldi/extractors/scale_component_ext.py
mo/front/kaldi/extractors/softmax_ext.py
mo/front/kaldi/extractors/specaugment_component_ext.py
mo/front/kaldi/extractors/splice_component_ext.py
mo/front/kaldi/extractors/tdnncomponent_ext.py
mo/front/kaldi/loader/__init__.py
mo/front/kaldi/loader/loader.py
mo/front/kaldi/loader/utils.py
@@ -915,6 +922,7 @@ mo/ops/softmax.py
mo/ops/space_to_batch.py
mo/ops/squeeze.py
mo/ops/strided_slice.py
mo/ops/tdnncomponent.py
mo/ops/tile.py
mo/ops/unsqueeze.py
mo/pipeline/__init__.py

4
model-optimizer/extensions/front/kaldi/memory_offset_adjustment.py
View File

@@ -100,8 +100,8 @@ class MemoryOffsetAdjustment(FrontReplacementSubgraph):
graph_condition = [lambda graph: graph.graph['fw'] == 'kaldi']
def run_before(self):
from extensions.front.kaldi.split_memoryoffsets import SplitMemoryOffsets
return [SplitMemoryOffsets]
from extensions.front.kaldi.split_recurrent_memoryoffset import SplitRecurrentMemoryOffset
return [SplitRecurrentMemoryOffset]
def find_and_replace_pattern(self, graph: Graph):
should_continue = False

49
model-optimizer/extensions/front/kaldi/split_memoryoffsets.py
View File

@@ -1,49 +0,0 @@
"""
Copyright (C) 2018-2020 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from mo.front.common.replacement import FrontReplacementPattern
from mo.graph.graph import Graph
from mo.ops.memoryoffset import MemoryOffset
from mo.ops.result import Result
class SplitMemoryOffsets(FrontReplacementPattern):
'''
Split MemoryOffsets in 2 parts to cut cycles
'''
enabled = True
run_not_recursively = True
def run_after(self):
from extensions.front.restore_ports import RestorePorts
return [RestorePorts]
def pattern(self):
return dict(
nodes=[
('mem_offset', dict(kind='op', op='MemoryOffset', splitted=False))
],
edges=[]
)
@staticmethod
def replace_pattern(graph: Graph, match: dict):
offset_node = match['mem_offset']
paired_node = MemoryOffset(graph, {'name': offset_node.pair_name, 'splitted': True, 'pair_name': offset_node.id,
't': offset_node.t, 'has_default': offset_node.has_default}).create_node()
offset_node['splitted'] = True
offset_node.out_port(0).get_connection().set_source(paired_node.out_port(0))
res_node = Result(graph, {'name': offset_node.id+"_output"}).create_node()
offset_node.out_port(0).connect(res_node.in_port(0))

62
model-optimizer/extensions/front/kaldi/split_recurrent_memoryoffset.py Normal file
View File

@@ -0,0 +1,62 @@
"""
Copyright (C) 2018-2020 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import networkx as nx
from mo.front.common.replacement import FrontReplacementSubgraph
from mo.graph.graph import Graph
from mo.ops.memoryoffset import MemoryOffset
from mo.ops.result import Result
from mo.utils.error import Error
from mo.utils.graph import Node
class SplitRecurrentMemoryOffset(FrontReplacementSubgraph):
"""
Splits MemoryOffsets in recurrent blocks (typically LSTM blocks) into 2 parts.
These parts then will be converted to ReadValue and Assign. Splitting complicates shape inference but
MemoryOffsets in recurrent blocks are cycled and, in order to make topological sort possible
during shape inference, they are splitted earlier on the front phase. In contrast,
MemoryOffsets in TDNN blocks are not cycled, so they will be splitted after shape infer on the middle.
Now only LSTM blocks with MemoryOffset are present.
"""
enabled = True
graph_condition = [lambda graph: graph.graph['fw'] == 'kaldi']
@staticmethod
def split_offset(offset_node: Node):
paired_node = MemoryOffset(offset_node.graph, {'name': offset_node.pair_name, 'splitted': True,
'pair_name': offset_node.id,
'element_size': offset_node['element_size'],
't': offset_node.t,
'has_default': offset_node.has_default}).create_node()
offset_node['splitted'] = True
offset_node.out_port(0).get_connection().set_source(paired_node.out_port(0))
res_node = Result(offset_node.graph, {'name': offset_node.id + '_output'}).create_node()
offset_node.out_port(0).connect(res_node.in_port(0))
def find_and_replace_pattern(self, graph: Graph):
for offset_node in graph.get_op_nodes(op='MemoryOffset', splitted=False):
try:
# if graph contains recurrent block -> split MemoryOffset to enable shape infer
nx.find_cycle(graph, offset_node.id)
except nx.NetworkXNoCycle as e:
# MemoryOffset node is not in a recurrent block -- no splitting is needed
return
if not offset_node.has_valid('element_size'):
raise Error("In a recurrent block 'element_size' for node {} is not set".format(offset_node.id))
SplitRecurrentMemoryOffset.split_offset(offset_node)

95
model-optimizer/extensions/front/kaldi/tdnn_component_replacer.py Normal file
View File

@@ -0,0 +1,95 @@
"""
Copyright (C) 2018-2020 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from extensions.ops.MatMul import FullyConnected
from mo.front.common.replacement import FrontReplacementPattern
from mo.front.tf.graph_utils import create_op_with_const_inputs
from mo.graph.graph import Graph, Node
from mo.graph.graph import rename_nodes
from mo.ops.concat import Concat
from mo.ops.memoryoffset import MemoryOffset
class TdnnComponentReplacer(FrontReplacementPattern):
'''
Expand TdnnComponent into MemoryOffsets, Concat and FullyConected nodes
BEFORE:
placeholder
|
TdnnComponent('time_offsets': t1, t2,... tk)
|
_______________________________________________________________
AFTER:
placeholder
__________________|___________________________
/ | \ \
MemoryOffset(t1) MemoryOffset(t2) ... MemoryOffset(tk)
\_____________ _____|______________/____________/
Concat
|
FullyConnected
|
'''
enabled = True
run_not_recursively = True
def run_before(self):
from extensions.front.kaldi.memory_offset_adjustment import MemoryOffsetAdjustment
return [MemoryOffsetAdjustment]
def find_and_replace_pattern(self, graph: Graph):
for node in graph.get_op_nodes(op='tdnncomponent'):
self.replace_tdnn(graph, node)
def replace_tdnn(self, graph: Graph, tdnn_node: Node):
tdnn_name = tdnn_node.soft_get('name', tdnn_node.id)
concat_node = Concat(graph, {'axis': 1}).create_node()
rename_nodes([(tdnn_node, tdnn_name + '/to_be_removed'), (concat_node, tdnn_name)])
for offset_ind, t in enumerate(tdnn_node['time_offsets']):
concat_node.add_input_port(offset_ind)
if t != 0:
memory_name = tdnn_name + '/MemoryOffset/' + str(abs(t))
memoryoffset_node = MemoryOffset(graph, {'name': memory_name, 't': t,
'pair_name': memory_name + '_out',
'has_default': False, 'splitted': False}).create_node()
tdnn_node.in_port(0).get_source().connect(memoryoffset_node.in_port(0))
memoryoffset_node.out_port(0).connect(concat_node.in_port(offset_ind))
else:
# 0 time delay is not allowed in IE, it's meaningless
# if time offset is 0 then connect input of tdnncomponent directly to Concat without memoryoffset
tdnn_node.in_port(0).get_source().connect(concat_node.in_port(offset_ind))
weights = tdnn_node['weights']
fc_inputs = {1: weights}
bias_term = False
if tdnn_node.has_valid('biases'):
assert len(tdnn_node['biases']) == weights.shape[0]
fc_inputs.update({2: tdnn_node['biases']})
bias_term = True
fc_node = create_op_with_const_inputs(graph, FullyConnected, fc_inputs,
{'name': tdnn_name + '/FC', 'out-size': weights.shape[0],
'transpose_weights': True, 'bias_term': bias_term})
concat_node.out_port(0).connect(fc_node.in_port(0))
tdnn_node.in_port(0).disconnect()
tdnn_node.out_port(0).get_connection().set_source(fc_node.out_port(0))

87
model-optimizer/extensions/front/kaldi/tdnn_component_replacer_test.py Normal file
View File

@@ -0,0 +1,87 @@
"""
Copyright (C) 2018-2020 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import unittest
import numpy as np
from generator import generator, generate
from extensions.front.kaldi.tdnn_component_replacer import TdnnComponentReplacer
from mo.utils.ir_engine.compare_graphs import compare_graphs
from mo.utils.unittest.graph import build_graph, regular_op, result, connect_front, const
@generator
class TdnnComponentReplacerTest(unittest.TestCase):
@generate(*[
([[1, 1, 1], [4, 4, 4]], [1, 2], [-1, 1],),
([[1, 1, 1], [4, 4, 4]], [1, 2], [-1, 1, 2, 10, 1000],),
([[1, 1, 1], [4, 4, 4]], [1, 2], [-1, 0]),
])
def test_tdnnreplacer(self, weights, biases, time_offsets):
def generate_offsets():
offset_edges = []
offset_nodes = {}
for i, t in enumerate(time_offsets):
offset_nodes.update(**regular_op('memoryoffset_' + str(i), {'type': None}))
if t != 0:
offset_edges.append(('placeholder', 'memoryoffset_' + str(i), {'out': 0, 'in': 0}))
offset_edges.append(('memoryoffset_' + str(i), 'concat', {'out': 0, 'in': i}))
else:
offset_edges.append(('placeholder', 'concat', {'out': 0, 'in': i}))
return offset_nodes, offset_edges
offset_nodes, ref_offset_edges = generate_offsets()
nodes = {
**offset_nodes,
**regular_op('placeholder', {'type': 'Parameter'}),
**regular_op('tdnncomponent', {'op': 'tdnncomponent',
'weights': np.array(weights),
'biases': np.array(biases),
'time_offsets': np.array(time_offsets)}),
**const('weights', np.array(weights)),
**const('biases', np.array(biases)),
**regular_op('concat', {'type': 'Concat', 'axis': 1}),
**regular_op('memoryoffset_0', {'type': None}),
**regular_op('memoryoffset_1', {'type': None}),
**regular_op('memoryoffset_2', {'type': None}),
**regular_op('fully_connected', {'type': 'FullyConnected'}),
**result('result'),
}
graph = build_graph(nodes, [
*connect_front('placeholder', 'tdnncomponent'),
*connect_front('tdnncomponent', 'result')
], nodes_with_edges_only=True)
graph.stage = 'front'
ref_graph = build_graph(nodes, [
*ref_offset_edges,
*connect_front('concat', '0:fully_connected'),
*connect_front('weights', '1:fully_connected'),
*connect_front('biases', '2:fully_connected'),
*connect_front('fully_connected', 'result')
], nodes_with_edges_only=True)
TdnnComponentReplacer().find_and_replace_pattern(graph)
(flag, resp) = compare_graphs(graph, ref_graph, 'result', check_op_attrs=True)
self.assertTrue(flag, resp)

48
model-optimizer/extensions/middle/split_tdnn_memoryoffset.py Normal file
View File

@@ -0,0 +1,48 @@
"""
Copyright (C) 2018-2020 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from mo.graph.graph import Graph
from mo.middle.replacement import MiddleReplacementPattern
from mo.ops.memoryoffset import MemoryOffset
from mo.ops.result import Result
class SplitTdnnMemoryOffset(MiddleReplacementPattern):
'''
Splits MemoryOffsets in TDNN blocks into 2 parts. These parts then will be converted to ReadValue and Assign.
'''
enabled = True
run_not_recursively = True
def run_before(self):
from extensions.middle.ReplaceMemoryOffsetWithSplice import ReplaceMemoryOffsetWithMemoryNodePattern, ReplaceMemoryOffsetNodePattern
return [ReplaceMemoryOffsetNodePattern, ReplaceMemoryOffsetWithMemoryNodePattern]
def find_and_replace_pattern(self, graph: Graph):
for offset_node in graph.get_op_nodes(op='MemoryOffset', splitted=False):
paired_node = MemoryOffset(graph, {'name': offset_node.pair_name, 'splitted': True, 'pair_name': offset_node.id,
't': offset_node.t, 'has_default': offset_node.has_default}).create_node()
offset_node['splitted'] = True
offset_node.out_port(0).get_connection().set_source(paired_node.out_port(0))
res_node = Result(graph, {'name': offset_node.id + "_output"}).create_node()
offset_node.out_port(0).connect(res_node.in_port(0))
# If 'element_size' is previously copied from Parameter of from node with defined dim
if offset_node.has_valid('element_size'):
paired_node['element_size'] = offset_node['element_size']
# Copy shape from previous node. Typically (but not always) for TDNN blocks this is the case
else:
paired_node['element_size'] = offset_node.in_port(0).data.get_shape()[1]

4
model-optimizer/mo/front/common/partial_infer/elemental.py
View File

@@ -30,8 +30,8 @@ def copy_shape_infer(node, value_infer=None):
Args:
node: graph node
"""
single_output_infer(node, lambda n: n.in_node().shape, value_infer)
single_output_infer(node, lambda n: n.in_port(0).data.get_shape(), value_infer)
def copy_value(node):
return None if node.in_node().value is None else node.in_node().value.copy()
return None if node.in_node().value is None else node.in_port(0).data.get_value()

32
model-optimizer/mo/front/kaldi/extractors/const_ext.py Normal file
View File

@@ -0,0 +1,32 @@
"""
Copyright (C) 2018-2020 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from mo.front.extractor import FrontExtractorOp
from mo.ops.const import Const
class ConstantExtractor(FrontExtractorOp):
op = 'Const'
enabled = True
@classmethod
def extract(cls, node):
attrs = {
'data_type': node.value.dtype,
'value': node.value,
}
Const.update_node_stat(node, attrs)
return cls.enabled

46
model-optimizer/mo/front/kaldi/extractors/generaldropout_ext.py Normal file
View File

@@ -0,0 +1,46 @@
"""
Copyright (C) 2018-2020 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from mo.front.extractor import FrontExtractorOp
from mo.front.kaldi.loader.utils import read_binary_integer32_token, collect_until_token, \
read_binary_float_token
from extensions.ops.identity import Identity
class GeneralDropoutComponentFrontExtractor(FrontExtractorOp):
op = 'generaldropoutcomponent'
enabled = True
@classmethod
def extract(cls, node):
pb = node.parameters
collect_until_token(pb, b'<Dim>')
dim = read_binary_integer32_token(pb)
collect_until_token(pb, b'<BlockDim>')
block_dim = read_binary_integer32_token(pb)
collect_until_token(pb, b'<TimePeriod>')
time_period = read_binary_integer32_token(pb)
collect_until_token(pb, b'<DropoutProportion>')
dropout_proporion = read_binary_float_token(pb)
# collect_until_token(pb, b'<Continuous>')
Identity.update_node_stat(node, {})
return cls.enabled

28
model-optimizer/mo/front/kaldi/extractors/mul_ext.py Normal file
View File

@@ -0,0 +1,28 @@
"""
Copyright (C) 2018-2020 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from extensions.ops.elementwise import Mul
from mo.front.extractor import FrontExtractorOp
class MulFrontExtractor(FrontExtractorOp):
op = 'Mul'
enabled = True
@classmethod
def extract(cls, node):
Mul.update_node_stat(node, {})
return cls.enabled

28
model-optimizer/mo/front/kaldi/extractors/specaugment_component_ext.py Normal file
View File

@@ -0,0 +1,28 @@
"""
Copyright (C) 2018-2020 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from extensions.ops.identity import Identity
from mo.front.extractor import FrontExtractorOp
class SpecAugmentComponentFrontExtractor(FrontExtractorOp):
op = 'specaugmenttimemaskcomponent'
enabled = True
@classmethod
def extract(cls, node):
Identity.update_node_stat(node, {})
return cls.enabled

71
model-optimizer/mo/front/kaldi/extractors/tdnncomponent_ext.py Normal file
View File

@@ -0,0 +1,71 @@
"""
Copyright (C) 2018-2020 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import numpy as np
from mo.front.extractor import FrontExtractorOp
from mo.front.kaldi.loader.utils import read_binary_bool_token, read_binary_integer32_token, collect_until_token, \
read_binary_float_token
from mo.front.kaldi.utils import read_binary_vector, read_binary_matrix
from mo.ops.tdnncomponent import TdnnComponent
class TdnnComponentFrontExtractor(FrontExtractorOp):
op = 'tdnncomponent'
enabled = True
@classmethod
def extract(cls, node):
pb = node.parameters
collect_until_token(pb, b'<MaxChange>')
max_change = read_binary_float_token(pb)
collect_until_token(pb, b'<L2Regularize>')
collect_until_token(pb, b'<LearningRate>')
collect_until_token(pb, b'<TimeOffsets>')
time_offsets = read_binary_vector(pb, False, np.int32)
collect_until_token(pb, b'<LinearParams>')
weights, weights_shape = read_binary_matrix(pb)
collect_until_token(pb, b'<BiasParams>')
bias_params = read_binary_vector(pb)
collect_until_token(pb, b'<OrthonormalConstraint>')
orthonormal_constraint = read_binary_float_token(pb) # used only on training
collect_until_token(pb, b'<UseNaturalGradient>')
use_natural_grad = read_binary_bool_token(pb) # used only on training
collect_until_token(pb, b'<NumSamplesHistory>')
num_samples_hist = read_binary_float_token(pb)
collect_until_token(pb, b'<AlphaInOut>')
alpha_in_out = read_binary_float_token(pb), read_binary_float_token(pb) # for training, usually (4, 4)
# according to Kaldi documentation http://kaldi-asr.org/doc/classkaldi_1_1nnet3_1_1TdnnComponent.html#details
# it looks like it's used only during training (but not 100% sure)
collect_until_token(pb, b'<RankInOut>')
rank_in_out = read_binary_integer32_token(pb), read_binary_integer32_token(pb)
biases = np.array(bias_params) if len(bias_params) != 0 else None
attrs = {
'weights': np.reshape(weights, weights_shape),
'biases': biases,
'time_offsets': time_offsets,
}
TdnnComponent.update_node_stat(node, attrs)
return cls.enabled

41
model-optimizer/mo/front/kaldi/loader/loader.py
View File

@@ -13,17 +13,21 @@
See the License for the specific language governing permissions and
limitations under the License.
"""
import logging as log
from io import IOBase
import networkx as nx
import numpy as np
from extensions.ops.elementwise import Mul
from extensions.ops.split import AttributedVariadicSplit
from mo.front.common.partial_infer.utils import float_array
from mo.front.kaldi.loader.utils import find_next_tag, read_placeholder, find_next_component, get_name_from_path, \
find_end_of_component, end_of_nnet_tag, read_binary_integer32_token, get_parameters, read_token_value, \
collect_until_token, collect_until_token_and_read, create_edge_attrs, get_args_for_specifier
from mo.graph.graph import Node, Graph
from mo.ops.const import Const
from mo.utils.error import Error
from mo.utils.utils import refer_to_faq_msg
@@ -74,7 +78,7 @@ def load_parallel_component(file_descr, graph: Graph, prev_layer_id):
split_id = graph.unique_id(prefix='NestedNets/VariadicSplit')
attrs = {'out_ports_count': nnet_count, 'size_splits': split_points, 'axis': 1, 'name': split_id}
variadic_split_node = AttributedVariadicSplit(graph, attrs).create_node()
prev_layer_node = Node(graph, prev_layer_id)
prev_layer_node = Node(graph, prev_layer_id)
prev_layer_node.add_output_port(0)
graph.create_edge(prev_layer_node, variadic_split_node, 0, 0)
@@ -247,9 +251,7 @@ def load_components(file_descr, graph, component_layer_map=None):
node = Node(graph, layer)
node['parameters'] = get_parameters(file_descr, start_index, end_index)
node['op'] = component_type
# read dim info where possible to simplify shape calculation for MemoryOffset
# shape calculation for MemoryOffset can't be done through shape of previous layer because
# it is separated in 2 parts to remove cycle from graph
# Read dim info where possible to simplify shape calculation for MemoryOffset
for o_n_name, params in node.get_outputs():
o_n = Node(graph, o_n_name)
if o_n['op'] == 'MemoryOffset' and dim != 0:
@@ -459,7 +461,7 @@ def parse_specifier(string, graph, layer_node_map):
out_port = len(Node(graph, node).out_nodes())
in_port = len(Node(graph, memory_name).in_nodes())
Node(graph, memory_name).add_input_port(in_port)
Node(graph, node).add_output_port(out_port)
Node(graph, node).add_output_port(out_port, skip_if_exist=True)
graph.create_edge(Node(graph, node), Node(graph, memory_name), out_port, in_port)
else:
memory_name = layer_node_map[layer_name]
@@ -480,13 +482,12 @@ def parse_specifier(string, graph, layer_node_map):
else:
sum_name = layer_node_map[layer_name]
i = 0
for node in nodes:
for i, node in enumerate(nodes):
out_port = len(Node(graph, node).out_nodes())
Node(graph, node).add_output_port(out_port)
Node(graph, node).add_output_port(out_port, skip_if_exist=True)
Node(graph, sum_name).add_input_port(i)
graph.add_edge(node, sum_name, **create_edge_attrs(node, sum_name, i))
i = i + 1
return sum_name
elif spec == b'IfDefined':
node_id = parse_specifier(args[0], graph, layer_node_map)
@@ -497,3 +498,25 @@ def parse_specifier(string, graph, layer_node_map):
elif spec == b'ReplaceIndex':
node = parse_specifier(args[0], graph, layer_node_map)
return node
elif spec == b'Scale':
node_name = parse_specifier(args[1], graph, layer_node_map)
scale_value = float(args[0])
layer_name = '{}/Mul/{}'.format(node_name, scale_value)
if layer_name not in layer_node_map:
scale_name = graph.unique_id(prefix=layer_name)
scale_node = Mul(graph, {'name': scale_name}).create_node()
layer_node_map[layer_name] = scale_name
scale_const_name = 'Const_{}'.format(scale_value)
const_node = Const(graph, {'name': scale_const_name, 'value': float_array([scale_value])}).create_node()
node = Node(graph, node_name)
graph.create_edge(const_node, scale_node, 0, 0)
out_port = len(node.out_nodes())
graph.create_edge(node, scale_node, out_port, 1)
else:
scale_name = layer_node_map[layer_name]
return scale_name

4
model-optimizer/mo/front/kaldi/loader/utils.py
View File

@@ -13,6 +13,7 @@
See the License for the specific language governing permissions and
limitations under the License.
"""
import io
import os
import struct
@@ -40,6 +41,7 @@ supported_components = [
'fixedaffinecomponent',
'fixedscalecomponent',
'fixedbiascomponent',
'generaldropoutcomponent',
'linearcomponent',
'logsoftmaxcomponent',
'lstmnonlinearitycomponent',
@@ -58,9 +60,11 @@ supported_components = [
'sigmoidcomponent',
'softmax',
'softmaxcomponent',
'specaugmenttimemaskcomponent',
'splicecomponent',
'sumgroupcomponent',
'tanhcomponent',
'tdnncomponent',
]

2
model-optimizer/mo/middle/passes/infer.py
View File

@@ -108,7 +108,7 @@ def partial_infer(graph: Graph, start_node: str = None):
graph.strict_mode = True
# Mark all nodes as not inferred yet
if not start_node is None:
if start_node is not None:
start_index = nodes.index(start_node)
nx.set_node_attributes(G=graph.subgraph(nodes[start_index:]), name='is_partial_inferred', values=False)
else:

40
model-optimizer/mo/ops/memoryoffset.py
View File

@@ -13,58 +13,34 @@
See the License for the specific language governing permissions and
limitations under the License.
"""
import numpy as np
from mo.front.common.partial_infer.elemental import copy_shape_infer
from mo.graph.graph import Graph, Node
from mo.ops.op import Op
from mo.utils.error import Error
class MemoryOffset(Op):
op = 'MemoryOffset'
enabled = True
enabled = False
def __init__(self, graph: Graph, attrs: dict):
super().__init__(graph, {
'op': 'MemoryOffset',
'type': None,
'pair_name': None,
'splitted': False,
'has_default': False,
'infer': __class__.infer,
'in_ports_count': 1,
'out_ports_count': 1,
}, attrs)
def supported_attrs(self):
return ['t']
@staticmethod
def infer(node: Node):
# MemoryOffset is split into 2 parts to avoid cycle in graph
# Calculate shape from shape of previous layer where possible
# In other cases information about shapes from initial Kaldi model used
if not node.in_port(0).disconnected():
copy_shape_infer(node)
pair_node = Node(node.graph, node.pair_name)
pair_node.out_port(0).data.set_shape(node.out_port(0).data.get_shape())
if node.has_valid('element_size'):
# element_size should be set by Kaldi loader or by MemoryOffsetAdjustment
node.out_port(0).data.set_shape([1, node['element_size']])
else:
pair_node = Node(node.graph, node.pair_name)
if pair_node.in_port(0).data.get_shape() is not None:
node.out_port(0).data.set_shape(pair_node.in_port(0).data.get_shape())
copy_shape_infer(pair_node)
elif pair_node.has_valid('element_size'):
# TODO Add here real batch
node.out_port(0).data.set_shape(np.array([1, pair_node['element_size']]))
elif pair_node.in_port(0).get_source().node.has_valid('out-size'):
out_size = pair_node.in_port(0).get_source().node['out-size']
node.out_port(0).data.set_shape(np.array([1, out_size]))
elif pair_node.in_port(0).get_source().node.op in ["Add", "ReLU"] and \
pair_node.in_port(0).get_source().node.in_port(0).get_source().node.has_valid('out-size'):
out_size = pair_node.in_port(0).get_source().node.in_port(0).get_source().node['out-size']
node.out_port(0).data.set_shape(np.array([1, out_size]))
elif pair_node.in_port(0).get_source().node.has_valid('in_dim'):
out_size = pair_node.in_port(0).get_source().node['in_dim']
node.out_port(0).data.set_shape(np.array([1, out_size]))
else:
raise Error("Can't calculate MemoryOffset shape for node {}. ".format(node.id) +
"Possibly you need to add shape for it through --input_shape")
# for TDNN blocks
copy_shape_infer(node)

32
model-optimizer/mo/ops/tdnncomponent.py Normal file
View File

@@ -0,0 +1,32 @@
"""
Copyright (C) 2018-2020 Intel Corporation
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from mo.graph.graph import Graph
from mo.ops.op import Op
class TdnnComponent(Op):
op = 'tdnncomponent'
enabled = False
def __init__(self, graph: Graph, attrs: dict):
super().__init__(graph, {
'type': None,
'op': self.op,
'infer': None,
'in_ports_count': 1,
'out_ports_count': 1,
}, attrs)