From cd6ffd76200df40d74529f0cadd75a270343ac6c Mon Sep 17 00:00:00 2001 From: Anastasiia Pnevskaia Date: Mon, 17 Jul 2023 13:41:33 +0200 Subject: [PATCH] Removed redundant copy of Tensor in TF Const translator (#18299) * Removed copy of consts in translator, added test. * Fixed memory loss for tf.Const. * Added test, minor corrections. * Update src/bindings/python/src/openvino/frontend/tensorflow/node_decoder.py Co-authored-by: Roman Kazantsev * Test corrections. * Added comment. --------- Co-authored-by: Roman Kazantsev --- .../frontend/tensorflow/node_decoder.py | 54 ++++++--- .../tensorflow_common/src/op/const.cpp | 2 +- .../mo_python_api_tests/test_mo_convert_tf.py | 105 ++++++++++++++++++ 3 files changed, 144 insertions(+), 17 deletions(-) diff --git a/src/bindings/python/src/openvino/frontend/tensorflow/node_decoder.py b/src/bindings/python/src/openvino/frontend/tensorflow/node_decoder.py index cc0ff44ea6e..ace64a2df11 100644 --- a/src/bindings/python/src/openvino/frontend/tensorflow/node_decoder.py +++ b/src/bindings/python/src/openvino/frontend/tensorflow/node_decoder.py @@ -54,20 +54,42 @@ class TFGraphNodeDecoder(DecoderBase): self.m_operation = operation self.m_inner_graph = inner_graph self.m_data_type = None + + # Copies value from inner buffer of TF_Operation to NodeDef class. + self.m_node_def = self.m_operation.node_def + + # TODO: Create parameter in convert_model() for turning on/off shared memory, ticket: 114971 + self.m_shared_memory = True + if self.m_operation.type == "Const": - data_type = tf.dtypes.DType(self.m_operation.node_def.attr["dtype"].type).name - self.m_data_type = data_type - # copies tensor value from node_def - value = self.m_operation.node_def.attr["value"].tensor - if data_type == "string": + self.m_data_type = tf.dtypes.DType(self.m_node_def.attr["dtype"].type).name + + # Copies tensor value to parsed TensorProto + value = self.m_node_def.attr["value"].tensor + + # As the tensor was copied, shared memory may be lost + # after destruction of GraphIteratorTFGraph() when convert_model() finishes its work. + # To prevent it we need to turn off sharing. + self.m_shared_memory = False + + if self.m_data_type == "string": self.m_parsed_content = [str(val) for val in value.string_val] else: - self.m_parsed_content = tf.make_ndarray(value) + if value.tensor_content: + shape = [d.size for d in value.tensor_shape.dim] + tensor_dtype = tf.dtypes.as_dtype(value.dtype) + dtype = tensor_dtype.as_numpy_dtype + # no copy of content + self.m_parsed_content = (np.frombuffer(value.tensor_content, + dtype=dtype).reshape(shape)) + else: + # TODO: remove copy of content for cases when tensor value is not in tensor_content field, ticket: 114797 + self.m_parsed_content = tf.make_ndarray(value) if self.m_operation.type == "Placeholder": - data_type = tf.dtypes.DType(self.m_operation.node_def.attr["dtype"].type).name - self.m_data_type = data_type - if data_type == "resource" and not self.m_inner_graph: + self.m_data_type = tf.dtypes.DType(self.m_node_def.attr["dtype"].type).name + + if self.m_data_type == "resource" and not self.m_inner_graph: variable_value = TFGraphNodeDecoder.get_variable(self.m_operation) if variable_value is not None: # does not copy data @@ -82,7 +104,7 @@ class TFGraphNodeDecoder(DecoderBase): def get_op_type(self) -> str: if self.m_operation.type == "Placeholder": - type_attr = tf.dtypes.DType(self.m_operation.node_def.attr["dtype"].type) + type_attr = tf.dtypes.DType(self.m_node_def.attr["dtype"].type) if type_attr.name == "resource" and not self.m_inner_graph: if TFGraphNodeDecoder.get_variable(self.m_operation) is not None: return "Const" @@ -105,11 +127,11 @@ class TFGraphNodeDecoder(DecoderBase): def get_attribute(self, name): if name == "shape" or name == "_output_shapes": - if self.m_operation.node_def.attr["shape"].shape.unknown_rank: + if self.m_node_def.attr["shape"].shape.unknown_rank: return OVAny(PartialShape.dynamic()) - shape_dims = self.m_operation.node_def.attr["shape"].shape.dim + shape_dims = self.m_node_def.attr["shape"].shape.dim shape = [dim.size for dim in shape_dims] - type_num = self.m_operation.node_def.attr["dtype"].type + type_num = self.m_node_def.attr["dtype"].type if type_num is not None and tf.dtypes.DType(type_num).name == "resource": if self.m_inner_graph: return OVAny(PartialShape.dynamic()) @@ -117,7 +139,7 @@ class TFGraphNodeDecoder(DecoderBase): return OVAny(PartialShape(list(variable_value.shape))) return OVAny(PartialShape(shape)) if name == "dtype": - type_num = self.m_operation.node_def.attr["dtype"].type + type_num = self.m_node_def.attr["dtype"].type if tf.dtypes.DType(type_num).name == "resource": if not self.m_inner_graph: variable_value = TFGraphNodeDecoder.get_variable(self.m_operation) @@ -133,10 +155,10 @@ class TFGraphNodeDecoder(DecoderBase): if isinstance(self.m_parsed_content, np.ndarray): return OVAny(Tensor(self.m_parsed_content)) return OVAny(Tensor(np.array([self.m_parsed_content]), shape=[1])) - ov_tensor = Tensor(self.m_parsed_content, shared_memory=True) + ov_tensor = Tensor(self.m_parsed_content, shared_memory=self.m_shared_memory) ov_tensor = OVAny(ov_tensor) return ov_tensor - attr_value = self.m_operation.node_def.attr[name] + attr_value = self.m_node_def.attr[name] return tf_attr_to_ov(attr_value) diff --git a/src/frontends/tensorflow_common/src/op/const.cpp b/src/frontends/tensorflow_common/src/op/const.cpp index 956d2f1c0bc..1228d42b486 100644 --- a/src/frontends/tensorflow_common/src/op/const.cpp +++ b/src/frontends/tensorflow_common/src/op/const.cpp @@ -28,7 +28,7 @@ OutputVector translate_const_op(const NodeContext& node) { } } else { auto tensor = node.get_attribute("value"); - const_node = std::make_shared(tensor.get_element_type(), tensor.get_shape(), tensor.data()); + const_node = std::make_shared(tensor); } set_node_name(node.get_name(), const_node); return {const_node}; diff --git a/tests/layer_tests/mo_python_api_tests/test_mo_convert_tf.py b/tests/layer_tests/mo_python_api_tests/test_mo_convert_tf.py index b7747f066cf..1548a16c6c8 100644 --- a/tests/layer_tests/mo_python_api_tests/test_mo_convert_tf.py +++ b/tests/layer_tests/mo_python_api_tests/test_mo_convert_tf.py @@ -9,6 +9,7 @@ import pytest from openvino.runtime import PartialShape, Model, Dimension from common.mo_convert_test_class import CommonMOConvertTest +from common.layer_test_class import CommonLayerTest def create_tf_graph_def(tmp_dir): @@ -684,6 +685,110 @@ class TestMoConvertTF(CommonMOConvertTest): test_params = {'input_model': saved_model_dir, 'use_new_frontend': False} self._test_by_ref_graph(temp_dir, test_params, graph_ref, compare_tensor_names=False) + def test_zero_copy(self, ie_device, precision, ir_version, temp_dir): + import tensorflow as tf + from openvino.tools.mo import convert_model + from openvino.runtime import compile_model + class LayerModel(tf.Module): + def __init__(self): + super(LayerModel, self).__init__() + self.var1 = tf.Variable([7., 5., 6.], name='var1') + self.var2 = tf.Variable([5., 7., 3.], name='var2') + + + @tf.function + def sub_function(self, input): + return input * self.var1 + self.var2 + + @tf.function() + def __call__(self, input): + return self.sub_function(input) + + # Create TF model with variables + keras_model = LayerModel() + test_input = np.array(7.).astype(np.float32) + + # Convert model to OV + ov_model = convert_model(keras_model, input_shape=[1]) + cmp_model = compile_model(ov_model) + + # Check model inference + ov_infer1 = cmp_model(test_input, ie_device) + fw_infer1 = keras_model(test_input).numpy() + + assert np.array_equal(ov_infer1['Identity:0'], fw_infer1) + assert np.array_equal(ov_infer1['Identity:0'], [54., 42., 45.]) + + # Change value of variables in original model + for val in keras_model.variables: + arr = val.value().__array__() + arr[0] = 0 + arr[1] = 1 + arr[2] = 2 + + # Check model inference + ov_infer2 = cmp_model(test_input) + fw_infer2 = keras_model(test_input).numpy() + + assert np.array_equal(ov_infer2['Identity:0'], fw_infer2) + assert np.array_equal(ov_infer2['Identity:0'], [ 0., 8., 16.]) + + + + def test_memory_loss(self, ie_device, precision, ir_version, temp_dir): + # This test checks that the memory allocated for constants + # is not lost after returning the model from convert_model() method. + import tensorflow as tf + tf.compat.v1.reset_default_graph() + + from openvino.tools.mo import convert_model + from openvino.runtime import compile_model + import gc + + with tf.compat.v1.Session() as sess: + inp1 = tf.compat.v1.placeholder(tf.float32, [3], 'Input') + const = tf.constant([0.5, 2.3, 7.8], dtype=tf.float32) + res = inp1 + const + + tf.compat.v1.global_variables_initializer() + tf_graph = sess.graph # tf.Graph + + if precision == 'FP32': + eps = 1e-4 + else: + eps = 5e-2 + + + test_input = np.array([2.1, 7.3, 4.6]).astype(np.float32) + + # Convert model to OV + ov_model = convert_model(tf_graph) + cmp_model = compile_model(ov_model) + + # Check model inference + ov_infer1 = cmp_model(test_input, ie_device) + + feed_dict = {"Input:0": test_input} + with tf.compat.v1.Session(graph=tf_graph) as sess: + fw_infer1 = sess.run('add:0', feed_dict=feed_dict) + + assert CommonLayerTest().compare_ie_results_with_framework(ov_infer1, {"add:0": fw_infer1}, eps) + assert CommonLayerTest().compare_ie_results_with_framework(ov_infer1, {"add:0": [2.6, 9.6, 12.4]}, eps) + + # run Garbage collector to ensure, that values from tf.constant are copied to ov.Const and + # we do not lose allocated memory. + gc.collect() + + # Check model inference + ov_infer2 = cmp_model(test_input, ie_device) + + feed_dict = {"Input:0": test_input} + with tf.compat.v1.Session(graph=tf_graph) as sess: + fw_infer2 = sess.run('add:0', feed_dict=feed_dict) + + assert CommonLayerTest().compare_ie_results_with_framework(ov_infer2, {"add:0": fw_infer2}, eps) + assert CommonLayerTest().compare_ie_results_with_framework(ov_infer1, {"add:0": [2.6, 9.6, 12.4]}, eps) + class TFConvertTest(unittest.TestCase): @pytest.mark.nightly