From a0614c4cb83ef2db8f834eb3a8b40fb0412dbc47 Mon Sep 17 00:00:00 2001 From: Georgy Krivoruchko Date: Wed, 22 Nov 2023 03:34:10 -0800 Subject: [PATCH] [TFLite] Initial support of Densify operation * Added Densify --- .../tensorflow_common/src/op/identity.cpp | 4 +- .../src/decoder_flatbuffer.cpp | 1 + .../tensorflow_lite/src/op_table.cpp | 2 +- .../tensorflow_lite/src/sparsity_info.cpp | 163 ++++++++++++++++++ .../tensorflow_lite/src/sparsity_info.hpp | 111 ++++++++++++ .../tensorflow_lite/src/tensor_lite_place.cpp | 1 + .../tensorflow_lite/src/tensor_lite_place.hpp | 6 +- src/frontends/tensorflow_lite/src/utils.cpp | 31 ++++ src/frontends/tensorflow_lite/src/utils.hpp | 4 +- .../tensorflow_lite/tests/convert_model.cpp | 1 + .../gen_scripts/generate_pose_detector.py | 24 +++ 11 files changed, 344 insertions(+), 4 deletions(-) create mode 100644 src/frontends/tensorflow_lite/src/sparsity_info.cpp create mode 100644 src/frontends/tensorflow_lite/src/sparsity_info.hpp create mode 100644 src/frontends/tensorflow_lite/tests/test_models/gen_scripts/generate_pose_detector.py diff --git a/src/frontends/tensorflow_common/src/op/identity.cpp b/src/frontends/tensorflow_common/src/op/identity.cpp index add9c45243a..1d2f1d50cef 100644 --- a/src/frontends/tensorflow_common/src/op/identity.cpp +++ b/src/frontends/tensorflow_common/src/op/identity.cpp @@ -21,7 +21,9 @@ OutputVector translate_identity_op(const NodeContext& node) { "StopGradient", "ReadVariableOp", "ShardedFilename", - "MergeV2Checkpoints"}; + "MergeV2Checkpoints", + // TF Lite nodes + "DENSIFY"}; default_op_checks(node, 1, supported_ops); auto input = node.get_input(0); diff --git a/src/frontends/tensorflow_lite/src/decoder_flatbuffer.cpp b/src/frontends/tensorflow_lite/src/decoder_flatbuffer.cpp index b6f28224a6a..c3b866bc970 100644 --- a/src/frontends/tensorflow_lite/src/decoder_flatbuffer.cpp +++ b/src/frontends/tensorflow_lite/src/decoder_flatbuffer.cpp @@ -92,6 +92,7 @@ std::shared_ptr DecoderFlatBuffe ov::frontend::tensorflow_lite::get_ov_type(tensor->type()), names, ov::frontend::tensorflow_lite::get_quantization(tensor->quantization()), + ov::frontend::tensorflow_lite::get_sparsity(tensor->shape(), tensor->sparsity()), (tensor_info.buffer && tensor_info.buffer->data() ? tensor_info.buffer->data()->data() : nullptr)); } diff --git a/src/frontends/tensorflow_lite/src/op_table.cpp b/src/frontends/tensorflow_lite/src/op_table.cpp index 96738f53321..b2f87bc7d49 100644 --- a/src/frontends/tensorflow_lite/src/op_table.cpp +++ b/src/frontends/tensorflow_lite/src/op_table.cpp @@ -66,7 +66,7 @@ std::map get_supported_ops() { // CUMSUM // CUSTOM // DELEGATE - // DENSIFY + {"DENSIFY", translate_identity_op}, {"DEPTH_TO_SPACE", DEQUANTIZE_INPUTS(depth_to_space)}, {"DEPTHWISE_CONV_2D", DEQUANTIZE_INPUTS(depthwise_conv2d)}, {"DEQUANTIZE", DEQUANTIZE_INPUTS(dequantize)}, diff --git a/src/frontends/tensorflow_lite/src/sparsity_info.cpp b/src/frontends/tensorflow_lite/src/sparsity_info.cpp new file mode 100644 index 00000000000..90a5b783ee4 --- /dev/null +++ b/src/frontends/tensorflow_lite/src/sparsity_info.cpp @@ -0,0 +1,163 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "sparsity_info.hpp" + +#include "schema_generated.h" + +bool ov::frontend::tensorflow_lite::SparsityInfo::is_copyable() const { + return false; +} + +// Unpacks sparse data. Supports only case when sparse tensor has only one sparse dimension [DENSE, ..., DENSE, SPARSE] +template +static void read_sparse_data(uint8_t* dest, + uint8_t* dest_end, + const size_t row_size, + const ::flatbuffers::Vector* indices, + const ::flatbuffers::Vector* segments) { + uint8_t* data = dest - row_size; // row size will be increased at first step + T last_idx = ~static_cast(0); + for (auto idx = indices->begin(); idx != indices->end(); ++idx) { + if (*idx <= last_idx) { + data += row_size; + } + FRONT_END_GENERAL_CHECK(data + *idx < dest_end, "Dense data is out of bounds"); + static_cast(static_cast(data))[*idx] = segments->Get(*idx); + last_idx = *idx; + } +} + +void* ov::frontend::tensorflow_lite::SparsityInfo::densify() { + size_t sparse_idx = 0; + for (; sparse_idx < m_dim_format.size(); ++sparse_idx) { + if (m_dim_format[sparse_idx] == ::tflite::DimensionType_SPARSE_CSR) + break; + } + FRONT_END_GENERAL_CHECK(sparse_idx < m_dim_format.size(), "Sparse dimension isn't found for sparse tensor"); + FRONT_END_GENERAL_CHECK(sparse_idx == (m_dim_format.size() - 1), + "Supports only sparse tensor with sparse dimension as a last dimension"); + + size_t total_size = 0, // Size of data in bytes + row_size = 0; // Size of data row in bytes + switch (m_data_desc[sparse_idx].segments_type) { + case ::tflite::SparseIndexVector_Uint8Vector: + total_size = 1; + break; + case ::tflite::SparseIndexVector_Uint16Vector: + total_size = 2; + break; + case ::tflite::SparseIndexVector_Int32Vector: + total_size = 4; + break; + } + row_size = total_size; // Byte size is same + for (size_t dim = 0; dim < m_shape.size(); ++dim) { + total_size *= m_shape[dim]; + switch (m_dim_format[dim]) { + case ::tflite::DimensionType_DENSE: + break; + case ::tflite::DimensionType_SPARSE_CSR: + row_size *= m_shape[dim]; + break; + default: + FRONT_END_THROW("Unsupported dimension type found"); + break; + } + } + FRONT_END_GENERAL_CHECK(total_size > 0, "Wrong sparse segment size found"); + + m_data.resize(total_size); + memset(m_data.data(), 0, total_size); + switch (m_data_desc[sparse_idx].indices_type) { + case ::tflite::SparseIndexVector_Uint8Vector: + switch (m_data_desc[sparse_idx].segments_type) { + case ::tflite::SparseIndexVector_Uint8Vector: + read_sparse_data(m_data.data(), + m_data.data() + total_size, + row_size, + static_cast(m_data_desc[sparse_idx].indices)->values(), + static_cast(m_data_desc[sparse_idx].segments)->values()); + break; + case ::tflite::SparseIndexVector_Uint16Vector: + read_sparse_data(m_data.data(), + m_data.data() + total_size, + row_size, + static_cast(m_data_desc[sparse_idx].indices)->values(), + static_cast(m_data_desc[sparse_idx].segments)->values()); + break; + case ::tflite::SparseIndexVector_Int32Vector: + read_sparse_data(m_data.data(), + m_data.data() + total_size, + row_size, + static_cast(m_data_desc[sparse_idx].indices)->values(), + static_cast(m_data_desc[sparse_idx].segments)->values()); + break; + default: + FRONT_END_THROW("Unsupported vector type"); + break; + } + break; + case ::tflite::SparseIndexVector_Uint16Vector: + switch (m_data_desc[sparse_idx].segments_type) { + case ::tflite::SparseIndexVector_Uint8Vector: + read_sparse_data(m_data.data(), + m_data.data() + total_size, + row_size, + static_cast(m_data_desc[sparse_idx].indices)->values(), + static_cast(m_data_desc[sparse_idx].segments)->values()); + break; + case ::tflite::SparseIndexVector_Uint16Vector: + read_sparse_data(m_data.data(), + m_data.data() + total_size, + row_size, + static_cast(m_data_desc[sparse_idx].indices)->values(), + static_cast(m_data_desc[sparse_idx].segments)->values()); + break; + case ::tflite::SparseIndexVector_Int32Vector: + read_sparse_data(m_data.data(), + m_data.data() + total_size, + row_size, + static_cast(m_data_desc[sparse_idx].indices)->values(), + static_cast(m_data_desc[sparse_idx].segments)->values()); + break; + default: + FRONT_END_THROW("Unsupported vector type"); + break; + } + break; + case ::tflite::SparseIndexVector_Int32Vector: + switch (m_data_desc[sparse_idx].segments_type) { + case ::tflite::SparseIndexVector_Uint8Vector: + read_sparse_data(m_data.data(), + m_data.data() + total_size, + row_size, + static_cast(m_data_desc[sparse_idx].indices)->values(), + static_cast(m_data_desc[sparse_idx].segments)->values()); + break; + case ::tflite::SparseIndexVector_Uint16Vector: + read_sparse_data(m_data.data(), + m_data.data() + total_size, + row_size, + static_cast(m_data_desc[sparse_idx].indices)->values(), + static_cast(m_data_desc[sparse_idx].segments)->values()); + break; + case ::tflite::SparseIndexVector_Int32Vector: + read_sparse_data(m_data.data(), + m_data.data() + total_size, + row_size, + static_cast(m_data_desc[sparse_idx].indices)->values(), + static_cast(m_data_desc[sparse_idx].segments)->values()); + break; + default: + FRONT_END_THROW("Unsupported vector type"); + break; + } + break; + default: + FRONT_END_THROW("Unsupported vector type"); + break; + } + return m_data.data(); +} \ No newline at end of file diff --git a/src/frontends/tensorflow_lite/src/sparsity_info.hpp b/src/frontends/tensorflow_lite/src/sparsity_info.hpp new file mode 100644 index 00000000000..807abe6e4bd --- /dev/null +++ b/src/frontends/tensorflow_lite/src/sparsity_info.hpp @@ -0,0 +1,111 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include + +#include "common_op_table.hpp" +#include "decoder_map.hpp" +#include "openvino/frontend/tensorflow_lite/visibility.hpp" + +namespace ov { +namespace frontend { +namespace tensorflow_lite { + +class SparsityInfo : public ov::RuntimeAttribute { +public: + struct SparsityDataDesc { + uint8_t segments_type; + const void* segments; + uint8_t indices_type; + const void* indices; + }; + + OPENVINO_RTTI("SparsityInfo"); + SparsityInfo() = default; + explicit SparsityInfo(const std::vector& shape, + const std::vector& traversal_order, + const std::vector& block_map, + const std::vector& dim_format, + const std::vector& data_desc) + : m_shape(shape), + m_traversal_order(traversal_order), + m_block_map(block_map), + m_dim_format(dim_format), + m_data_desc(data_desc) { + enable(); + } + + bool is_copyable() const override; + + // Getters and setters for private attributes + const std::vector& get_shape() const { + return m_shape; + } + void set_shape(const std::vector& shape) { + m_shape = shape; + } + const std::vector& get_traversal_order() const { + return m_traversal_order; + } + void set_traversal_order(const std::vector& traversal_order) { + m_traversal_order = traversal_order; + } + const std::vector& get_block_map() const { + return m_block_map; + } + void set_block_map(const std::vector& block_map) { + m_block_map = block_map; + } + const std::vector& get_dim_format() const { + return m_dim_format; + } + void set_dim_format(const std::vector& dim_format) { + m_dim_format = dim_format; + } + const std::vector& get_data_desc() const { + return m_data_desc; + } + void set_data_desc(const std::vector& data_desc) { + m_data_desc = data_desc; + } + bool is_disabled() const { + return m_disabled; + } + void disable() { + m_disabled = true; + } + void enable() { + // We dont count on data_desc in case other data is absent + m_disabled = (m_shape.size() == 0 || m_traversal_order.size() == 0 || m_block_map.size() == 0 || + m_dim_format.size() == 0); + } + // Unpack sparse tensor and returns dense data + void* dense_data() { + if (m_disabled) + FRONT_END_THROW("Try to get dense data from unsupported tensor"); + if (m_data.size() > 0) + return m_data.data(); + else + return densify(); + } + +private: + std::vector m_shape; // Target shape of the tensor + std::vector m_traversal_order; // Sparse tensor traversal order + std::vector m_block_map; // Block map of the tensor + std::vector m_dim_format; // List of dimension's format + std::vector m_data_desc; // Tensor data descriptors + std::vector m_data; // Dense data + bool m_disabled; + + // Unpack sparse tensor and return pointer on unpacked data + void* densify(); +}; +} // namespace tensorflow_lite +} // namespace frontend +} // namespace ov diff --git a/src/frontends/tensorflow_lite/src/tensor_lite_place.cpp b/src/frontends/tensorflow_lite/src/tensor_lite_place.cpp index 240acc10bfc..f461d783e2c 100644 --- a/src/frontends/tensorflow_lite/src/tensor_lite_place.cpp +++ b/src/frontends/tensorflow_lite/src/tensor_lite_place.cpp @@ -11,6 +11,7 @@ void ov::frontend::tensorflow_lite::TensorLitePlace::translate(ov::Output& names, std::shared_ptr quantization, + std::shared_ptr sparsity, const void* data) : ov::frontend::tensorflow::TensorPlace(input_model, pshape, type, names), m_quantization(quantization), - m_data(data){}; + m_sparsity(sparsity), + m_data(m_sparsity == nullptr || m_sparsity->is_disabled() ? data : m_sparsity->dense_data()){}; void translate(ov::Output& output, bool convert_tensor_attrs_to_nodes = false); @@ -56,6 +59,7 @@ public: protected: std::shared_ptr m_quantization; + std::shared_ptr m_sparsity; int64_t m_input_idx = -1, m_output_idx = -1; const void* m_data; }; diff --git a/src/frontends/tensorflow_lite/src/utils.cpp b/src/frontends/tensorflow_lite/src/utils.cpp index 44170a2cbe4..66930282d0b 100644 --- a/src/frontends/tensorflow_lite/src/utils.cpp +++ b/src/frontends/tensorflow_lite/src/utils.cpp @@ -28,6 +28,37 @@ std::shared_ptr ov::frontend::t return quantization; } +std::shared_ptr ov::frontend::tensorflow_lite::get_sparsity( + const flatbuffers::Vector* tf_shape, + const tflite::SparsityParameters* tf_sparsity) { + if (tf_shape == nullptr) + return {}; + if (tf_sparsity == nullptr) + return {}; + auto sparsity = std::make_shared(); + sparsity->set_shape({tf_shape->begin(), tf_shape->end()}); + if (tf_sparsity->traversal_order() != nullptr) + sparsity->set_traversal_order({tf_sparsity->traversal_order()->begin(), tf_sparsity->traversal_order()->end()}); + if (tf_sparsity->block_map() != nullptr) + sparsity->set_block_map({tf_sparsity->block_map()->begin(), tf_sparsity->block_map()->end()}); + if (tf_sparsity->dim_metadata() != nullptr) { + std::vector dim_format = {}; + std::vector data_desc = {}; + dim_format.reserve(tf_sparsity->dim_metadata()->size()); + data_desc.reserve(tf_sparsity->dim_metadata()->size()); + for (auto it = tf_sparsity->dim_metadata()->begin(); it != tf_sparsity->dim_metadata()->end(); ++it) { + dim_format.push_back(it->format()); + data_desc.push_back({static_cast(it->array_segments_type()), + it->array_segments(), + static_cast(it->array_indices_type()), + it->array_indices()}); + } + sparsity->set_dim_format(dim_format); + sparsity->set_data_desc(data_desc); + } + return sparsity; +} + ov::element::Type ov::frontend::tensorflow_lite::get_ov_type(const tflite::TensorType& tf_type) { static const std::map type_map{ {tflite::TensorType_FLOAT32, element::f32}, diff --git a/src/frontends/tensorflow_lite/src/utils.hpp b/src/frontends/tensorflow_lite/src/utils.hpp index 640c7683bf1..7bcbff4340c 100644 --- a/src/frontends/tensorflow_lite/src/utils.hpp +++ b/src/frontends/tensorflow_lite/src/utils.hpp @@ -8,13 +8,13 @@ #include "place.hpp" #include "quantization_info.hpp" #include "schema_generated.h" +#include "sparsity_info.hpp" namespace ov { namespace frontend { namespace tensorflow_lite { class TensorLitePlace; -class QuantizationInfo; ov::element::Type get_ov_type(const tflite::TensorType& tf_type); ov::PartialShape get_ov_shape(const flatbuffers::Vector* tf_shape, @@ -22,6 +22,8 @@ ov::PartialShape get_ov_shape(const flatbuffers::Vector* tf_shape, std::shared_ptr get_quantization(const tflite::QuantizationParameters* tf_quantization); void apply_quantization(ov::Output& output, ov::element::Type type); void dequantize_inputs(OutputVector& deq_inputs); +std::shared_ptr get_sparsity(const flatbuffers::Vector* tf_shape, + const tflite::SparsityParameters* tf_sparsity); template OutputVector get_indexed_outputs(const T& outputs); diff --git a/src/frontends/tensorflow_lite/tests/convert_model.cpp b/src/frontends/tensorflow_lite/tests/convert_model.cpp index 99d1e2c8326..9a444037ef3 100644 --- a/src/frontends/tensorflow_lite/tests/convert_model.cpp +++ b/src/frontends/tensorflow_lite/tests/convert_model.cpp @@ -12,6 +12,7 @@ using TFLiteConvertModelTest = FrontEndConvertModelTest; static const std::vector models{ std::string("2in_2out/2in_2out.tflite"), + std::string("downloads/pose_detector.tflite"), std::string("bad_header/zerolen.tflite"), std::string("bad_header/wrong_len_3.tflite"), std::string("bad_header/wrong_pos.tflite"), diff --git a/src/frontends/tensorflow_lite/tests/test_models/gen_scripts/generate_pose_detector.py b/src/frontends/tensorflow_lite/tests/test_models/gen_scripts/generate_pose_detector.py new file mode 100644 index 00000000000..4ee4d4636c8 --- /dev/null +++ b/src/frontends/tensorflow_lite/tests/test_models/gen_scripts/generate_pose_detector.py @@ -0,0 +1,24 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# This is a way to test Densify operation. Should be removed after enabling Layer-based test + +import numpy as np +import os +import sys +import urllib.request +import zipfile + +path_to_model_dir = os.path.join(sys.argv[1], "downloads") +tflite_file_name = 'pose_detector.tflite' +tflite_model_path = os.path.join(path_to_model_dir, tflite_file_name) +task_file_name = 'pose_landmarker_lite.task' +task_file_path = os.path.join(path_to_model_dir, task_file_name) +if not os.path.exists(path_to_model_dir): + os.makedirs(path_to_model_dir, exist_ok=True) +if not os.path.exists(tflite_model_path): + if not os.path.exists(task_file_path): + urllib.request.urlretrieve("https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/latest/pose_landmarker_lite.task", task_file_path) + with zipfile.ZipFile(task_file_path, "r") as f: + f.extract(tflite_file_name, path_to_model_dir) +