[TFLite] Initial support of Densify operation

* Added Densify

src/frontends/tensorflow_common/src/op/identity.cpp

@@ -21,7 +21,9 @@ OutputVector translate_identity_op(const NodeContext& node) {
                                     "StopGradient",
                                     "ReadVariableOp",
                                     "ShardedFilename",
-                                    "MergeV2Checkpoints"};
+                                    "MergeV2Checkpoints",
+                                    // TF Lite nodes
+                                    "DENSIFY"};
     default_op_checks(node, 1, supported_ops);
     auto input = node.get_input(0);
 

src/frontends/tensorflow_lite/src/decoder_flatbuffer.cpp

@@ -92,6 +92,7 @@ std::shared_ptr<ov::frontend::tensorflow_lite::TensorLitePlace> DecoderFlatBuffe
         ov::frontend::tensorflow_lite::get_ov_type(tensor->type()),
         names,
         ov::frontend::tensorflow_lite::get_quantization(tensor->quantization()),
+        ov::frontend::tensorflow_lite::get_sparsity(tensor->shape(), tensor->sparsity()),
         (tensor_info.buffer && tensor_info.buffer->data() ? tensor_info.buffer->data()->data() : nullptr));
 }
 

src/frontends/tensorflow_lite/src/op_table.cpp

@@ -66,7 +66,7 @@ std::map<std::string, CreatorFunction> get_supported_ops() {
         // CUMSUM
         // CUSTOM
         // DELEGATE
-        // DENSIFY
+        {"DENSIFY", translate_identity_op},
         {"DEPTH_TO_SPACE", DEQUANTIZE_INPUTS(depth_to_space)},
         {"DEPTHWISE_CONV_2D", DEQUANTIZE_INPUTS(depthwise_conv2d)},
         {"DEQUANTIZE", DEQUANTIZE_INPUTS(dequantize)},

src/frontends/tensorflow_lite/src/sparsity_info.cpp

@@ -0,0 +1,163 @@
+// Copyright (C) 2018-2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "sparsity_info.hpp"
+
+#include "schema_generated.h"
+
+bool ov::frontend::tensorflow_lite::SparsityInfo::is_copyable() const {
+    return false;
+}
+
+// Unpacks sparse data. Supports only case when sparse tensor has only one sparse dimension [DENSE, ..., DENSE, SPARSE]
+template <typename T, typename U>
+static void read_sparse_data(uint8_t* dest,
+                             uint8_t* dest_end,
+                             const size_t row_size,
+                             const ::flatbuffers::Vector<T>* indices,
+                             const ::flatbuffers::Vector<U>* segments) {
+    uint8_t* data = dest - row_size;  // row size will be increased at first step
+    T last_idx = ~static_cast<T>(0);
+    for (auto idx = indices->begin(); idx != indices->end(); ++idx) {
+        if (*idx <= last_idx) {
+            data += row_size;
+        }
+        FRONT_END_GENERAL_CHECK(data + *idx < dest_end, "Dense data is out of bounds");
+        static_cast<U*>(static_cast<void*>(data))[*idx] = segments->Get(*idx);
+        last_idx = *idx;
+    }
+}
+
+void* ov::frontend::tensorflow_lite::SparsityInfo::densify() {
+    size_t sparse_idx = 0;
+    for (; sparse_idx < m_dim_format.size(); ++sparse_idx) {
+        if (m_dim_format[sparse_idx] == ::tflite::DimensionType_SPARSE_CSR)
+            break;
+    }
+    FRONT_END_GENERAL_CHECK(sparse_idx < m_dim_format.size(), "Sparse dimension isn't found for sparse tensor");
+    FRONT_END_GENERAL_CHECK(sparse_idx == (m_dim_format.size() - 1),
+                            "Supports only sparse tensor with sparse dimension as a last dimension");
+
+    size_t total_size = 0,  // Size of data in bytes
+        row_size = 0;       // Size of data row in bytes
+    switch (m_data_desc[sparse_idx].segments_type) {
+    case ::tflite::SparseIndexVector_Uint8Vector:
+        total_size = 1;
+        break;
+    case ::tflite::SparseIndexVector_Uint16Vector:
+        total_size = 2;
+        break;
+    case ::tflite::SparseIndexVector_Int32Vector:
+        total_size = 4;
+        break;
+    }
+    row_size = total_size;  // Byte size is same
+    for (size_t dim = 0; dim < m_shape.size(); ++dim) {
+        total_size *= m_shape[dim];
+        switch (m_dim_format[dim]) {
+        case ::tflite::DimensionType_DENSE:
+            break;
+        case ::tflite::DimensionType_SPARSE_CSR:
+            row_size *= m_shape[dim];
+            break;
+        default:
+            FRONT_END_THROW("Unsupported dimension type found");
+            break;
+        }
+    }
+    FRONT_END_GENERAL_CHECK(total_size > 0, "Wrong sparse segment size found");
+
+    m_data.resize(total_size);
+    memset(m_data.data(), 0, total_size);
+    switch (m_data_desc[sparse_idx].indices_type) {
+    case ::tflite::SparseIndexVector_Uint8Vector:
+        switch (m_data_desc[sparse_idx].segments_type) {
+        case ::tflite::SparseIndexVector_Uint8Vector:
+            read_sparse_data(m_data.data(),
+                             m_data.data() + total_size,
+                             row_size,
+                             static_cast<const ::tflite::Uint8Vector*>(m_data_desc[sparse_idx].indices)->values(),
+                             static_cast<const ::tflite::Uint8Vector*>(m_data_desc[sparse_idx].segments)->values());
+            break;
+        case ::tflite::SparseIndexVector_Uint16Vector:
+            read_sparse_data(m_data.data(),
+                             m_data.data() + total_size,
+                             row_size,
+                             static_cast<const ::tflite::Uint8Vector*>(m_data_desc[sparse_idx].indices)->values(),
+                             static_cast<const ::tflite::Uint16Vector*>(m_data_desc[sparse_idx].segments)->values());
+            break;
+        case ::tflite::SparseIndexVector_Int32Vector:
+            read_sparse_data(m_data.data(),
+                             m_data.data() + total_size,
+                             row_size,
+                             static_cast<const ::tflite::Uint8Vector*>(m_data_desc[sparse_idx].indices)->values(),
+                             static_cast<const ::tflite::Int32Vector*>(m_data_desc[sparse_idx].segments)->values());
+            break;
+        default:
+            FRONT_END_THROW("Unsupported vector type");
+            break;
+        }
+        break;
+    case ::tflite::SparseIndexVector_Uint16Vector:
+        switch (m_data_desc[sparse_idx].segments_type) {
+        case ::tflite::SparseIndexVector_Uint8Vector:
+            read_sparse_data(m_data.data(),
+                             m_data.data() + total_size,
+                             row_size,
+                             static_cast<const ::tflite::Uint16Vector*>(m_data_desc[sparse_idx].indices)->values(),
+                             static_cast<const ::tflite::Uint8Vector*>(m_data_desc[sparse_idx].segments)->values());
+            break;
+        case ::tflite::SparseIndexVector_Uint16Vector:
+            read_sparse_data(m_data.data(),
+                             m_data.data() + total_size,
+                             row_size,
+                             static_cast<const ::tflite::Uint16Vector*>(m_data_desc[sparse_idx].indices)->values(),
+                             static_cast<const ::tflite::Uint16Vector*>(m_data_desc[sparse_idx].segments)->values());
+            break;
+        case ::tflite::SparseIndexVector_Int32Vector:
+            read_sparse_data(m_data.data(),
+                             m_data.data() + total_size,
+                             row_size,
+                             static_cast<const ::tflite::Uint16Vector*>(m_data_desc[sparse_idx].indices)->values(),
+                             static_cast<const ::tflite::Int32Vector*>(m_data_desc[sparse_idx].segments)->values());
+            break;
+        default:
+            FRONT_END_THROW("Unsupported vector type");
+            break;
+        }
+        break;
+    case ::tflite::SparseIndexVector_Int32Vector:
+        switch (m_data_desc[sparse_idx].segments_type) {
+        case ::tflite::SparseIndexVector_Uint8Vector:
+            read_sparse_data(m_data.data(),
+                             m_data.data() + total_size,
+                             row_size,
+                             static_cast<const ::tflite::Int32Vector*>(m_data_desc[sparse_idx].indices)->values(),
+                             static_cast<const ::tflite::Uint8Vector*>(m_data_desc[sparse_idx].segments)->values());
+            break;
+        case ::tflite::SparseIndexVector_Uint16Vector:
+            read_sparse_data(m_data.data(),
+                             m_data.data() + total_size,
+                             row_size,
+                             static_cast<const ::tflite::Int32Vector*>(m_data_desc[sparse_idx].indices)->values(),
+                             static_cast<const ::tflite::Uint16Vector*>(m_data_desc[sparse_idx].segments)->values());
+            break;
+        case ::tflite::SparseIndexVector_Int32Vector:
+            read_sparse_data(m_data.data(),
+                             m_data.data() + total_size,
+                             row_size,
+                             static_cast<const ::tflite::Int32Vector*>(m_data_desc[sparse_idx].indices)->values(),
+                             static_cast<const ::tflite::Int32Vector*>(m_data_desc[sparse_idx].segments)->values());
+            break;
+        default:
+            FRONT_END_THROW("Unsupported vector type");
+            break;
+        }
+        break;
+    default:
+        FRONT_END_THROW("Unsupported vector type");
+        break;
+    }
+    return m_data.data();
+}

src/frontends/tensorflow_lite/src/sparsity_info.hpp

@@ -0,0 +1,111 @@
+// Copyright (C) 2018-2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <functional>
+#include <string>
+#include <vector>
+
+#include "common_op_table.hpp"
+#include "decoder_map.hpp"
+#include "openvino/frontend/tensorflow_lite/visibility.hpp"
+
+namespace ov {
+namespace frontend {
+namespace tensorflow_lite {
+
+class SparsityInfo : public ov::RuntimeAttribute {
+public:
+    struct SparsityDataDesc {
+        uint8_t segments_type;
+        const void* segments;
+        uint8_t indices_type;
+        const void* indices;
+    };
+
+    OPENVINO_RTTI("SparsityInfo");
+    SparsityInfo() = default;
+    explicit SparsityInfo(const std::vector<int32_t>& shape,
+                          const std::vector<int32_t>& traversal_order,
+                          const std::vector<int32_t>& block_map,
+                          const std::vector<uint16_t>& dim_format,
+                          const std::vector<SparsityDataDesc>& data_desc)
+        : m_shape(shape),
+          m_traversal_order(traversal_order),
+          m_block_map(block_map),
+          m_dim_format(dim_format),
+          m_data_desc(data_desc) {
+        enable();
+    }
+
+    bool is_copyable() const override;
+
+    // Getters and setters for private attributes
+    const std::vector<int32_t>& get_shape() const {
+        return m_shape;
+    }
+    void set_shape(const std::vector<int32_t>& shape) {
+        m_shape = shape;
+    }
+    const std::vector<int32_t>& get_traversal_order() const {
+        return m_traversal_order;
+    }
+    void set_traversal_order(const std::vector<int32_t>& traversal_order) {
+        m_traversal_order = traversal_order;
+    }
+    const std::vector<int32_t>& get_block_map() const {
+        return m_block_map;
+    }
+    void set_block_map(const std::vector<int32_t>& block_map) {
+        m_block_map = block_map;
+    }
+    const std::vector<uint16_t>& get_dim_format() const {
+        return m_dim_format;
+    }
+    void set_dim_format(const std::vector<uint16_t>& dim_format) {
+        m_dim_format = dim_format;
+    }
+    const std::vector<SparsityDataDesc>& get_data_desc() const {
+        return m_data_desc;
+    }
+    void set_data_desc(const std::vector<SparsityDataDesc>& data_desc) {
+        m_data_desc = data_desc;
+    }
+    bool is_disabled() const {
+        return m_disabled;
+    }
+    void disable() {
+        m_disabled = true;
+    }
+    void enable() {
+        // We dont count on data_desc in case other data is absent
+        m_disabled = (m_shape.size() == 0 || m_traversal_order.size() == 0 || m_block_map.size() == 0 ||
+                      m_dim_format.size() == 0);
+    }
+    // Unpack sparse tensor and returns dense data
+    void* dense_data() {
+        if (m_disabled)
+            FRONT_END_THROW("Try to get dense data from unsupported tensor");
+        if (m_data.size() > 0)
+            return m_data.data();
+        else
+            return densify();
+    }
+
+private:
+    std::vector<int32_t> m_shape;               // Target shape of the tensor
+    std::vector<int32_t> m_traversal_order;     // Sparse tensor traversal order
+    std::vector<int32_t> m_block_map;           // Block map of the tensor
+    std::vector<uint16_t> m_dim_format;         // List of dimension's format
+    std::vector<SparsityDataDesc> m_data_desc;  // Tensor data descriptors
+    std::vector<uint8_t> m_data;                // Dense data
+    bool m_disabled;
+
+    // Unpack sparse tensor and return pointer on unpacked data
+    void* densify();
+};
+}  // namespace tensorflow_lite
+}  // namespace frontend
+}  // namespace ov

src/frontends/tensorflow_lite/src/tensor_lite_place.cpp

@@ -11,6 +11,7 @@ void ov::frontend::tensorflow_lite::TensorLitePlace::translate(ov::Output<ov::No
                                                                bool convert_tensor_attrs_to_nodes) {
     output.set_names({*get_names().begin()});
     output.get_rt_info()[ov::frontend::tensorflow_lite::QuantizationInfo::get_type_info_static()] = m_quantization;
+    output.get_rt_info()[ov::frontend::tensorflow_lite::SparsityInfo::get_type_info_static()] = m_sparsity;
     if (convert_tensor_attrs_to_nodes)
         apply_quantization(output, get_element_type());
 }

src/frontends/tensorflow_lite/src/tensor_lite_place.hpp

@@ -10,6 +10,7 @@
 #include "openvino/frontend/tensorflow_lite/visibility.hpp"
 #include "place.hpp"
 #include "quantization_info.hpp"
+#include "sparsity_info.hpp"
 
 namespace ov {
 namespace frontend {
@@ -22,10 +23,12 @@ public:
                     ov::element::Type type,
                     const std::vector<std::string>& names,
                     std::shared_ptr<ov::frontend::tensorflow_lite::QuantizationInfo> quantization,
+                    std::shared_ptr<ov::frontend::tensorflow_lite::SparsityInfo> sparsity,
                     const void* data)
         : ov::frontend::tensorflow::TensorPlace(input_model, pshape, type, names),
           m_quantization(quantization),
-          m_data(data){};
+          m_sparsity(sparsity),
+          m_data(m_sparsity == nullptr || m_sparsity->is_disabled() ? data : m_sparsity->dense_data()){};
 
     void translate(ov::Output<ov::Node>& output, bool convert_tensor_attrs_to_nodes = false);
 
@@ -56,6 +59,7 @@ public:
 
 protected:
     std::shared_ptr<ov::frontend::tensorflow_lite::QuantizationInfo> m_quantization;
+    std::shared_ptr<ov::frontend::tensorflow_lite::SparsityInfo> m_sparsity;
     int64_t m_input_idx = -1, m_output_idx = -1;
     const void* m_data;
 };

src/frontends/tensorflow_lite/src/utils.cpp

@@ -28,6 +28,37 @@ std::shared_ptr<ov::frontend::tensorflow_lite::QuantizationInfo> ov::frontend::t
     return quantization;
 }
 
+std::shared_ptr<ov::frontend::tensorflow_lite::SparsityInfo> ov::frontend::tensorflow_lite::get_sparsity(
+    const flatbuffers::Vector<int32_t>* tf_shape,
+    const tflite::SparsityParameters* tf_sparsity) {
+    if (tf_shape == nullptr)
+        return {};
+    if (tf_sparsity == nullptr)
+        return {};
+    auto sparsity = std::make_shared<ov::frontend::tensorflow_lite::SparsityInfo>();
+    sparsity->set_shape({tf_shape->begin(), tf_shape->end()});
+    if (tf_sparsity->traversal_order() != nullptr)
+        sparsity->set_traversal_order({tf_sparsity->traversal_order()->begin(), tf_sparsity->traversal_order()->end()});
+    if (tf_sparsity->block_map() != nullptr)
+        sparsity->set_block_map({tf_sparsity->block_map()->begin(), tf_sparsity->block_map()->end()});
+    if (tf_sparsity->dim_metadata() != nullptr) {
+        std::vector<uint16_t> dim_format = {};
+        std::vector<ov::frontend::tensorflow_lite::SparsityInfo::SparsityDataDesc> data_desc = {};
+        dim_format.reserve(tf_sparsity->dim_metadata()->size());
+        data_desc.reserve(tf_sparsity->dim_metadata()->size());
+        for (auto it = tf_sparsity->dim_metadata()->begin(); it != tf_sparsity->dim_metadata()->end(); ++it) {
+            dim_format.push_back(it->format());
+            data_desc.push_back({static_cast<uint8_t>(it->array_segments_type()),
+                                 it->array_segments(),
+                                 static_cast<uint8_t>(it->array_indices_type()),
+                                 it->array_indices()});
+        }
+        sparsity->set_dim_format(dim_format);
+        sparsity->set_data_desc(data_desc);
+    }
+    return sparsity;
+}
+
 ov::element::Type ov::frontend::tensorflow_lite::get_ov_type(const tflite::TensorType& tf_type) {
     static const std::map<tflite::TensorType, ov::element::Type> type_map{
         {tflite::TensorType_FLOAT32, element::f32},

src/frontends/tensorflow_lite/src/utils.hpp

@@ -8,13 +8,13 @@
 #include "place.hpp"
 #include "quantization_info.hpp"
 #include "schema_generated.h"
+#include "sparsity_info.hpp"
 
 namespace ov {
 namespace frontend {
 namespace tensorflow_lite {
 
 class TensorLitePlace;
-class QuantizationInfo;
 
 ov::element::Type get_ov_type(const tflite::TensorType& tf_type);
 ov::PartialShape get_ov_shape(const flatbuffers::Vector<int32_t>* tf_shape,
@@ -22,6 +22,8 @@ ov::PartialShape get_ov_shape(const flatbuffers::Vector<int32_t>* tf_shape,
 std::shared_ptr<QuantizationInfo> get_quantization(const tflite::QuantizationParameters* tf_quantization);
 void apply_quantization(ov::Output<ov::Node>& output, ov::element::Type type);
 void dequantize_inputs(OutputVector& deq_inputs);
+std::shared_ptr<SparsityInfo> get_sparsity(const flatbuffers::Vector<int32_t>* tf_shape,
+                                           const tflite::SparsityParameters* tf_sparsity);
 
 template <typename T>
 OutputVector get_indexed_outputs(const T& outputs);

src/frontends/tensorflow_lite/tests/convert_model.cpp

@@ -12,6 +12,7 @@ using TFLiteConvertModelTest = FrontEndConvertModelTest;
 
 static const std::vector<std::string> models{
     std::string("2in_2out/2in_2out.tflite"),
+    std::string("downloads/pose_detector.tflite"),
     std::string("bad_header/zerolen.tflite"),
     std::string("bad_header/wrong_len_3.tflite"),
     std::string("bad_header/wrong_pos.tflite"),

src/frontends/tensorflow_lite/tests/test_models/gen_scripts/generate_pose_detector.py

@@ -0,0 +1,24 @@
+# Copyright (C) 2018-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# This is a way to test Densify operation. Should be removed after enabling Layer-based test
+
+import numpy as np
+import os
+import sys
+import urllib.request
+import zipfile
+
+path_to_model_dir = os.path.join(sys.argv[1], "downloads")
+tflite_file_name = 'pose_detector.tflite'
+tflite_model_path = os.path.join(path_to_model_dir, tflite_file_name)
+task_file_name = 'pose_landmarker_lite.task'
+task_file_path = os.path.join(path_to_model_dir, task_file_name)
+if not os.path.exists(path_to_model_dir):
+    os.makedirs(path_to_model_dir, exist_ok=True)
+if not os.path.exists(tflite_model_path):
+    if not os.path.exists(task_file_path):
+        urllib.request.urlretrieve("https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/latest/pose_landmarker_lite.task", task_file_path)
+    with zipfile.ZipFile(task_file_path, "r") as f:
+        f.extract(tflite_file_name, path_to_model_dir)
+