[BENCHMARK_APP] Introduce Numpy array loading for C++ benchmark app/Fix a bug that would cause Python Numpy array loading to fail (#14021)

* [C++/BENCHMARK_APP] Introduce Numpy array loading for C++ benchmark app

* [DOCS/BENCHMARK_APP] Update docs to reflect changes, update list of available extensions from OpenCV, align help messages

* Update inputs_filling.cpp

* Update tools/benchmark_tool/openvino/tools/benchmark/utils/inputs_filling.py

Co-authored-by: Zlobin Vladimir <vladimir.zlobin@intel.com>

* Update samples/cpp/benchmark_app/inputs_filling.cpp

Co-authored-by: Zlobin Vladimir <vladimir.zlobin@intel.com>

* [C++/Python] Implement quality-of-life improvements from PR comments

* [C++] Fix compilation errors, fix linter output

* [C++/PYTHON] Apply requested changes

* Update samples/cpp/benchmark_app/main.cpp

Co-authored-by: Zlobin Vladimir <vladimir.zlobin@intel.com>

* Update samples/cpp/benchmark_app/utils.cpp

Co-authored-by: Zlobin Vladimir <vladimir.zlobin@intel.com>

* [PYTHON] Separate loading of numpy arrays similar to images

* [PYTHON] Remove unnecessary 'Prepare xxx file' print

* Update README again because IF OPENCV.. dissapeared for some reason

* Update second README with missing IF OPENCV..

* [C++] Remove unnecessary vector print function

* [C++ Add Numpy processing function - TODO link it to the tensor filling

* Reverse OneDnn plugin modification

* [C++] Numpy array loading for C++

* [C++] Add (almost) all missing types of data

* Reverse submodule modifications

* [C++/PYTHON] Fix compilation errors, clean code

* [C++] Modify supported extensions, add numpy checking to utils, add numpy to get_image_info method

* Update samples/cpp/benchmark_app/inputs_filling.cpp

Co-authored-by: Zlobin Vladimir <vladimir.zlobin@intel.com>

* [C++] Fix utils header file to reflect unordered set change

* [PYTHON/C++] Fix compilation errors in C++ code, fix Python dynamic shapes numpy loading

* [C++] Fix explicit instantiation of NumpyArray reader

* [C++] Clang format, minor syntax fixes

* [PYTHON/C++] Remove unnecessary data types, introduce a new approach to cast data of different types from format_rt_reader, remove uppercase types from Python precision parameters

* [PYTHON] Update README to reflect new precision settings

* [PYTHON] Fix README, fix clang format

* [C++] Clean headers

* [C++] Fix uninitialized variable error

* [C++/PYTHON] Fixed choices in Python benchmark, fixed types in C++ benchmark

* [C++] Fixed ov::float16conversion, fixed Python types map - removed redundancies

* [C++] Add back boolean support

* [C++] Fix compilation errors

---------

Co-authored-by: Zlobin Vladimir <vladimir.zlobin@intel.com>

samples/cpp/common/format_reader/MnistUbyte.cpp

@@ -49,6 +49,8 @@ MnistUbyte::MnistUbyte(const std::string& filename) {
     }
 
     size_t size = _width * _height * 1;
+    _shape.push_back(_height);
+    _shape.push_back(_width);
 
     _data.reset(new unsigned char[size], std::default_delete<unsigned char[]>());
     size_t count = 0;

samples/cpp/common/format_reader/bmp.cpp

@@ -37,6 +37,8 @@ BitMap::BitMap(const string& filename) {
     bool rowsReversed = infoHeader.height < 0;
     _width = infoHeader.width;
     _height = abs(infoHeader.height);
+    _shape.push_back(_height);
+    _shape.push_back(_width);
 
     if (infoHeader.bits != 24) {
         cerr << "[BMP] 24bpp only supported. But input has:" << infoHeader.bits << "\n";

samples/cpp/common/format_reader/format_reader.cpp

@@ -6,9 +6,11 @@
 
 // clang-format off
 #include "bmp.h"
+#include "npy.h"
 #include "MnistUbyte.h"
 #include "yuv_nv12.h"
 #include "opencv_wrapper.h"
+
 #include "format_reader.h"
 // clang-format on
 
@@ -18,6 +20,7 @@ std::vector<Registry::CreatorFunction> Registry::_data;
 
 Register<MnistUbyte> MnistUbyte::reg;
 Register<YUV_NV12> YUV_NV12::reg;
+Register<NumpyArray> NumpyArray::reg;
 #ifdef USE_OPENCV
 Register<OCVReader> OCVReader::reg;
 #else

samples/cpp/common/format_reader/format_reader.h

@@ -42,6 +42,8 @@ protected:
     size_t _width = 0;
     /// \brief data
     std::shared_ptr<unsigned char> _data;
+    /// \brief shape - data shape
+    std::vector<size_t> _shape;
 
 public:
     virtual ~Reader() = default;
@@ -62,6 +64,14 @@ public:
         return _height;
     }
 
+    /**
+     * \brief Get full shape vector
+     * @return vector of size_t values determining data shape
+     */
+    std::vector<size_t> shape() const {
+        return _shape;
+    }
+
     /**
      * \brief Get input data ptr
      * @return shared pointer with input data

samples/cpp/common/format_reader/npy.cpp

@@ -0,0 +1,117 @@
+// Copyright (C) 2018-2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+// clang-format off
+#include <fstream>
+#include <iterator>
+#include <sstream>
+#include <string>
+#include <iostream>
+#include <algorithm>
+
+#include "npy.h"
+// clang-format on
+
+using namespace FormatReader;
+
+NumpyArray::NumpyArray(const std::string& filename) {
+    auto pos = filename.rfind('.');
+    if (pos == std::string::npos)
+        return;
+    if (filename.substr(pos + 1) != "npy")
+        return;
+
+    std::ifstream file(filename, std::ios::binary);
+    if (!file.is_open()) {
+        return;
+    }
+
+    file.seekg(0, std::ios_base::end);
+    if (!file.good()) {
+        return;
+    }
+    auto full_file_size = static_cast<std::size_t>(file.tellg());
+    file.seekg(0, std::ios_base::beg);
+
+    std::string magic_string(6, ' ');
+    file.read(&magic_string[0], magic_string.size());
+    if (magic_string != "\x93NUMPY") {
+        return;
+    }
+
+    file.ignore(2);
+    unsigned short header_size;
+    file.read((char*)&header_size, sizeof(header_size));
+
+    std::string header(header_size, ' ');
+    file.read(&header[0], header.size());
+
+    int idx, from, to;
+
+    // Verify fortran order is false
+    const std::string fortran_key = "'fortran_order':";
+    idx = header.find(fortran_key);
+    if (idx == -1) {
+        return;
+    }
+
+    from = header.find_last_of(' ', idx + fortran_key.size()) + 1;
+    to = header.find(',', from);
+    auto fortran_value = header.substr(from, to - from);
+    if (fortran_value != "False") {
+        return;
+    }
+
+    // Verify array shape matches the input's
+    const std::string shape_key = "'shape':";
+    idx = header.find(shape_key);
+    if (idx == -1) {
+        return;
+    }
+
+    from = header.find('(', idx + shape_key.size()) + 1;
+    to = header.find(')', from);
+
+    std::string shape_data = header.substr(from, to - from);
+
+    if (!shape_data.empty()) {
+        shape_data.erase(std::remove(shape_data.begin(), shape_data.end(), ','), shape_data.end());
+
+        std::istringstream shape_data_stream(shape_data);
+        size_t value;
+        while (shape_data_stream >> value) {
+            _shape.push_back(value);
+        }
+    }
+
+    // Batch / Height / Width / Other dims
+    // If batch is present, height and width are at least 1
+    if (_shape.size()) {
+        _height = _shape.size() >= 2 ? _shape.at(1) : 1;
+        _width = _shape.size() >= 3 ? _shape.at(2) : 1;
+    } else {
+        _height = 0;
+        _width = 0;
+    }
+
+    // Verify array data type matches input's
+    std::string dataTypeKey = "'descr':";
+    idx = header.find(dataTypeKey);
+    if (idx == -1) {
+        return;
+    }
+
+    from = header.find('\'', idx + dataTypeKey.size()) + 1;
+    to = header.find('\'', from);
+    type = header.substr(from, to - from);
+
+    _size = full_file_size - static_cast<std::size_t>(file.tellg());
+
+    _data.reset(new unsigned char[_size], std::default_delete<unsigned char[]>());
+    for (size_t i = 0; i < _size; i++) {
+        unsigned char buffer = 0;
+        file.read(reinterpret_cast<char*>(&buffer), sizeof(buffer));
+        _data.get()[i] = buffer;
+    }
+}

samples/cpp/common/format_reader/npy.h

@@ -0,0 +1,52 @@
+// Copyright (C) 2018-2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+/**
+ * \brief NumpyArray reader
+ * \file npy.h
+ */
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+// clang-format off
+#include "format_reader.h"
+#include "register.h"
+// clang-format on
+
+namespace FormatReader {
+/**
+ * \class NumpyArray
+ * \brief Reader for NPY files
+ */
+class NumpyArray : public Reader {
+private:
+    static Register<NumpyArray> reg;
+    std::string type;
+    size_t _size = 0;
+
+public:
+    /**
+     * \brief Constructor of NumpyArray reader
+     * @param filename - path to input data
+     * @return NumpyArray reader object
+     */
+    explicit NumpyArray(const std::string& filename);
+    virtual ~NumpyArray() {}
+
+    /**
+     * \brief Get size
+     * @return size
+     */
+    size_t size() const override {
+        return _size;
+    }
+
+    std::shared_ptr<unsigned char> getData(size_t width = 0, size_t height = 0) override {
+        return _data;
+    }
+};
+}  // namespace FormatReader

samples/cpp/common/format_reader/opencv_wrapper.cpp

@@ -27,6 +27,8 @@ OCVReader::OCVReader(const string& filename) {
     _size = img.size().width * img.size().height * img.channels();
     _width = img.size().width;
     _height = img.size().height;
+    _shape.push_back(_height);
+    _shape.push_back(_width);
 }
 
 std::shared_ptr<unsigned char> OCVReader::getData(size_t width = 0, size_t height = 0) {