From 5512c363588cd5c92613f8ed79695fce3eb918d3 Mon Sep 17 00:00:00 2001 From: Anton Romanov Date: Tue, 16 Nov 2021 14:13:16 +0300 Subject: [PATCH] remove opencv dependecies in samples (#8123) * remove opencv dependecies ins samples * fixed output --- .../samples/benchmark_app/CMakeLists.txt | 6 +- .../utils/include/samples/args_helper.hpp | 1 + .../common/utils/include/samples/common.hpp | 30 +++++ .../samples/common/utils/src/args_helper.cpp | 32 +++++ .../hello_classification/CMakeLists.txt | 3 +- .../samples/hello_classification/main.cpp | 125 ++++++------------ .../samples/hello_reshape_ssd/CMakeLists.txt | 3 +- .../samples/hello_reshape_ssd/main.cpp | 116 ++++++++++++---- .../reshape_ssd_extension.hpp | 3 + 9 files changed, 201 insertions(+), 118 deletions(-) diff --git a/inference-engine/samples/benchmark_app/CMakeLists.txt b/inference-engine/samples/benchmark_app/CMakeLists.txt index 4913084f94d..b040829b422 100644 --- a/inference-engine/samples/benchmark_app/CMakeLists.txt +++ b/inference-engine/samples/benchmark_app/CMakeLists.txt @@ -4,6 +4,10 @@ set(TARGET_NAME "benchmark_app") +if(ENABLE_OPENCV) + set(OPENCV_MODULES core) +endif() + file (GLOB SRC ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) file (GLOB HDR ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp) @@ -11,7 +15,7 @@ ie_add_sample(NAME ${TARGET_NAME} SOURCES ${SRC} HEADERS ${HDR} DEPENDENCIES format_reader ie_samples_utils - OPENCV_DEPENDENCIES core) + OPENCV_DEPENDENCIES ${OPENCV_MODULES}) find_package(OpenCL) diff --git a/inference-engine/samples/common/utils/include/samples/args_helper.hpp b/inference-engine/samples/common/utils/include/samples/args_helper.hpp index f216f370f5d..e05f1ee905e 100644 --- a/inference-engine/samples/common/utils/include/samples/args_helper.hpp +++ b/inference-engine/samples/common/utils/include/samples/args_helper.hpp @@ -38,3 +38,4 @@ void processLayout(InferenceEngine::CNNNetwork& network, const std::string& iol); void printInputAndOutputsInfo(const InferenceEngine::CNNNetwork& network); +void printInputAndOutputsInfo(const ov::Function& network); diff --git a/inference-engine/samples/common/utils/include/samples/common.hpp b/inference-engine/samples/common/utils/include/samples/common.hpp index 397e1f0ab35..728071441e0 100644 --- a/inference-engine/samples/common/utils/include/samples/common.hpp +++ b/inference-engine/samples/common/utils/include/samples/common.hpp @@ -33,6 +33,36 @@ # endif #endif +/** + * @brief Unicode string wrappers + */ +#if defined(ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) +# define tchar wchar_t +# define tstring std::wstring +# define tmain wmain +# define TSTRING2STRING(tstr) wstring2string(tstr) +#else +# define tchar char +# define tstring std::string +# define tmain main +# define TSTRING2STRING(tstr) tstr +#endif + +#if defined(ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) + +/** + * @brief Convert wstring to string + * @param ref on wstring + * @return string + */ +inline std::string wstring2string(const std::wstring& wstr) { + std::string str; + for (auto&& wc : wstr) + str += static_cast(wc); + return str; +} +#endif + /** * @brief trim from start (in place) * @param s - string to trim diff --git a/inference-engine/samples/common/utils/src/args_helper.cpp b/inference-engine/samples/common/utils/src/args_helper.cpp index 43bf96d86a6..a1cf3219054 100644 --- a/inference-engine/samples/common/utils/src/args_helper.cpp +++ b/inference-engine/samples/common/utils/src/args_helper.cpp @@ -326,3 +326,35 @@ void printInputAndOutputsInfo(const InferenceEngine::CNNNetwork& network) { << layer.second->getLayout() << std::endl; } } + +void printInputAndOutputsInfo(const ov::Function& network) { + slog::info << "model name: " << network.get_friendly_name() << slog::endl; + + const std::vector> inputs = network.inputs(); + for (const ov::Output input : inputs) { + slog::info << " inputs" << slog::endl; + + const std::string name = input.get_names().empty() ? "NONE" : input.get_any_name(); + slog::info << " input name: " << name << slog::endl; + + const ov::element::Type type = input.get_element_type(); + slog::info << " input type: " << type << slog::endl; + + const ov::Shape shape = input.get_shape(); + slog::info << " input shape: " << shape << slog::endl; + } + + const std::vector> outputs = network.outputs(); + for (const ov::Output output : outputs) { + slog::info << " outputs" << slog::endl; + + const std::string name = output.get_names().empty() ? "NONE" : output.get_any_name(); + slog::info << " output name: " << name << slog::endl; + + const ov::element::Type type = output.get_element_type(); + slog::info << " output type: " << type << slog::endl; + + const ov::Shape shape = output.get_shape(); + slog::info << " output shape: " << shape << slog::endl; + } +} diff --git a/inference-engine/samples/hello_classification/CMakeLists.txt b/inference-engine/samples/hello_classification/CMakeLists.txt index 9b0509bab82..bef82baa3ce 100644 --- a/inference-engine/samples/hello_classification/CMakeLists.txt +++ b/inference-engine/samples/hello_classification/CMakeLists.txt @@ -4,6 +4,5 @@ ie_add_sample(NAME hello_classification SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/main.cpp" - DEPENDENCIES ie_samples_utils - OPENCV_DEPENDENCIES core imgcodecs) + DEPENDENCIES format_reader ie_samples_utils) diff --git a/inference-engine/samples/hello_classification/main.cpp b/inference-engine/samples/hello_classification/main.cpp index baa359cbbd6..f352623d88d 100644 --- a/inference-engine/samples/hello_classification/main.cpp +++ b/inference-engine/samples/hello_classification/main.cpp @@ -4,119 +4,72 @@ #include #include -#include #include #include -#include "openvino/core/layout.hpp" +// clang-format off #include "openvino/openvino.hpp" -#include "samples/classification_results.h" + +#include "samples/args_helper.hpp" #include "samples/common.hpp" -#include "samples/ocv_common.hpp" +#include "samples/classification_results.h" +#include "samples/slog.hpp" +#include "format_reader_ptr.h" +// clang-format on using namespace ov::preprocess; -/** - * @brief Define names based depends on Unicode path support - */ -#if defined(ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) -# define tcout std::wcout -# define file_name_t std::wstring -# define imread_t imreadW -# define ClassificationResult_t ClassificationResultW -#else -# define tcout std::cout -# define file_name_t std::string -# define imread_t cv::imread -# define ClassificationResult_t ClassificationResult -#endif - -#if defined(ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) -/** - * @brief Realization cv::imread with support Unicode paths - */ -cv::Mat imreadW(std::wstring input_image_path) { - cv::Mat image; - std::ifstream input_image_stream; - input_image_stream.open(input_image_path.c_str(), std::iostream::binary | std::ios_base::ate | std::ios_base::in); - if (input_image_stream.is_open()) { - if (input_image_stream.good()) { - input_image_stream.seekg(0, std::ios::end); - std::size_t file_size = input_image_stream.tellg(); - input_image_stream.seekg(0, std::ios::beg); - std::vector buffer(0); - std::copy(std::istreambuf_iterator(input_image_stream), - std::istreambuf_iterator(), - std::back_inserter(buffer)); - image = cv::imdecode(cv::Mat(1, file_size, CV_8UC1, &buffer[0]), cv::IMREAD_COLOR); - } else { - tcout << "Input file '" << input_image_path << "' processing error" << std::endl; - } - input_image_stream.close(); - } else { - tcout << "Unable to read input file '" << input_image_path << "'" << std::endl; - } - return image; -} - -/** - * @brief Convert wstring to string - * @param ref on wstring - * @return string - */ -std::string simpleConvert(const std::wstring& wstr) { - std::string str; - for (auto&& wc : wstr) - str += static_cast(wc); - return str; -} - /** * @brief Main with support Unicode paths, wide strings */ -int wmain(int argc, wchar_t* argv[]) { -#else - -int main(int argc, char* argv[]) { -#endif +int tmain(int argc, tchar* argv[]) { try { - // -------- Get OpenVINO Runtime version -------- - slog::info << "OpenVINO runtime: " << ov::get_openvino_version() << slog::endl; + // -------- Get OpenVINO runtime version -------- + slog::info << ov::get_openvino_version() << slog::endl; // -------- Parsing and validation of input arguments -------- if (argc != 4) { - tcout << "Usage : " << argv[0] << " " << std::endl; + slog::info << "Usage : " << argc << " " << slog::endl; return EXIT_FAILURE; } - const file_name_t input_model{argv[1]}; - const file_name_t input_image_path{argv[2]}; -#if defined(ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) - const std::string device_name = simpleConvert(argv[3]); -#else - const std::string device_name{argv[3]}; -#endif + const std::string args = TSTRING2STRING(argv[0]); + const std::string model_path = TSTRING2STRING(argv[1]); + const std::string image_path = TSTRING2STRING(argv[2]); + const std::string device_name = TSTRING2STRING(argv[3]); // -------- Step 1. Initialize OpenVINO Runtime Core -------- ov::runtime::Core core; // -------- Step 2. Read a model -------- - auto model = core.read_model(input_model); + slog::info << "Loading model files: " << model_path << slog::endl; + auto model = core.read_model(model_path); + printInputAndOutputsInfo(*model); OPENVINO_ASSERT(model->get_parameters().size() == 1, "Sample supports models with 1 input only"); OPENVINO_ASSERT(model->get_results().size() == 1, "Sample supports models with 1 output only"); - // -------- Step 3. Initialize inference engine core + // -------- Step 3. Set up input // Read input image to a tensor and set it to an infer request // without resize and layout conversions - cv::Mat image = imread_t(input_image_path); - // just wrap Mat data by ov::runtime::Tensor without allocating of new memory - ov::runtime::Tensor input_tensor = wrapMat2Tensor(image); + FormatReader::ReaderPtr reader(image_path.c_str()); + if (reader.get() == nullptr) { + slog::warn << "Image " + image_path + " cannot be read!" << slog::endl; + throw std::logic_error(""); + } + + ov::element::Type input_type = ov::element::u8; + ov::Shape input_shape = {1, reader->height(), reader->width(), 3}; + std::shared_ptr input_data = reader->getData(); + + // just wrap image data by ov::runtime::Tensor without allocating of new memory + ov::runtime::Tensor input_tensor = ov::runtime::Tensor(input_type, input_shape, input_data.get()); + const ov::Shape tensor_shape = input_tensor.get_shape(); + const ov::Layout tensor_layout{"NHWC"}; // -------- Step 4. Apply preprocessing -------- - const ov::Layout tensor_layout{"NHWC"}; // clang-format off model = PrePostProcessor(). @@ -147,8 +100,8 @@ int main(int argc, char* argv[]) { // - precision of tensor is supposed to be 'f32' tensor(OutputTensorInfo(). set_element_type(ov::element::f32))). - // 6) Apply preprocessing modifing the original 'model' - build(model); + // 6) Apply preprocessing modifing the original 'model' + build(model); // clang-format on // -------- Step 5. Loading a model to the device -------- @@ -165,18 +118,16 @@ int main(int argc, char* argv[]) { infer_request.infer(); // -------- Step 9. Process output - ov::runtime::Tensor output_tensor = infer_request.get_output_tensor(); + const ov::runtime::Tensor& output_tensor = infer_request.get_output_tensor(); // Print classification results - ClassificationResult_t classification_result(output_tensor, {input_image_path}); + ClassificationResult classification_result(output_tensor, {image_path}); classification_result.show(); // ----------------------------------------------------------------------------------------------------- } catch (const std::exception& ex) { std::cerr << ex.what() << std::endl; return EXIT_FAILURE; } - std::cout << "This sample is an API example, for any performance measurements " - "please use the dedicated benchmark_app tool" - << std::endl; + return EXIT_SUCCESS; } diff --git a/inference-engine/samples/hello_reshape_ssd/CMakeLists.txt b/inference-engine/samples/hello_reshape_ssd/CMakeLists.txt index 5be6137ab2a..ae7283c3634 100644 --- a/inference-engine/samples/hello_reshape_ssd/CMakeLists.txt +++ b/inference-engine/samples/hello_reshape_ssd/CMakeLists.txt @@ -5,5 +5,4 @@ ie_add_sample(NAME hello_reshape_ssd SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/main.cpp" HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/reshape_ssd_extension.hpp" - DEPENDENCIES ie_samples_utils - OPENCV_DEPENDENCIES core imgproc imgcodecs) + DEPENDENCIES format_reader ie_samples_utils) diff --git a/inference-engine/samples/hello_reshape_ssd/main.cpp b/inference-engine/samples/hello_reshape_ssd/main.cpp index b91c2e11186..13d30a8c50b 100644 --- a/inference-engine/samples/hello_reshape_ssd/main.cpp +++ b/inference-engine/samples/hello_reshape_ssd/main.cpp @@ -2,10 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 // +#include + #include #include #include -#include +#include #include #include @@ -54,12 +56,19 @@ int main(int argc, char* argv[]) { std::string input_name; SizeVector input_shape; std::tie(input_name, input_shape) = *input_shapes.begin(); - cv::Mat image = cv::imread(input_image_path); + FormatReader::ReaderPtr reader(input_image_path.c_str()); + if (reader.get() == nullptr) { + std::cout << "Image " + input_image_path + " cannot be read!" << std::endl; + return 1; + } + size_t image_width, image_height; + image_width = reader->width(); + image_height = reader->height(); input_shape[0] = batch_size; - input_shape[2] = static_cast(image.rows); - input_shape[3] = static_cast(image.cols); + input_shape[2] = image_height; + input_shape[3] = image_width; input_shapes[input_name] = input_shape; - std::cout << "Resizing network to the image size = [" << image.rows << "x" << image.cols << "] " + std::cout << "Resizing network to the image size = [" << image_height << "x" << image_width << "] " << "with batch = " << batch_size << std::endl; network.reshape(input_shapes); // ----------------------------------------------------------------------------------------------------- @@ -132,10 +141,51 @@ int main(int argc, char* argv[]) { // --------------------------- Step 6. Prepare input // -------------------------------------------------------- - Blob::Ptr input = infer_request.GetBlob(input_name); - for (size_t b = 0; b < batch_size; b++) { - matU8ToBlob(image, input, b); + /** Collect images data ptrs **/ + std::shared_ptr image_data, original_image_data; + /** Store image data **/ + std::shared_ptr original_data(reader->getData()); + std::shared_ptr data_reader( + reader->getData(input_info->getTensorDesc().getDims()[3], input_info->getTensorDesc().getDims()[2])); + if (data_reader.get() != nullptr) { + original_image_data = original_data; + image_data = data_reader; + } else { + throw std::logic_error("Valid input images were not found!"); } + + /** Creating input blob **/ + Blob::Ptr image_input = infer_request.GetBlob(input_name); + + /** Filling input tensor with images. First b channel, then g and r channels **/ + MemoryBlob::Ptr mimage = as(image_input); + if (!mimage) { + std::cout << "We expect image blob to be inherited from MemoryBlob, but by fact we were not able " + "to cast imageInput to MemoryBlob" + << std::endl; + return 1; + } + // locked memory holder should be alive all time while access to its buffer happens + auto minputHolder = mimage->wmap(); + + size_t num_channels = mimage->getTensorDesc().getDims()[1]; + size_t image_size = mimage->getTensorDesc().getDims()[3] * mimage->getTensorDesc().getDims()[2]; + + unsigned char* data = minputHolder.as(); + + /** Iterate over all input images **/ + for (size_t image_id = 0; image_id < batch_size; ++image_id) { + /** Iterate over all pixel in image (b,g,r) **/ + for (size_t pid = 0; pid < image_size; pid++) { + /** Iterate over all channels **/ + for (size_t ch = 0; ch < num_channels; ++ch) { + /** [images stride + channels stride + pixel id ] all in bytes **/ + data[image_id * image_size * num_channels + ch * image_size + pid] = + image_data.get()[pid * num_channels + ch]; + } + } + } + // ----------------------------------------------------------------------------------------------------- // --------------------------- Step 7. Do inference @@ -156,37 +206,51 @@ int main(int argc, char* argv[]) { auto moutputHolder = moutput->rmap(); const float* detection = moutputHolder.as(); + std::vector> boxes(batch_size); + std::vector> classes(batch_size); + /* Each detection has image_id that denotes processed image */ for (size_t cur_proposal = 0; cur_proposal < max_proposal_count; cur_proposal++) { - float image_id = detection[cur_proposal * object_size + 0]; - float label = detection[cur_proposal * object_size + 1]; - float confidence = detection[cur_proposal * object_size + 2]; - /* CPU and GPU devices have difference in DetectionOutput layer, so we - * need both checks */ - if (image_id < 0 || confidence == 0.0f) { - continue; + auto image_id = static_cast(detection[cur_proposal * object_size + 0]); + if (image_id < 0) { + break; } - float xmin = detection[cur_proposal * object_size + 3] * image.cols; - float ymin = detection[cur_proposal * object_size + 4] * image.rows; - float xmax = detection[cur_proposal * object_size + 5] * image.cols; - float ymax = detection[cur_proposal * object_size + 6] * image.rows; + float confidence = detection[cur_proposal * object_size + 2]; + auto label = static_cast(detection[cur_proposal * object_size + 1]); + auto xmin = detection[cur_proposal * object_size + 3] * image_width; + auto ymin = detection[cur_proposal * object_size + 4] * image_height; + auto xmax = detection[cur_proposal * object_size + 5] * image_width; + auto ymax = detection[cur_proposal * object_size + 6] * image_height; if (confidence > 0.5f) { /** Drawing only objects with >50% probability **/ - std::ostringstream conf; - conf << ":" << std::fixed << std::setprecision(3) << confidence; - cv::rectangle(image, cv::Point2f(xmin, ymin), cv::Point2f(xmax, ymax), cv::Scalar(0, 0, 255)); + classes[image_id].push_back(label); + boxes[image_id].push_back(static_cast(xmin)); + boxes[image_id].push_back(static_cast(ymin)); + boxes[image_id].push_back(static_cast(xmax - xmin)); + boxes[image_id].push_back(static_cast(ymax - ymin)); + std::cout << "[" << cur_proposal << "," << label << "] element, prob = " << confidence << ", bbox = (" << xmin << "," << ymin << ")-(" << xmax << "," << ymax << ")" << ", batch id = " << image_id << std::endl; } } - cv::imwrite("hello_reshape_ssd_output.jpg", image); - std::cout << "The resulting image was saved in the file: " - "hello_reshape_ssd_output.jpg" - << std::endl; + for (size_t batch_id = 0; batch_id < batch_size; ++batch_id) { + addRectangles(original_image_data.get(), + image_height, + image_width, + boxes[batch_id], + classes[batch_id], + BBOX_THICKNESS); + const std::string image_path = "hello_reshape_ssd_output.bmp"; + if (writeOutputBmp(image_path, original_image_data.get(), image_height, image_width)) { + std::cout << "The resulting image was saved in the file: " + image_path << std::endl; + } else { + throw std::logic_error(std::string("Can't create a file: ") + image_path); + } + } // ----------------------------------------------------------------------------------------------------- } catch (const std::exception& ex) { std::cerr << ex.what() << std::endl; diff --git a/inference-engine/samples/hello_reshape_ssd/reshape_ssd_extension.hpp b/inference-engine/samples/hello_reshape_ssd/reshape_ssd_extension.hpp index 1b1d6c934ff..fdc4837e31b 100644 --- a/inference-engine/samples/hello_reshape_ssd/reshape_ssd_extension.hpp +++ b/inference-engine/samples/hello_reshape_ssd/reshape_ssd_extension.hpp @@ -12,6 +12,9 @@ #define CUSTOM_RELU_TYPE "CustomReLU" +/* thickness of a line (in pixels) to be used for bounding boxes */ +#define BBOX_THICKNESS 2 + class CustomReLUImpl : public InferenceEngine::ILayerExecImpl { public: explicit CustomReLUImpl(const std::shared_ptr& node) : _node(node) {}