From b241d5227eca7e3760bd30010a634cf946fb4c1f Mon Sep 17 00:00:00 2001 From: Ilya Churaev Date: Thu, 23 Dec 2021 12:45:02 +0300 Subject: [PATCH] Moved compile_tool to new API (#8501) * Moved compile_tool to new API * Fixed comments and added new tests * Fixed comments * Fixed build * Fixed comments * Fixed unit tests * Fixed compilation * Fixed legacy message * Fixed readme * Fixed comments * FIxed build * Fixed build * Fixed tests * Applied comments Co-authored-by: Ilya Lavrenov --- .../utils/include/samples/args_helper.hpp | 11 ++ samples/cpp/common/utils/src/args_helper.cpp | 183 ++++++++++++++++-- src/core/tests/preprocess.cpp | 43 ++++ tools/compile_tool/README.md | 30 +-- tools/compile_tool/main.cpp | 105 +++++++--- 5 files changed, 321 insertions(+), 51 deletions(-) diff --git a/samples/cpp/common/utils/include/samples/args_helper.hpp b/samples/cpp/common/utils/include/samples/args_helper.hpp index 2a162a9d69a..eea28cbe394 100644 --- a/samples/cpp/common/utils/include/samples/args_helper.hpp +++ b/samples/cpp/common/utils/include/samples/args_helper.hpp @@ -43,3 +43,14 @@ void processLayout(InferenceEngine::CNNNetwork& network, void printInputAndOutputsInfo(const InferenceEngine::CNNNetwork& network); void printInputAndOutputsInfo(const ov::Model& network); + +void configurePrePostProcessing(std::shared_ptr& function, + const std::string& ip, + const std::string& op, + const std::string& iop, + const std::string& il, + const std::string& ol, + const std::string& iol, + const std::string& iml, + const std::string& oml, + const std::string& ioml); diff --git a/samples/cpp/common/utils/src/args_helper.cpp b/samples/cpp/common/utils/src/args_helper.cpp index 74072891d94..50cd936662d 100644 --- a/samples/cpp/common/utils/src/args_helper.cpp +++ b/samples/cpp/common/utils/src/args_helper.cpp @@ -157,18 +157,18 @@ InferenceEngine::Precision getPrecision(std::string value, const supported_preci InferenceEngine::Precision getPrecision(const std::string& value) { static const supported_precisions_t supported_precisions = { - {"FP32", InferenceEngine::Precision::FP32}, - {"FP16", InferenceEngine::Precision::FP16}, - {"BF16", InferenceEngine::Precision::BF16}, - {"U64", InferenceEngine::Precision::U64}, - {"I64", InferenceEngine::Precision::I64}, - {"U32", InferenceEngine::Precision::U32}, - {"I32", InferenceEngine::Precision::I32}, - {"U16", InferenceEngine::Precision::U16}, - {"I16", InferenceEngine::Precision::I16}, - {"U8", InferenceEngine::Precision::U8}, - {"I8", InferenceEngine::Precision::I8}, - {"BOOL", InferenceEngine::Precision::BOOL}, + {"FP32", InferenceEngine::Precision::FP32}, {"f32", InferenceEngine::Precision::FP32}, + {"FP16", InferenceEngine::Precision::FP16}, {"f16", InferenceEngine::Precision::FP16}, + {"BF16", InferenceEngine::Precision::BF16}, {"bf16", InferenceEngine::Precision::BF16}, + {"U64", InferenceEngine::Precision::U64}, {"u64", InferenceEngine::Precision::U64}, + {"I64", InferenceEngine::Precision::I64}, {"i64", InferenceEngine::Precision::I64}, + {"U32", InferenceEngine::Precision::U32}, {"u32", InferenceEngine::Precision::U32}, + {"I32", InferenceEngine::Precision::I32}, {"i32", InferenceEngine::Precision::I32}, + {"U16", InferenceEngine::Precision::U16}, {"u16", InferenceEngine::Precision::U16}, + {"I16", InferenceEngine::Precision::I16}, {"i16", InferenceEngine::Precision::I16}, + {"U8", InferenceEngine::Precision::U8}, {"u8", InferenceEngine::Precision::U8}, + {"I8", InferenceEngine::Precision::I8}, {"i8", InferenceEngine::Precision::I8}, + {"BOOL", InferenceEngine::Precision::BOOL}, {"boolean", InferenceEngine::Precision::BOOL}, }; return getPrecision(value, supported_precisions); @@ -197,6 +197,32 @@ void setPrecisions(const InferenceEngine::CNNNetwork& network, const std::string } } +using supported_type_t = std::unordered_map; +ov::element::Type getType(std::string value, const supported_type_t& supported_precisions) { + std::transform(value.begin(), value.end(), value.begin(), ::toupper); + + const auto precision = supported_precisions.find(value); + if (precision == supported_precisions.end()) { + throw std::logic_error("\"" + value + "\"" + " is not a valid precision"); + } + + return precision->second; +} +ov::element::Type getType(const std::string& value) { + static const supported_type_t supported_types = { + {"FP32", ov::element::f32}, {"f32", ov::element::f32}, {"FP16", ov::element::f16}, + {"f16", ov::element::f16}, {"BF16", ov::element::bf16}, {"bf16", ov::element::bf16}, + {"U64", ov::element::u64}, {"u64", ov::element::u64}, {"I64", ov::element::i64}, + {"i64", ov::element::i64}, {"U32", ov::element::u32}, {"u32", ov::element::u32}, + {"I32", ov::element::i32}, {"i32", ov::element::i32}, {"U16", ov::element::u16}, + {"u16", ov::element::u16}, {"I16", ov::element::i16}, {"i16", ov::element::i16}, + {"U8", ov::element::u8}, {"u8", ov::element::u8}, {"I8", ov::element::i8}, + {"i8", ov::element::i8}, {"BOOL", ov::element::boolean}, {"boolean", ov::element::boolean}, + }; + + return getType(value, supported_types); +} + } // namespace void processPrecision(InferenceEngine::CNNNetwork& network, @@ -299,7 +325,6 @@ void setLayouts(const InferenceEngine::CNNNetwork& network, const std::string io } } } - } // namespace void processLayout(InferenceEngine::CNNNetwork& network, @@ -373,3 +398,135 @@ void printInputAndOutputsInfo(const ov::Model& network) { slog::info << " output shape: " << shape << slog::endl; } } + +void configurePrePostProcessing(std::shared_ptr& model, + const std::string& ip, + const std::string& op, + const std::string& iop, + const std::string& il, + const std::string& ol, + const std::string& iol, + const std::string& iml, + const std::string& oml, + const std::string& ioml) { + auto preprocessor = ov::preprocess::PrePostProcessor(model); + const auto inputs = model->inputs(); + const auto outputs = model->outputs(); + if (!ip.empty()) { + auto type = getType(ip); + for (size_t i = 0; i < inputs.size(); i++) { + preprocessor.input(i).tensor().set_element_type(type); + } + } + + if (!op.empty()) { + auto type = getType(op); + for (size_t i = 0; i < outputs.size(); i++) { + preprocessor.output(i).tensor().set_element_type(type); + } + } + + if (!iop.empty()) { + const auto user_precisions_map = parseArgMap(iop); + for (auto&& item : user_precisions_map) { + const auto& tensor_name = item.first; + const auto type = getType(item.second); + + bool tensorFound = false; + for (size_t i = 0; i < inputs.size(); i++) { + if (inputs[i].get_names().count(tensor_name)) { + preprocessor.input(i).tensor().set_element_type(type); + tensorFound = true; + break; + } + } + if (!tensorFound) { + for (size_t i = 0; i < outputs.size(); i++) { + if (outputs[i].get_names().count(tensor_name)) { + preprocessor.output(i).tensor().set_element_type(type); + tensorFound = true; + break; + } + } + } + OPENVINO_ASSERT(!tensorFound, "Model doesn't have input/output with tensor name: ", tensor_name); + } + } + if (!il.empty()) { + for (size_t i = 0; i < inputs.size(); i++) { + preprocessor.input(i).tensor().set_layout(ov::Layout(il)); + } + } + + if (!ol.empty()) { + for (size_t i = 0; i < outputs.size(); i++) { + preprocessor.output(i).tensor().set_layout(ov::Layout(ol)); + } + } + + if (!iol.empty()) { + const auto user_precisions_map = parseArgMap(iol); + for (auto&& item : user_precisions_map) { + const auto& tensor_name = item.first; + + bool tensorFound = false; + for (size_t i = 0; i < inputs.size(); i++) { + if (inputs[i].get_names().count(tensor_name)) { + preprocessor.input(i).tensor().set_layout(ov::Layout(item.second)); + tensorFound = true; + break; + } + } + if (!tensorFound) { + for (size_t i = 0; i < outputs.size(); i++) { + if (outputs[i].get_names().count(tensor_name)) { + preprocessor.output(i).tensor().set_layout(ov::Layout(item.second)); + tensorFound = true; + break; + } + } + } + OPENVINO_ASSERT(!tensorFound, "Model doesn't have input/output with tensor name: ", tensor_name); + } + } + + if (!iml.empty()) { + for (size_t i = 0; i < inputs.size(); i++) { + preprocessor.input(i).model().set_layout(ov::Layout(iml)); + } + } + + if (!oml.empty()) { + for (size_t i = 0; i < outputs.size(); i++) { + preprocessor.output(i).model().set_layout(ov::Layout(oml)); + } + } + + if (!ioml.empty()) { + const auto user_precisions_map = parseArgMap(ioml); + for (auto&& item : user_precisions_map) { + const auto& tensor_name = item.first; + + bool tensorFound = false; + for (size_t i = 0; i < inputs.size(); i++) { + if (inputs[i].get_names().count(tensor_name)) { + preprocessor.input(i).model().set_layout(ov::Layout(item.second)); + tensorFound = true; + break; + } + } + if (!tensorFound) { + for (size_t i = 0; i < outputs.size(); i++) { + if (outputs[i].get_names().count(tensor_name)) { + preprocessor.output(i).model().set_layout(ov::Layout(item.second)); + tensorFound = true; + break; + } + } + } + OPENVINO_ASSERT(!tensorFound, "Model doesn't have input/output with tensor name: ", tensor_name); + } + } + + model = preprocessor.build(); +} diff --git a/src/core/tests/preprocess.cpp b/src/core/tests/preprocess.cpp index 5a9fa25d9bb..d1f8c3a5356 100644 --- a/src/core/tests/preprocess.cpp +++ b/src/core/tests/preprocess.cpp @@ -80,6 +80,8 @@ TEST(pre_post_process, convert_element_type_and_scale) { TEST(pre_post_process, convert_element_type_implicit) { auto f = create_simple_function(element::i32, Shape{1, 3, 224, 224}); + EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::i32); + EXPECT_EQ(f->get_results().front()->get_element_type(), element::i32); auto p = PrePostProcessor(f); p.input().tensor().set_element_type(element::f32); f = p.build(); @@ -87,6 +89,28 @@ TEST(pre_post_process, convert_element_type_implicit) { EXPECT_EQ(f->get_results().front()->get_element_type(), element::i32); } +TEST(pre_post_process, convert_element_type_implicit_several_time) { + auto f = create_simple_function(element::i32, Shape{1, 3, 224, 224}); + EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::i32); + EXPECT_EQ(f->get_results().front()->get_element_type(), element::i32); + PrePostProcessor preprocessor(f); + preprocessor.input().tensor().set_layout(ov::Layout("NHWC")); + preprocessor.input().model().set_layout(ov::Layout("NCHW")); + preprocessor.input().tensor().set_element_type(element::f16); + preprocessor.input().tensor().set_element_type(element::i32); + preprocessor.input().tensor().set_element_type(element::u32); + preprocessor.input().tensor().set_element_type(element::f32); + preprocessor.output().tensor().set_element_type(element::f16); + preprocessor.output().tensor().set_element_type(element::i32); + preprocessor.output().tensor().set_element_type(element::u32); + preprocessor.output().tensor().set_element_type(element::f32); + preprocessor.output().tensor().set_element_type(element::u64); + f = preprocessor.build(); + EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::f32); + EXPECT_EQ(f->get_parameters().front()->get_layout().to_string(), "[N,H,W,C]"); + EXPECT_EQ(f->get_results().front()->get_element_type(), element::u64); +} + TEST(pre_post_process, convert_element_type_same) { auto f = create_simple_function(element::i32, Shape{1, 3, 224, 224}); auto old_size = f->get_ops().size(); @@ -448,6 +472,25 @@ TEST(pre_post_process, convert_color_duplicate_internal_subnames_mean) { EXPECT_NO_THROW(f = p.build()); } +TEST(pre_post_process, convert_layout_implicit_several_time) { + auto f = create_simple_function(element::i32, Shape{1, 3, 224, 224}); + EXPECT_EQ(f->get_parameters().front()->get_element_type(), element::i32); + EXPECT_EQ(f->get_results().front()->get_element_type(), element::i32); + PrePostProcessor preprocessor(f); + preprocessor.input().tensor().set_layout("NWHC"); + preprocessor.input().tensor().set_layout("CHWN"); + preprocessor.input().tensor().set_layout("NHCW"); + preprocessor.input().tensor().set_layout("NCHW"); + preprocessor.input().tensor().set_layout("NHWC"); + preprocessor.output().tensor().set_layout("NWHC"); + preprocessor.output().tensor().set_layout("CHWN"); + preprocessor.output().tensor().set_layout("NHCW"); + preprocessor.output().tensor().set_layout("NCHW"); + f = preprocessor.build(); + EXPECT_EQ(f->get_parameters().front()->get_layout().to_string(), "[N,H,W,C]"); + EXPECT_EQ(f->get_results().front()->get_layout().to_string(), "[N,C,H,W]"); +} + TEST(pre_post_process, unsupported_model_color_format) { auto f = create_simple_function(element::f32, PartialShape{1, 4, 4, 3}); EXPECT_THROW(auto p = PrePostProcessor(f); p.input().tensor().set_color_format(ColorFormat::NV12_SINGLE_PLANE); diff --git a/tools/compile_tool/README.md b/tools/compile_tool/README.md index 279425f7f77..23142f22368 100644 --- a/tools/compile_tool/README.md +++ b/tools/compile_tool/README.md @@ -21,10 +21,8 @@ Running the application with the `-h` option yields the following usage message: ```sh ./compile_tool -h -Inference Engine: - API version ............ 2.1 - Build .................. custom_vv/compile-tool_8b57af00330063c7f302aaac4d41805de21fc54a - Description ....... API +OpenVINO Runtime version ......... 2022.1.0 +Build ........... custom_changed_compile_tool_183a1adfcd7a001974fe1c5cfa21ec859b70ca2c compile_tool [OPTIONS] @@ -44,24 +42,31 @@ compile_tool [OPTIONS] Notice that quotes are required. Overwrites precision from ip and op options for specified layers. -il Optional. Specifies layout for all input layers of the network. - -ol Optional. Specifies layout for all input layers of the network. + -ol Optional. Specifies layout for all output layers of the network. -iol "" Optional. Specifies layout for input and output layers by name. Example: -iol "input:NCHW, output:NHWC". Notice that quotes are required. Overwrites layout from il and ol options for specified layers. + -iml Optional. Specifies model layout for all input layers of the network. + -oml Optional. Specifies model layout for all output layers of the network. + -ioml "" Optional. Specifies model layout for input and output tensors by name. + Example: -ionl "input:NCHW, output:NHWC". + Notice that quotes are required. + Overwrites layout from il and ol options for specified layers. + -ov_api_1_0 Optional. Compile model to legacy format for usage in Inference Engine API, + by default compiles to OV 2.0 API MYRIAD-specific options: - -VPU_NUMBER_OF_SHAVES Optional. Specifies number of shaves. + -VPU_NUMBER_OF_SHAVES Optional. Specifies number of shaves. Should be set with "VPU_NUMBER_OF_CMX_SLICES". Overwrites value from config. - -VPU_NUMBER_OF_CMX_SLICES Optional. Specifies number of CMX slices. + -VPU_NUMBER_OF_CMX_SLICES Optional. Specifies number of CMX slices. Should be set with "VPU_NUMBER_OF_SHAVES". Overwrites value from config. - -VPU_TILING_CMX_LIMIT_KB Optional. Specifies CMX limit for data tiling. + -VPU_TILING_CMX_LIMIT_KB Optional. Specifies CMX limit for data tiling. Value should be equal or greater than -1. Overwrites value from config. - ``` Running the application with the empty list of options yields an error message. @@ -75,11 +80,10 @@ For example, to compile a blob for inference on an Intel® Neural Compute Stick ### Import a Compiled Blob File to Your Application To import a blob with the network from a generated file into your application, use the -`InferenceEngine::Core::ImportNetwork` method: +`ov::runtime::Core::import_model` method: ```cpp -InferenceEngine::Core ie; +ov::runtime::Core ie; std::ifstream file{"model_name.blob"}; -InferenceEngine::ExecutableNetwork = ie.ImportNetwork(file, "MYRIAD", {}); +ov::runtime::CompiledModel compiled_model = ie.import_model(file, "MYRIAD", {}); ``` - diff --git a/tools/compile_tool/main.cpp b/tools/compile_tool/main.cpp index 81a7de0de1d..e57167ebf12 100644 --- a/tools/compile_tool/main.cpp +++ b/tools/compile_tool/main.cpp @@ -15,6 +15,7 @@ #include #include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include #include #include @@ -59,7 +60,7 @@ static constexpr char inputs_layout_message[] = "Optional. Specifies layout for all input layers of the network."; static constexpr char outputs_layout_message[] = - "Optional. Specifies layout for all input layers of the network."; + "Optional. Specifies layout for all output layers of the network."; static constexpr char iol_message[] = "Optional. Specifies layout for input and output layers by name.\n" @@ -67,6 +68,22 @@ static constexpr char iol_message[] = " Notice that quotes are required.\n" " Overwrites layout from il and ol options for specified layers."; +static constexpr char inputs_model_layout_message[] = + "Optional. Specifies model layout for all input layers of the network."; + +static constexpr char outputs_model_layout_message[] = + "Optional. Specifies model layout for all output layers of the network."; + +static constexpr char ioml_message[] = + "Optional. Specifies model layout for input and output tensors by name.\n" +" Example: -ionl \"input:NCHW, output:NHWC\".\n" +" Notice that quotes are required.\n" +" Overwrites layout from il and ol options for specified layers."; + +static constexpr char api1_message[] = + "Optional. Compile model to legacy format for usage in Inference Engine API,\n" +" by default compiles to OV 2.0 API"; + // MYRIAD-specific static constexpr char number_of_shaves_message[] = "Optional. Specifies number of shaves.\n" @@ -95,6 +112,10 @@ DEFINE_string(iop, "", iop_message); DEFINE_string(il, "", inputs_layout_message); DEFINE_string(ol, "", outputs_layout_message); DEFINE_string(iol, "", iol_message); +DEFINE_string(iml, "", inputs_model_layout_message); +DEFINE_string(oml, "", outputs_model_layout_message); +DEFINE_string(ioml, "", ioml_message); +DEFINE_bool(ov_api_1_0, false, api1_message); DEFINE_string(VPU_NUMBER_OF_SHAVES, "", number_of_shaves_message); DEFINE_string(VPU_NUMBER_OF_CMX_SLICES, "", number_of_cmx_slices_message); DEFINE_string(VPU_TILING_CMX_LIMIT_KB, "", tiling_cmx_limit_message); @@ -114,6 +135,10 @@ static void showUsage() { std::cout << " -il " << inputs_layout_message << std::endl; std::cout << " -ol " << outputs_layout_message << std::endl; std::cout << " -iol \"\" " << iol_message << std::endl; + std::cout << " -iml " << inputs_model_layout_message << std::endl; + std::cout << " -oml " << outputs_model_layout_message << std::endl; + std::cout << " -ioml \"\" " << ioml_message << std::endl; + std::cout << " -ov_api_1_0 " << api1_message << std::endl; std::cout << std::endl; std::cout << " MYRIAD-specific options: " << std::endl; std::cout << " -VPU_NUMBER_OF_SHAVES " << number_of_shaves_message << std::endl; @@ -263,41 +288,71 @@ int main(int argc, char* argv[]) { TimeDiff loadNetworkTimeElapsed {0}; try { - std::cout << "Inference Engine: " << InferenceEngine::GetInferenceEngineVersion() << std::endl; - std::cout << std::endl; + const auto& version = ov::get_openvino_version(); + std::cout << version.description << " version ......... "; + std::cout << OPENVINO_VERSION_MAJOR << "." << OPENVINO_VERSION_MINOR << "." << OPENVINO_VERSION_PATCH << std::endl; + + std::cout << "Build ........... "; + std::cout << version.buildNumber << std::endl; if (!parseCommandLine(&argc, &argv)) { return EXIT_SUCCESS; } + if (FLAGS_ov_api_1_0) { + InferenceEngine::Core ie; + if (!FLAGS_log_level.empty()) { + ie.SetConfig({{CONFIG_KEY(LOG_LEVEL), FLAGS_log_level}}, FLAGS_d); + } - InferenceEngine::Core ie; - if (!FLAGS_log_level.empty()) { - ie.SetConfig({{CONFIG_KEY(LOG_LEVEL), FLAGS_log_level}}, FLAGS_d); - } + auto network = ie.ReadNetwork(FLAGS_m); - auto network = ie.ReadNetwork(FLAGS_m); + setDefaultIO(network); + processPrecision(network, FLAGS_ip, FLAGS_op, FLAGS_iop); + processLayout(network, FLAGS_il, FLAGS_ol, FLAGS_iol); - setDefaultIO(network); - processPrecision(network, FLAGS_ip, FLAGS_op, FLAGS_iop); - processLayout(network, FLAGS_il, FLAGS_ol, FLAGS_iol); + printInputAndOutputsInfo(network); - printInputAndOutputsInfo(network); + auto timeBeforeLoadNetwork = std::chrono::steady_clock::now(); + auto executableNetwork = ie.LoadNetwork(network, FLAGS_d, configure()); + loadNetworkTimeElapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - timeBeforeLoadNetwork); - auto timeBeforeLoadNetwork = std::chrono::steady_clock::now(); - auto executableNetwork = ie.LoadNetwork(network, FLAGS_d, configure()); - loadNetworkTimeElapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - timeBeforeLoadNetwork); + std::string outputName = FLAGS_o; + if (outputName.empty()) { + outputName = getFileNameFromPath(fileNameNoExt(FLAGS_m)) + ".blob"; + } - std::string outputName = FLAGS_o; - if (outputName.empty()) { - outputName = getFileNameFromPath(fileNameNoExt(FLAGS_m)) + ".blob"; - } - - std::ofstream outputFile{outputName, std::ios::out | std::ios::binary}; - if (!outputFile.is_open()) { - std::cout << "Output file " << outputName << " can't be opened for writing" << std::endl; - return EXIT_FAILURE; + std::ofstream outputFile{outputName, std::ios::out | std::ios::binary}; + if (!outputFile.is_open()) { + std::cout << "Output file " << outputName << " can't be opened for writing" << std::endl; + return EXIT_FAILURE; + } else { + executableNetwork.Export(outputFile); + } } else { - executableNetwork.Export(outputFile); + ov::runtime::Core core; + if (!FLAGS_log_level.empty()) { + core.set_config({{CONFIG_KEY(LOG_LEVEL), FLAGS_log_level}}, FLAGS_d); + } + + auto model = core.read_model(FLAGS_m); + + configurePrePostProcessing(model, FLAGS_ip, FLAGS_op, FLAGS_iop, FLAGS_il, FLAGS_ol, FLAGS_iol, FLAGS_iml, FLAGS_oml, FLAGS_ioml); + printInputAndOutputsInfo(*model); + auto timeBeforeLoadNetwork = std::chrono::steady_clock::now(); + auto compiledModel = core.compile_model(model, FLAGS_d, configure()); + loadNetworkTimeElapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - timeBeforeLoadNetwork); + std::string outputName = FLAGS_o; + if (outputName.empty()) { + outputName = getFileNameFromPath(fileNameNoExt(FLAGS_m)) + ".blob"; + } + + std::ofstream outputFile{outputName, std::ios::out | std::ios::binary}; + if (!outputFile.is_open()) { + std::cout << "Output file " << outputName << " can't be opened for writing" << std::endl; + return EXIT_FAILURE; + } else { + compiledModel.export_model(outputFile); + } } } catch (const std::exception& error) { std::cerr << error.what() << std::endl;