merge loadnetwork and importnetwork for cpu plugin (#18024)

* merget loadnetwork and importnetwork for cpu plugin

* fix testcase

* remove multithreading config saving

* modify test case

* separate function

* modify function name

* save model_prefer_threads to cache

* remove function encapsulation for config

src/plugins/intel_cpu/src/config.h

@@ -90,6 +90,8 @@ struct Config {
 
     bool isLegacyApi = false;
 
+    int modelPreferThreads = -1;
+
 #ifdef CPU_DEBUG_CAPS
     DebugCapsConfig debugCaps;
     void applyDebugCapsProperties();

src/plugins/intel_cpu/src/cpu_streams_calculation.cpp

@@ -353,25 +353,12 @@ std::vector<std::vector<int>> get_streams_info_table(const int input_streams,
 }
 
 int get_model_prefer_threads(const int num_streams,
-                             const Config::LatencyThreadingMode latency_threading_mode,
                              const std::vector<std::vector<int>> proc_type_table,
                              const std::shared_ptr<ngraph::Function>& ngraphFunc,
-                             const ov::threading::IStreamsExecutor::Config streamExecutorConfig) {
-    const int sockets = get_default_latency_streams(latency_threading_mode);
+                             Config& config) {
+    const int sockets = get_default_latency_streams(config.latencyThreadingMode);
     auto model_prefer = 0;
-    // latency
-    if (num_streams <= sockets && num_streams > 0) {
-        if (proc_type_table[0][EFFICIENT_CORE_PROC] > 0 && proc_type_table[0][MAIN_CORE_PROC] > 0) {
-            bool fp_intesive = !ov::op::util::has_op_with_type<ngraph::op::FakeQuantize>(ngraphFunc);
-            const int int8_threshold = 4;  // ~relative efficiency of the VNNI-intensive code for Big vs Little cores;
-            const int fp32_threshold = 2;  // ~relative efficiency of the AVX2 fp32 code for Big vs Little cores;
-            // by default the latency case uses (faster) Big cores only, depending on the compute ratio
-            model_prefer = proc_type_table[0][MAIN_CORE_PROC] > (proc_type_table[0][EFFICIENT_CORE_PROC] /
-                                                                 (fp_intesive ? fp32_threshold : int8_threshold))
-                               ? proc_type_table[0][MAIN_CORE_PROC]
-                               : proc_type_table[0][MAIN_CORE_PROC] + proc_type_table[0][EFFICIENT_CORE_PROC];
-        }
-    } else { // throughput
+    if (-1 == config.modelPreferThreads) {
         const auto isa = dnnl::get_effective_cpu_isa();
         float isaSpecificThreshold = 1.0f;
         switch (isa) {
@@ -397,25 +384,41 @@ int get_model_prefer_threads(const int num_streams,
         const float L2_cache_size = dnnl::utils::get_cache_size(2 /*level*/, true /*per core */);
         ov::MemBandwidthPressure networkToleranceForLowCache =
             ov::MemBandwidthPressureTolerance(ngraphFunc, L2_cache_size, memThresholdAssumeLimitedForISA);
-        model_prefer = ov::threading::IStreamsExecutor::Config::StreamMode::DEFAULT;
+        config.modelPreferThreads = ov::threading::IStreamsExecutor::Config::StreamMode::DEFAULT;
         if (networkToleranceForLowCache.max_mem_tolerance == ov::MemBandwidthPressure::UNKNOWN) {
             if ((networkToleranceForLowCache.ratio_compute_convs == ov::MemBandwidthPressure::ALL) ||
                 (networkToleranceForLowCache.ratio_compute_deconvs == ov::MemBandwidthPressure::ALL)) {
                 // all relevant layers (convs, etc) are compute-limited, the most aggressive val for #streams
-                model_prefer = 1;
+                config.modelPreferThreads = 1;
             }  // otherwise (no recognized layers) falling back to the default value
         } else if (networkToleranceForLowCache.max_mem_tolerance > memThresholdAssumeLimitedForISA) {
             // network is below the ISA-specific threshold
-            model_prefer = 1;
+            config.modelPreferThreads = 1;
         } else if (networkToleranceForLowCache.max_mem_tolerance > ov::MemBandwidthPressure::LIMITED) {
             // network is below general threshold
-            model_prefer = 2;
+            config.modelPreferThreads = 2;
         }
-        if (model_prefer == 1 && proc_type_table[0][EFFICIENT_CORE_PROC] == 0 && sockets == 1) {
-            model_prefer = 2;
+        if (config.modelPreferThreads == 1 && proc_type_table[0][EFFICIENT_CORE_PROC] == 0 && sockets == 1) {
+            config.modelPreferThreads = 2;
         }
     }
 
+    // latency
+    if (num_streams <= sockets && num_streams > 0) {
+        if (proc_type_table[0][EFFICIENT_CORE_PROC] > 0 && proc_type_table[0][MAIN_CORE_PROC] > 0) {
+            bool fp_intesive = !ov::op::util::has_op_with_type<ngraph::op::FakeQuantize>(ngraphFunc);
+            const int int8_threshold = 4;  // ~relative efficiency of the VNNI-intensive code for Big vs Little cores;
+            const int fp32_threshold = 2;  // ~relative efficiency of the AVX2 fp32 code for Big vs Little cores;
+            // by default the latency case uses (faster) Big cores only, depending on the compute ratio
+            model_prefer = proc_type_table[0][MAIN_CORE_PROC] > (proc_type_table[0][EFFICIENT_CORE_PROC] /
+                                                                 (fp_intesive ? fp32_threshold : int8_threshold))
+                               ? proc_type_table[0][MAIN_CORE_PROC]
+                               : proc_type_table[0][MAIN_CORE_PROC] + proc_type_table[0][EFFICIENT_CORE_PROC];
+        }
+    } else {  // throughput
+        model_prefer = config.modelPreferThreads;
+    }
+
     return model_prefer;
 }
 
@@ -439,11 +442,7 @@ std::vector<std::vector<int>> generate_stream_info(const int streams,
                                                        config.latencyThreadingMode,
                                                        proc_type_table);
     if (-1 == preferred_nthreads_per_stream) {
-        model_prefer_threads = get_model_prefer_threads(streams,
-                                                        config.latencyThreadingMode,
-                                                        proc_type_table,
-                                                        ngraphFunc,
-                                                        executor_config);
+        model_prefer_threads = get_model_prefer_threads(streams, proc_type_table, ngraphFunc, config);
     }
 
     executor_config._streams_info_table = get_streams_info_table(executor_config._streams,

src/plugins/intel_cpu/src/cpu_streams_calculation.hpp

@@ -56,18 +56,16 @@ std::vector<std::vector<int>> get_streams_info_table(const int input_streams,
  * @param[in]  num_streams is target streams set by user via NUM_STREAMS or hints.
  *               - input "0" mean function generate the optimal number of streams
  *               - LATENCY hint equals 1 stream.
- * @param[in]  latency_threading_mode is the scope of candidate processors per stream for latency hint
- *               - user can select all processors per numa node, per socket, or per platform.
  * @param[in]  proc_type_table candidate processors available at this time
  *               - candidate processors have benn updated based on properties like "Ecore only" in previous function
  * @param[in]  ngraphFunc ngraph function
+ * @param[in]  config intel cpu configuration
  * @return     model_prefer_threads "0" means generating the optimal threads per stream based on platform
  */
 int get_model_prefer_threads(const int num_streams,
-                             const Config::LatencyThreadingMode latency_threading_mode,
                              const std::vector<std::vector<int>> proc_type_table,
                              const std::shared_ptr<ngraph::Function>& ngraphFunc,
-                             const ov::threading::IStreamsExecutor::Config streamExecutorConfig);
+                             Config& config);
 
 /**
  * @brief      Generate streams information according to processors type table

src/plugins/intel_cpu/src/plugin.cpp

@@ -283,11 +283,9 @@ void Engine::ApplyPerformanceHints(std::map<std::string, std::string> &config, c
 
 void Engine::GetPerformanceStreams(Config& config, const std::shared_ptr<ngraph::Function>& ngraphFunc) {
     const auto perf_hint_name = config.perfHintsConfig.ovPerfHint;
-    // save hints parameters to model rt_info
-    ov::AnyMap hints_props;
-    std::string hint_name;
     const int latency_streams = get_default_latency_streams(config.latencyThreadingMode);
     int streams;
+
     if (config.streamExecutorConfig._streams_changed) {
         streams = config.streamExecutorConfig._streams;
     } else if (perf_hint_name == CONFIG_VALUE(LATENCY)) {
@@ -298,22 +296,56 @@ void Engine::GetPerformanceStreams(Config& config, const std::shared_ptr<ngraph:
         streams = config.streamExecutorConfig._streams == 1 ? 0 : config.streamExecutorConfig._streams;
     }
 
-    const auto latency_name = std::string(CONFIG_VALUE(LATENCY)) + "_" + std::string(ov::num_streams.name());
-    const auto tput_name = std::string(CONFIG_VALUE(THROUGHPUT)) + "_" + std::string(ov::num_streams.name());
-
-#if defined(OPENVINO_ARCH_ARM) || defined(OPENVINO_ARCH_ARM64)
-    // TODO: This WA for CI checks will be removed after export/import refactor for MT 2.0 which will be track in CVS-112149
-    streams = 1;
-#endif
-
     get_num_streams(streams, ngraphFunc, config);
 
-    hints_props.insert({latency_name, std::to_string(latency_streams)});
-    hints_props.insert({tput_name, std::to_string(config.streamExecutorConfig._streams)});
-    ngraphFunc->set_rt_info(hints_props, "intel_cpu_hints_config");
     config._config[CONFIG_KEY(CPU_THROUGHPUT_STREAMS)] = std::to_string(config.streamExecutorConfig._streams);
 }
 
+void Engine::CalculateStreams(Config& conf, const std::shared_ptr<ngraph::Function>& function, bool imported) {
+    // import config props from caching model
+    if (imported && !is_cpu_map_available()) {
+        if (function->has_rt_info("intel_cpu_hints_config") && !conf.perfHintsConfig.ovPerfHint.empty()) {
+            const auto mode_name = conf.perfHintsConfig.ovPerfHint;
+            if (mode_name == CONFIG_VALUE(LATENCY) || mode_name == CONFIG_VALUE(THROUGHPUT)) {
+                const auto& hints_config = function->get_rt_info<ov::AnyMap>("intel_cpu_hints_config");
+                const auto hints_param_name = mode_name + "_" + std::string(ov::num_streams.name());
+                const auto it = hints_config.find(hints_param_name);
+                if (it != hints_config.end()) {
+                    conf.readProperties({{std::string(ov::num_streams.name()), it->second.as<std::string>()}});
+                } else {
+                    IE_THROW() << "Cache file doesn't contain precalculated number of streams for mode " << mode_name;
+                }
+            }
+        }
+    }
+
+    if (is_cpu_map_available()) {
+        const auto model_prefer_name = std::string("MODEL_PREFER_THREADS");
+        if (imported && function->has_rt_info("intel_cpu_hints_config")) {
+            // load model_prefer_threads from cache
+            int cache_model_prefer;
+            const auto& hints_config = function->get_rt_info<ov::AnyMap>("intel_cpu_hints_config");
+            const auto it_model_prefer = hints_config.find(model_prefer_name);
+            if (it_model_prefer != hints_config.end()) {
+                try {
+                    cache_model_prefer = it_model_prefer->second.as<int>();
+                } catch (const std::exception&) {
+                    OPENVINO_THROW("Cache file doesn't have valid value for " + model_prefer_name);
+                }
+
+                conf.modelPreferThreads = cache_model_prefer;
+            }
+        }
+        GetPerformanceStreams(conf, function);
+        // save model_prefer_threads to model rt_info when loading network
+        if (!imported) {
+            ov::AnyMap hints_props;
+            hints_props.insert({model_prefer_name, std::to_string(conf.modelPreferThreads)});
+            function->set_rt_info(hints_props, "intel_cpu_hints_config");
+        }
+    }
+}
+
 StreamCfg Engine::GetNumStreams(InferenceEngine::IStreamsExecutor::ThreadBindingType thread_binding_type,
                                         int stream_mode,
                                         const bool enable_hyper_thread) const {
@@ -493,9 +525,7 @@ Engine::LoadExeNetworkImpl(const InferenceEngine::CNNNetwork &network, const std
     Config conf = engConfig;
 
     conf.readProperties(config);
-    if (is_cpu_map_available()) {
-        GetPerformanceStreams(conf, nGraphFunc);
-    }
+    CalculateStreams(conf, nGraphFunc);
 
     // SSE runtime check is needed for some ATOM machine, which is x86-64 but w/o SSE
     static Xbyak::util::Cpu cpu;
@@ -786,25 +816,9 @@ InferenceEngine::IExecutableNetworkInternal::Ptr Engine::ImportNetwork(std::istr
     Config conf = engConfig;
     conf.readProperties(config);
 
-    // import config props from caching model
     auto function = cnnnetwork.getFunction();
-    if (function->has_rt_info("intel_cpu_hints_config") && !conf.perfHintsConfig.ovPerfHint.empty()) {
-        const auto mode_name = conf.perfHintsConfig.ovPerfHint;
-        if (mode_name == CONFIG_VALUE(LATENCY) || mode_name == CONFIG_VALUE(THROUGHPUT)) {
-            const auto& hints_config = function->get_rt_info<ov::AnyMap>("intel_cpu_hints_config");
-            const auto hints_param_name = mode_name + "_" + std::string(ov::num_streams.name());
-            const auto it = hints_config.find(hints_param_name);
-            if (it != hints_config.end()) {
-                conf.readProperties({{std::string(ov::num_streams.name()), it->second.as<std::string>()}});
-            } else {
-                IE_THROW() << "Cache file doesn't contain precalculated number of streams for mode " << mode_name;
-            }
-        }
-    }
 
-    if (is_cpu_map_available()) {
-        get_num_streams(conf.streamExecutorConfig._streams, function, conf);
-    }
+    CalculateStreams(conf, function, true);
 
     auto execNetwork = std::make_shared<ExecNetwork>(cnnnetwork, conf, extensionManager, shared_from_this());
 

src/plugins/intel_cpu/src/plugin.h

@@ -49,6 +49,8 @@ private:
 
     void GetPerformanceStreams(Config &config, const std::shared_ptr<ngraph::Function>& ngraphFunc);
 
+    void CalculateStreams(Config& conf, const std::shared_ptr<ngraph::Function>& ngraphFunc, bool imported = false);
+
     StreamCfg GetNumStreams(InferenceEngine::IStreamsExecutor::ThreadBindingType thread_binding_type,
                             int stream_mode,
                             const bool enable_hyper_thread = true) const;

src/plugins/intel_cpu/tests/functional/behavior/export_import.cpp

@@ -15,7 +15,9 @@
 
 namespace {
 
-class ExportOptimalNumStreams : public ::testing::TestWithParam<std::string> {};
+using PropertiesParams = std::tuple<std::string, std::vector<ov::AnyMap>>;
+
+class ExportOptimalNumStreams : public ::testing::TestWithParam<PropertiesParams> {};
 
 std::shared_ptr<ov::Model> MakeMatMulModel() {
     const ov::Shape input_shape = {1, 4096};
@@ -35,34 +37,102 @@ std::shared_ptr<ov::Model> MakeMatMulModel() {
 
 TEST_P(ExportOptimalNumStreams, OptimalNumStreams) {
     auto original_model = MakeMatMulModel();
-    std::string deviceName = GetParam();
     ov::Core core;
-    auto tput_mode = ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT);
-    auto latency_mode = ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY);
+    std::string device_name;
+    std::vector<ov::AnyMap> properties;
+    std::tie(device_name, properties) = GetParam();
+    auto original_properties_input = properties[0];
+    auto new_properties_input = properties[1];
 
-    auto original_tp_network = core.compile_model(original_model, deviceName, tput_mode);
-    auto original_latency_network = core.compile_model(original_model, deviceName, latency_mode);
+    auto GetProperties = [&](ov::CompiledModel& network) {
+        std::vector<std::string> properties;
+        properties.push_back(network.get_property(ov::hint::performance_mode.name()).as<std::string>());
+        properties.push_back(network.get_property(ov::num_streams.name()).as<std::string>());
+        properties.push_back(network.get_property(ov::hint::scheduling_core_type.name()).as<std::string>());
+        properties.push_back(network.get_property(ov::hint::enable_hyper_threading.name()).as<std::string>());
+        properties.push_back(network.get_property(ov::hint::enable_cpu_pinning.name()).as<std::string>());
+        properties.push_back(network.get_property(ov::inference_num_threads.name()).as<std::string>());
+        properties.push_back(network.get_property(ov::hint::num_requests.name()).as<std::string>());
+        return properties;
+    };
 
-    auto nstreams_tp_original = original_tp_network.get_property(ov::num_streams.name()).as<std::string>();
-    auto nstreams_latency_original = original_latency_network.get_property(ov::num_streams.name()).as<std::string>();
+    auto original_network = core.compile_model(original_model, device_name, original_properties_input);
+    auto original_properties_output = GetProperties(original_network);
 
-    std::stringstream exported_stream;
-    original_tp_network.export_model(exported_stream);
+    auto new_network = core.compile_model(original_model, device_name, new_properties_input);
+    auto new_properties_output = GetProperties(new_network);
+
+    std::stringstream exported_model;
+    original_network.export_model(exported_model);
+
+    // import_model with original config can create the same multi_thread setting as compile_model
     {
-        std::stringstream ss(exported_stream.str());
-        auto imported_tp_network = core.import_model(ss, deviceName, tput_mode);
-        auto nstreams_tp_imported = imported_tp_network.get_property(ov::num_streams.name()).as<std::string>();
-        EXPECT_EQ(nstreams_tp_original, nstreams_tp_imported);
+        std::stringstream ss(exported_model.str());
+        auto imported_network = core.import_model(ss, device_name, original_properties_input);
+        auto imported_properties_output = GetProperties(imported_network);
+
+        EXPECT_EQ(original_properties_output[0], imported_properties_output[0]);
+        EXPECT_EQ(original_properties_output[1], imported_properties_output[1]);
+        EXPECT_EQ(original_properties_output[2], imported_properties_output[2]);
+        EXPECT_EQ(original_properties_output[3], imported_properties_output[3]);
+        EXPECT_EQ(original_properties_output[4], imported_properties_output[4]);
+        EXPECT_EQ(original_properties_output[5], imported_properties_output[5]);
+        EXPECT_EQ(original_properties_output[6], imported_properties_output[6]);
     }
 
+    // import_model with new properties can create the same multi_thread setting as compile_model with new properties
     {
-        std::stringstream ss(exported_stream.str());
-        auto imported_latency_network = core.import_model(ss, deviceName, latency_mode);
-        auto nstreams_latency_imported = imported_latency_network.get_property(ov::num_streams.name()).as<std::string>();
-        EXPECT_EQ(nstreams_latency_original, nstreams_latency_imported);
+        std::stringstream ss(exported_model.str());
+        auto imported_network = core.import_model(ss, device_name, new_properties_input);
+        auto imported_properties_output = GetProperties(imported_network);
+
+        EXPECT_EQ(new_properties_output[0], imported_properties_output[0]);
+        EXPECT_EQ(new_properties_output[1], imported_properties_output[1]);
+        EXPECT_EQ(new_properties_output[2], imported_properties_output[2]);
+        EXPECT_EQ(new_properties_output[3], imported_properties_output[3]);
+        EXPECT_EQ(new_properties_output[4], imported_properties_output[4]);
+        EXPECT_EQ(new_properties_output[5], imported_properties_output[5]);
+        EXPECT_EQ(new_properties_output[6], imported_properties_output[6]);
     }
 }
 
-INSTANTIATE_TEST_CASE_P(smoke_ExportImportTest, ExportOptimalNumStreams, ::testing::Values(std::string("CPU")));
+const std::vector<ov::AnyMap> testing_property_for_streams = {{ov::num_streams(1)}, {ov::num_streams(2)}};
+
+const std::vector<ov::AnyMap> testing_property_for_threads = {{ov::inference_num_threads(1)},
+                                                              {ov::inference_num_threads(4)}};
+
+const std::vector<ov::AnyMap> testing_property_for_performance_mode = {
+    {ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)},
+    {ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)}};
+
+const std::vector<ov::AnyMap> testing_property_for_scheduling_core_type_1 = {
+    {ov::hint::scheduling_core_type(ov::hint::SchedulingCoreType::ANY_CORE)},
+    {ov::hint::scheduling_core_type(ov::hint::SchedulingCoreType::PCORE_ONLY)}};
+
+const std::vector<ov::AnyMap> testing_property_for_scheduling_core_type_2 = {
+    {ov::hint::scheduling_core_type(ov::hint::SchedulingCoreType::PCORE_ONLY)},
+    {ov::hint::scheduling_core_type(ov::hint::SchedulingCoreType::ECORE_ONLY)}};
+
+const std::vector<ov::AnyMap> testing_property_for_scheduling_core_type_3 = {
+    {ov::hint::scheduling_core_type(ov::hint::SchedulingCoreType::ANY_CORE)},
+    {ov::hint::scheduling_core_type(ov::hint::SchedulingCoreType::ECORE_ONLY)}};
+
+const std::vector<ov::AnyMap> testing_property_for_enable_hyper_threading = {{ov::hint::enable_hyper_threading(true)},
+                                                                             {ov::hint::enable_hyper_threading(false)}};
+
+const std::vector<ov::AnyMap> testing_property_for_enable_cpu_pinning = {{ov::hint::enable_cpu_pinning(true)},
+                                                                         {ov::hint::enable_cpu_pinning(false)}};
+
+INSTANTIATE_TEST_CASE_P(smoke_ExportImportTest,
+                        ExportOptimalNumStreams,
+                        ::testing::Combine(::testing::Values(std::string("CPU")),
+                                           ::testing::Values(testing_property_for_streams,
+                                                             testing_property_for_threads,
+                                                             testing_property_for_performance_mode,
+                                                             testing_property_for_scheduling_core_type_1,
+                                                             testing_property_for_scheduling_core_type_2,
+                                                             testing_property_for_scheduling_core_type_3,
+                                                             testing_property_for_enable_hyper_threading,
+                                                             testing_property_for_enable_cpu_pinning)));
 
 }  // namespace