diff --git a/src/plugins/intel_cpu/src/config.h b/src/plugins/intel_cpu/src/config.h index 4be16563c89..65237c52e20 100644 --- a/src/plugins/intel_cpu/src/config.h +++ b/src/plugins/intel_cpu/src/config.h @@ -90,6 +90,8 @@ struct Config { bool isLegacyApi = false; + int modelPreferThreads = -1; + #ifdef CPU_DEBUG_CAPS DebugCapsConfig debugCaps; void applyDebugCapsProperties(); diff --git a/src/plugins/intel_cpu/src/cpu_streams_calculation.cpp b/src/plugins/intel_cpu/src/cpu_streams_calculation.cpp index 5c958e3eb76..ff074c03d58 100644 --- a/src/plugins/intel_cpu/src/cpu_streams_calculation.cpp +++ b/src/plugins/intel_cpu/src/cpu_streams_calculation.cpp @@ -353,25 +353,12 @@ std::vector> get_streams_info_table(const int input_streams, } int get_model_prefer_threads(const int num_streams, - const Config::LatencyThreadingMode latency_threading_mode, const std::vector> proc_type_table, const std::shared_ptr& ngraphFunc, - const ov::threading::IStreamsExecutor::Config streamExecutorConfig) { - const int sockets = get_default_latency_streams(latency_threading_mode); + Config& config) { + const int sockets = get_default_latency_streams(config.latencyThreadingMode); auto model_prefer = 0; - // latency - if (num_streams <= sockets && num_streams > 0) { - if (proc_type_table[0][EFFICIENT_CORE_PROC] > 0 && proc_type_table[0][MAIN_CORE_PROC] > 0) { - bool fp_intesive = !ov::op::util::has_op_with_type(ngraphFunc); - const int int8_threshold = 4; // ~relative efficiency of the VNNI-intensive code for Big vs Little cores; - const int fp32_threshold = 2; // ~relative efficiency of the AVX2 fp32 code for Big vs Little cores; - // by default the latency case uses (faster) Big cores only, depending on the compute ratio - model_prefer = proc_type_table[0][MAIN_CORE_PROC] > (proc_type_table[0][EFFICIENT_CORE_PROC] / - (fp_intesive ? fp32_threshold : int8_threshold)) - ? proc_type_table[0][MAIN_CORE_PROC] - : proc_type_table[0][MAIN_CORE_PROC] + proc_type_table[0][EFFICIENT_CORE_PROC]; - } - } else { // throughput + if (-1 == config.modelPreferThreads) { const auto isa = dnnl::get_effective_cpu_isa(); float isaSpecificThreshold = 1.0f; switch (isa) { @@ -397,25 +384,41 @@ int get_model_prefer_threads(const int num_streams, const float L2_cache_size = dnnl::utils::get_cache_size(2 /*level*/, true /*per core */); ov::MemBandwidthPressure networkToleranceForLowCache = ov::MemBandwidthPressureTolerance(ngraphFunc, L2_cache_size, memThresholdAssumeLimitedForISA); - model_prefer = ov::threading::IStreamsExecutor::Config::StreamMode::DEFAULT; + config.modelPreferThreads = ov::threading::IStreamsExecutor::Config::StreamMode::DEFAULT; if (networkToleranceForLowCache.max_mem_tolerance == ov::MemBandwidthPressure::UNKNOWN) { if ((networkToleranceForLowCache.ratio_compute_convs == ov::MemBandwidthPressure::ALL) || (networkToleranceForLowCache.ratio_compute_deconvs == ov::MemBandwidthPressure::ALL)) { // all relevant layers (convs, etc) are compute-limited, the most aggressive val for #streams - model_prefer = 1; + config.modelPreferThreads = 1; } // otherwise (no recognized layers) falling back to the default value } else if (networkToleranceForLowCache.max_mem_tolerance > memThresholdAssumeLimitedForISA) { // network is below the ISA-specific threshold - model_prefer = 1; + config.modelPreferThreads = 1; } else if (networkToleranceForLowCache.max_mem_tolerance > ov::MemBandwidthPressure::LIMITED) { // network is below general threshold - model_prefer = 2; + config.modelPreferThreads = 2; } - if (model_prefer == 1 && proc_type_table[0][EFFICIENT_CORE_PROC] == 0 && sockets == 1) { - model_prefer = 2; + if (config.modelPreferThreads == 1 && proc_type_table[0][EFFICIENT_CORE_PROC] == 0 && sockets == 1) { + config.modelPreferThreads = 2; } } + // latency + if (num_streams <= sockets && num_streams > 0) { + if (proc_type_table[0][EFFICIENT_CORE_PROC] > 0 && proc_type_table[0][MAIN_CORE_PROC] > 0) { + bool fp_intesive = !ov::op::util::has_op_with_type(ngraphFunc); + const int int8_threshold = 4; // ~relative efficiency of the VNNI-intensive code for Big vs Little cores; + const int fp32_threshold = 2; // ~relative efficiency of the AVX2 fp32 code for Big vs Little cores; + // by default the latency case uses (faster) Big cores only, depending on the compute ratio + model_prefer = proc_type_table[0][MAIN_CORE_PROC] > (proc_type_table[0][EFFICIENT_CORE_PROC] / + (fp_intesive ? fp32_threshold : int8_threshold)) + ? proc_type_table[0][MAIN_CORE_PROC] + : proc_type_table[0][MAIN_CORE_PROC] + proc_type_table[0][EFFICIENT_CORE_PROC]; + } + } else { // throughput + model_prefer = config.modelPreferThreads; + } + return model_prefer; } @@ -439,11 +442,7 @@ std::vector> generate_stream_info(const int streams, config.latencyThreadingMode, proc_type_table); if (-1 == preferred_nthreads_per_stream) { - model_prefer_threads = get_model_prefer_threads(streams, - config.latencyThreadingMode, - proc_type_table, - ngraphFunc, - executor_config); + model_prefer_threads = get_model_prefer_threads(streams, proc_type_table, ngraphFunc, config); } executor_config._streams_info_table = get_streams_info_table(executor_config._streams, diff --git a/src/plugins/intel_cpu/src/cpu_streams_calculation.hpp b/src/plugins/intel_cpu/src/cpu_streams_calculation.hpp index 4c425bb1570..db8baba4a18 100644 --- a/src/plugins/intel_cpu/src/cpu_streams_calculation.hpp +++ b/src/plugins/intel_cpu/src/cpu_streams_calculation.hpp @@ -56,18 +56,16 @@ std::vector> get_streams_info_table(const int input_streams, * @param[in] num_streams is target streams set by user via NUM_STREAMS or hints. * - input "0" mean function generate the optimal number of streams * - LATENCY hint equals 1 stream. - * @param[in] latency_threading_mode is the scope of candidate processors per stream for latency hint - * - user can select all processors per numa node, per socket, or per platform. * @param[in] proc_type_table candidate processors available at this time * - candidate processors have benn updated based on properties like "Ecore only" in previous function * @param[in] ngraphFunc ngraph function + * @param[in] config intel cpu configuration * @return model_prefer_threads "0" means generating the optimal threads per stream based on platform */ int get_model_prefer_threads(const int num_streams, - const Config::LatencyThreadingMode latency_threading_mode, const std::vector> proc_type_table, const std::shared_ptr& ngraphFunc, - const ov::threading::IStreamsExecutor::Config streamExecutorConfig); + Config& config); /** * @brief Generate streams information according to processors type table diff --git a/src/plugins/intel_cpu/src/plugin.cpp b/src/plugins/intel_cpu/src/plugin.cpp index f303ca3eeee..7bff483554e 100644 --- a/src/plugins/intel_cpu/src/plugin.cpp +++ b/src/plugins/intel_cpu/src/plugin.cpp @@ -283,11 +283,9 @@ void Engine::ApplyPerformanceHints(std::map &config, c void Engine::GetPerformanceStreams(Config& config, const std::shared_ptr& ngraphFunc) { const auto perf_hint_name = config.perfHintsConfig.ovPerfHint; - // save hints parameters to model rt_info - ov::AnyMap hints_props; - std::string hint_name; const int latency_streams = get_default_latency_streams(config.latencyThreadingMode); int streams; + if (config.streamExecutorConfig._streams_changed) { streams = config.streamExecutorConfig._streams; } else if (perf_hint_name == CONFIG_VALUE(LATENCY)) { @@ -298,22 +296,56 @@ void Engine::GetPerformanceStreams(Config& config, const std::shared_ptrset_rt_info(hints_props, "intel_cpu_hints_config"); config._config[CONFIG_KEY(CPU_THROUGHPUT_STREAMS)] = std::to_string(config.streamExecutorConfig._streams); } +void Engine::CalculateStreams(Config& conf, const std::shared_ptr& function, bool imported) { + // import config props from caching model + if (imported && !is_cpu_map_available()) { + if (function->has_rt_info("intel_cpu_hints_config") && !conf.perfHintsConfig.ovPerfHint.empty()) { + const auto mode_name = conf.perfHintsConfig.ovPerfHint; + if (mode_name == CONFIG_VALUE(LATENCY) || mode_name == CONFIG_VALUE(THROUGHPUT)) { + const auto& hints_config = function->get_rt_info("intel_cpu_hints_config"); + const auto hints_param_name = mode_name + "_" + std::string(ov::num_streams.name()); + const auto it = hints_config.find(hints_param_name); + if (it != hints_config.end()) { + conf.readProperties({{std::string(ov::num_streams.name()), it->second.as()}}); + } else { + IE_THROW() << "Cache file doesn't contain precalculated number of streams for mode " << mode_name; + } + } + } + } + + if (is_cpu_map_available()) { + const auto model_prefer_name = std::string("MODEL_PREFER_THREADS"); + if (imported && function->has_rt_info("intel_cpu_hints_config")) { + // load model_prefer_threads from cache + int cache_model_prefer; + const auto& hints_config = function->get_rt_info("intel_cpu_hints_config"); + const auto it_model_prefer = hints_config.find(model_prefer_name); + if (it_model_prefer != hints_config.end()) { + try { + cache_model_prefer = it_model_prefer->second.as(); + } catch (const std::exception&) { + OPENVINO_THROW("Cache file doesn't have valid value for " + model_prefer_name); + } + + conf.modelPreferThreads = cache_model_prefer; + } + } + GetPerformanceStreams(conf, function); + // save model_prefer_threads to model rt_info when loading network + if (!imported) { + ov::AnyMap hints_props; + hints_props.insert({model_prefer_name, std::to_string(conf.modelPreferThreads)}); + function->set_rt_info(hints_props, "intel_cpu_hints_config"); + } + } +} + StreamCfg Engine::GetNumStreams(InferenceEngine::IStreamsExecutor::ThreadBindingType thread_binding_type, int stream_mode, const bool enable_hyper_thread) const { @@ -493,9 +525,7 @@ Engine::LoadExeNetworkImpl(const InferenceEngine::CNNNetwork &network, const std Config conf = engConfig; conf.readProperties(config); - if (is_cpu_map_available()) { - GetPerformanceStreams(conf, nGraphFunc); - } + CalculateStreams(conf, nGraphFunc); // SSE runtime check is needed for some ATOM machine, which is x86-64 but w/o SSE static Xbyak::util::Cpu cpu; @@ -786,25 +816,9 @@ InferenceEngine::IExecutableNetworkInternal::Ptr Engine::ImportNetwork(std::istr Config conf = engConfig; conf.readProperties(config); - // import config props from caching model auto function = cnnnetwork.getFunction(); - if (function->has_rt_info("intel_cpu_hints_config") && !conf.perfHintsConfig.ovPerfHint.empty()) { - const auto mode_name = conf.perfHintsConfig.ovPerfHint; - if (mode_name == CONFIG_VALUE(LATENCY) || mode_name == CONFIG_VALUE(THROUGHPUT)) { - const auto& hints_config = function->get_rt_info("intel_cpu_hints_config"); - const auto hints_param_name = mode_name + "_" + std::string(ov::num_streams.name()); - const auto it = hints_config.find(hints_param_name); - if (it != hints_config.end()) { - conf.readProperties({{std::string(ov::num_streams.name()), it->second.as()}}); - } else { - IE_THROW() << "Cache file doesn't contain precalculated number of streams for mode " << mode_name; - } - } - } - if (is_cpu_map_available()) { - get_num_streams(conf.streamExecutorConfig._streams, function, conf); - } + CalculateStreams(conf, function, true); auto execNetwork = std::make_shared(cnnnetwork, conf, extensionManager, shared_from_this()); diff --git a/src/plugins/intel_cpu/src/plugin.h b/src/plugins/intel_cpu/src/plugin.h index 5f55640cf67..218fb1c73d0 100644 --- a/src/plugins/intel_cpu/src/plugin.h +++ b/src/plugins/intel_cpu/src/plugin.h @@ -49,6 +49,8 @@ private: void GetPerformanceStreams(Config &config, const std::shared_ptr& ngraphFunc); + void CalculateStreams(Config& conf, const std::shared_ptr& ngraphFunc, bool imported = false); + StreamCfg GetNumStreams(InferenceEngine::IStreamsExecutor::ThreadBindingType thread_binding_type, int stream_mode, const bool enable_hyper_thread = true) const; diff --git a/src/plugins/intel_cpu/tests/functional/behavior/export_import.cpp b/src/plugins/intel_cpu/tests/functional/behavior/export_import.cpp index 335eb73ab30..92946bc1a0d 100644 --- a/src/plugins/intel_cpu/tests/functional/behavior/export_import.cpp +++ b/src/plugins/intel_cpu/tests/functional/behavior/export_import.cpp @@ -15,7 +15,9 @@ namespace { -class ExportOptimalNumStreams : public ::testing::TestWithParam {}; +using PropertiesParams = std::tuple>; + +class ExportOptimalNumStreams : public ::testing::TestWithParam {}; std::shared_ptr MakeMatMulModel() { const ov::Shape input_shape = {1, 4096}; @@ -35,34 +37,102 @@ std::shared_ptr MakeMatMulModel() { TEST_P(ExportOptimalNumStreams, OptimalNumStreams) { auto original_model = MakeMatMulModel(); - std::string deviceName = GetParam(); ov::Core core; - auto tput_mode = ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT); - auto latency_mode = ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY); + std::string device_name; + std::vector properties; + std::tie(device_name, properties) = GetParam(); + auto original_properties_input = properties[0]; + auto new_properties_input = properties[1]; - auto original_tp_network = core.compile_model(original_model, deviceName, tput_mode); - auto original_latency_network = core.compile_model(original_model, deviceName, latency_mode); + auto GetProperties = [&](ov::CompiledModel& network) { + std::vector properties; + properties.push_back(network.get_property(ov::hint::performance_mode.name()).as()); + properties.push_back(network.get_property(ov::num_streams.name()).as()); + properties.push_back(network.get_property(ov::hint::scheduling_core_type.name()).as()); + properties.push_back(network.get_property(ov::hint::enable_hyper_threading.name()).as()); + properties.push_back(network.get_property(ov::hint::enable_cpu_pinning.name()).as()); + properties.push_back(network.get_property(ov::inference_num_threads.name()).as()); + properties.push_back(network.get_property(ov::hint::num_requests.name()).as()); + return properties; + }; - auto nstreams_tp_original = original_tp_network.get_property(ov::num_streams.name()).as(); - auto nstreams_latency_original = original_latency_network.get_property(ov::num_streams.name()).as(); + auto original_network = core.compile_model(original_model, device_name, original_properties_input); + auto original_properties_output = GetProperties(original_network); - std::stringstream exported_stream; - original_tp_network.export_model(exported_stream); + auto new_network = core.compile_model(original_model, device_name, new_properties_input); + auto new_properties_output = GetProperties(new_network); + + std::stringstream exported_model; + original_network.export_model(exported_model); + + // import_model with original config can create the same multi_thread setting as compile_model { - std::stringstream ss(exported_stream.str()); - auto imported_tp_network = core.import_model(ss, deviceName, tput_mode); - auto nstreams_tp_imported = imported_tp_network.get_property(ov::num_streams.name()).as(); - EXPECT_EQ(nstreams_tp_original, nstreams_tp_imported); + std::stringstream ss(exported_model.str()); + auto imported_network = core.import_model(ss, device_name, original_properties_input); + auto imported_properties_output = GetProperties(imported_network); + + EXPECT_EQ(original_properties_output[0], imported_properties_output[0]); + EXPECT_EQ(original_properties_output[1], imported_properties_output[1]); + EXPECT_EQ(original_properties_output[2], imported_properties_output[2]); + EXPECT_EQ(original_properties_output[3], imported_properties_output[3]); + EXPECT_EQ(original_properties_output[4], imported_properties_output[4]); + EXPECT_EQ(original_properties_output[5], imported_properties_output[5]); + EXPECT_EQ(original_properties_output[6], imported_properties_output[6]); } + // import_model with new properties can create the same multi_thread setting as compile_model with new properties { - std::stringstream ss(exported_stream.str()); - auto imported_latency_network = core.import_model(ss, deviceName, latency_mode); - auto nstreams_latency_imported = imported_latency_network.get_property(ov::num_streams.name()).as(); - EXPECT_EQ(nstreams_latency_original, nstreams_latency_imported); + std::stringstream ss(exported_model.str()); + auto imported_network = core.import_model(ss, device_name, new_properties_input); + auto imported_properties_output = GetProperties(imported_network); + + EXPECT_EQ(new_properties_output[0], imported_properties_output[0]); + EXPECT_EQ(new_properties_output[1], imported_properties_output[1]); + EXPECT_EQ(new_properties_output[2], imported_properties_output[2]); + EXPECT_EQ(new_properties_output[3], imported_properties_output[3]); + EXPECT_EQ(new_properties_output[4], imported_properties_output[4]); + EXPECT_EQ(new_properties_output[5], imported_properties_output[5]); + EXPECT_EQ(new_properties_output[6], imported_properties_output[6]); } } -INSTANTIATE_TEST_CASE_P(smoke_ExportImportTest, ExportOptimalNumStreams, ::testing::Values(std::string("CPU"))); +const std::vector testing_property_for_streams = {{ov::num_streams(1)}, {ov::num_streams(2)}}; + +const std::vector testing_property_for_threads = {{ov::inference_num_threads(1)}, + {ov::inference_num_threads(4)}}; + +const std::vector testing_property_for_performance_mode = { + {ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)}, + {ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)}}; + +const std::vector testing_property_for_scheduling_core_type_1 = { + {ov::hint::scheduling_core_type(ov::hint::SchedulingCoreType::ANY_CORE)}, + {ov::hint::scheduling_core_type(ov::hint::SchedulingCoreType::PCORE_ONLY)}}; + +const std::vector testing_property_for_scheduling_core_type_2 = { + {ov::hint::scheduling_core_type(ov::hint::SchedulingCoreType::PCORE_ONLY)}, + {ov::hint::scheduling_core_type(ov::hint::SchedulingCoreType::ECORE_ONLY)}}; + +const std::vector testing_property_for_scheduling_core_type_3 = { + {ov::hint::scheduling_core_type(ov::hint::SchedulingCoreType::ANY_CORE)}, + {ov::hint::scheduling_core_type(ov::hint::SchedulingCoreType::ECORE_ONLY)}}; + +const std::vector testing_property_for_enable_hyper_threading = {{ov::hint::enable_hyper_threading(true)}, + {ov::hint::enable_hyper_threading(false)}}; + +const std::vector testing_property_for_enable_cpu_pinning = {{ov::hint::enable_cpu_pinning(true)}, + {ov::hint::enable_cpu_pinning(false)}}; + +INSTANTIATE_TEST_CASE_P(smoke_ExportImportTest, + ExportOptimalNumStreams, + ::testing::Combine(::testing::Values(std::string("CPU")), + ::testing::Values(testing_property_for_streams, + testing_property_for_threads, + testing_property_for_performance_mode, + testing_property_for_scheduling_core_type_1, + testing_property_for_scheduling_core_type_2, + testing_property_for_scheduling_core_type_3, + testing_property_for_enable_hyper_threading, + testing_property_for_enable_cpu_pinning))); } // namespace