[DOC] Add multi threading for 2023.0 release in CPU plugin document (#17638)

docs/OV_Runtime_UG/supported_plugins/CPU.md

@@ -323,6 +323,9 @@ All parameters must be set before calling ``ov::Core::compile_model()`` in order
 - ``ov::hint::performance_mode``
 - ``ov::hint::execution_mode``
 - ``ov::hint::num_request``
+- ``ov::hint::scheduling_core_type``
+- ``ov::hint::enable_hyper_threading``
+- ``ov::hint::enable_cpu_pinning``
 - ``ov::num_streams``
 - ``ov::affinity``
 - ``ov::inference_num_threads``
@@ -350,6 +353,56 @@ For some performance-critical DL operations, the CPU plugin uses third-party lib
 Optimization guide
 ###########################################################
 
+Multi-Threading Optimization
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+CPU inference will infer an input or multiple inputs in parallel on multiple logical processors. 
+
+User can use the following properties to limit available CPU resource for model inference. If the platform or operating system can support this behavior, OpenVINO Runtime will perform multi-threading scheduling based on limited available CPU resources.
+
+- ``ov::inference_num_threads`` limits number of logical processors used for CPU inference. 
+  If the number set by the user is greater than the number of logical processors on the platform, multi-threading scheduler only uses the platform number for CPU inference.
+- ``ov::hint::scheduling_core_type`` limits the type of CPU cores for CPU inference when user runs inference on a hybird platform that includes both Performance-cores (P-cores) with Efficient-cores (E-cores). 
+  If user platform only has one type of CPU cores, this property has no effect, and CPU inference always uses this unique core type.
+- ``ov::hint::enable_hyper_threading`` limits the use of one or two logical processors per CPU core when platform has CPU hyperthreading enabled.
+  If there is only one logical processor per CPU core, such as Efficient-cores, this property has no effect, and CPU inference uses all logical processors.
+
+.. tab:: C++
+
+   .. doxygensnippet:: docs/snippets/cpu/multi_threading.cpp
+      :language: cpp
+      :fragment: [ov:intel_cpu:multi_threading:part0]
+
+.. tab:: Python
+
+   .. doxygensnippet:: docs/snippets/cpu/multi_threading.py
+      :language: python
+      :fragment: [ov:intel_cpu:multi_threading:part0]
+
+.. note:: 
+   
+   ``ov::hint::scheduling_core_type`` and ``ov::hint::enable_hyper_threading`` only support Intel® x86-64 CPU on Linux and Windows in current release.
+   
+By default, OpenVINO Runtime will enable CPU threads pinning for better performance. User also can use property ``ov::hint::enable_cpu_pinning`` to switch it off. Disable threads pinning might be benefitial in complex applications with several workloads executed in parallel.
+
+.. tab:: C++
+
+   .. doxygensnippet:: docs/snippets/cpu/multi_threading.cpp
+      :language: cpp
+      :fragment: [ov:intel_cpu:multi_threading:part1]
+
+.. tab:: Python
+
+   .. doxygensnippet:: docs/snippets/cpu/multi_threading.py
+      :language: python
+      :fragment: [ov:intel_cpu:multi_threading:part1]
+
+user can check the :doc:`optimization guide <openvino_docs_deployment_optimization_guide_tput_advanced>` for details on multi-stream execution
+
+.. note:: 
+   
+   ``ov::hint::enable_cpu_pinning`` only support Linux in current release.
+   
 Denormals Optimization
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
@@ -444,7 +497,6 @@ from perf counters log. The "exec type" field will contain the implementation ty
 
    MatMul_1800         EXECUTED         layerType: FullyConnected         execType: brgemm_avx512_amx_sparse_I8 realTime (ms): 0.050000  cpuTime (ms): 0.050000
 
-
 Limitations
 -----------------------------------------------------------
 

docs/snippets/cpu/multi_threading.cpp

@@ -0,0 +1,37 @@
+// Copyright (C) 2023 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+#include <openvino/runtime/core.hpp>
+
+#include "openvino/runtime/intel_cpu/properties.hpp"
+
+int main() {
+    try {
+        std::string modelPath = "model.xml";
+        std::string device = "CPU";
+        ov::AnyMap config;
+        ov::Core core;
+        core.set_property(ov::inference_num_threads(1));
+        auto model = core.read_model(modelPath);
+        //! [ov:intel_cpu:multi_threading:part0]
+        // Use one logical processor for inference
+        auto compiled_model_1 = core.compile_model(model, device, ov::inference_num_threads(1));
+
+        // Use logical processors of Efficient-cores for inference on hybrid platform
+        auto compiled_model_2 = core.compile_model(model, device, ov::hint::scheduling_core_type(ECORE_ONLY));
+
+        // Use one logical processor per CPU core for inference when hyper threading is on
+        auto compiled_model_3 = core.compile_model(model, device, ov::hint::enable_hyper_threading(false));
+        //! [ov:intel_cpu:multi_threading:part0]
+
+        //! [ov:intel_cpu:multi_threading:part1]
+        // Disable CPU threads pinning for inference when system support it
+        auto compiled_model_4 = core.compile_model(model, device, ov::hint::enable_cpu_pinning(false));
+        //! [ov:intel_cpu:multi_threading:part1]
+        if ((!compiled_model_1) || (!compiled_model_2) || (!compiled_model_3) || (!compiled_model_4)) {
+            throw std::runtime_error("error");
+        }
+    } catch (...) {
+    }
+    return 0;
+}

docs/snippets/cpu/multi_threading.py

@@ -0,0 +1,31 @@
+# Copyright (C) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+import openvino.runtime as ov
+from openvino.runtime import Core, Type, OVAny, properties
+
+device_name = 'CPU'
+xml_path = 'model.xml'
+core = ov.Core()
+core.set_property("CPU", ov.properties.intel_cpu.sparse_weights_decompression_rate(0.8))
+model = core.read_model(model=xml_path)
+# ! [ov:intel_cpu:multi_threading:part0]
+# Use one logical processor for inference
+compiled_model_1 = core.compile_model(model=model, device_name=device_name, config={properties.inference_num_threads(1)})
+
+# Use logical processors of Efficient-cores for inference on hybrid platform
+compiled_model_2 = core.compile_model(model=model, device_name=device_name, config={properties.hint.scheduling_core_type(properties.hint.SchedulingCoreType.ECORE_ONLY)})
+
+# Use one logical processor per CPU core for inference when hyper threading is on
+compiled_model_3 = core.compile_model(model=model, device_name=device_name, config={properties.hint.enable_hyper_threading(False)})
+# ! [ov:intel_cpu:multi_threading:part0]
+
+# ! [ov:intel_cpu:multi_threading:part1]
+# Disable CPU threads pinning for inference when system supoprt it
+compiled_model_4 = core.compile_model(model=model, device_name=device_name, config={properties.hint.enable_cpu_pinning(False)})
+# ! [ov:intel_cpu:multi_threading:part1]
+assert compiled_model_1
+assert compiled_model_2
+assert compiled_model_3
+assert compiled_model_4