IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rebasing the perf-modes-2021.3 to the 2021.4

Browse Source 
Caveats:
the (explicit) setting #streams is not disabled (as it was before for experiments with DLBenchmark), and the logic slighlty differ (streamsSet)
This commit is contained in:
myshevts
2021-07-01 11:37:08 +03:00
parent 0361fc8e2d
commit 1ae1edc0ed
9 changed files with 403 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
inference-engine/include/ie_plugin_config.hpp
+7
View File
@@ -223,6 +223,13 @@ namespace PluginConfigParams {
#define CONFIG_VALUE(name) InferenceEngine::PluginConfigParams::name
#define DECLARE_CONFIG_VALUE(name) static constexpr auto name = #name
/**
* @brief High-level OpenVINO Performance Modes/Presets
*/
DECLARE_CONFIG_KEY(OV_PERFORMANCE_MODE);
DECLARE_CONFIG_VALUE(LATENCY);
DECLARE_CONFIG_VALUE(THROUGHPUT);
/**
* @brief generic boolean values
*/
inference-engine/samples/benchmark_app/benchmark_app.hpp
+7 -1
View File
@@ -20,8 +20,11 @@ static const char input_message[] = "Optional. Path to a folder with images and/
static const char model_message[] = "Required. Path to an .xml/.onnx/.prototxt file with a trained model or to a .blob files with "
"a trained compiled model.";
/// @brief message for execution performance mode
static const char mode_message[] = "Optional. Selects OpenVINO Performance Mode/Preset. Default value is \"throughput (tput)\".";
/// @brief message for execution mode
static const char api_message[] = "Optional. Enable Sync/Async API. Default value is \"async\".";
static const char api_message[] = "Optional (deprecated). Enable Sync/Async API. Default value is \"async\".";
/// @brief message for assigning cnn calculation to device
static const char target_device_message[] = "Optional. Specify a target device to infer on (the list of available devices is shown below). "
@@ -157,6 +160,9 @@ DEFINE_string(i, "", input_message);
/// It is a required parameter
DEFINE_string(m, "", model_message);
/// @brief Define execution mode
DEFINE_string(mode, CONFIG_VALUE(THROUGHPUT), mode_message);
/// @brief Define execution mode
DEFINE_string(api, "async", api_message);
inference-engine/samples/benchmark_app/main.cpp
+27 -15
View File
@@ -198,6 +198,14 @@ int main(int argc, char* argv[]) {
// ----------------- 3. Setting device configuration
// -----------------------------------------------------------
next_step();
std::string ov_perf_mode;
if (FLAGS_mode == "throughput" || FLAGS_mode == "THROUGHPUT" || FLAGS_mode == "tput")
ov_perf_mode = CONFIG_VALUE(THROUGHPUT);
else if (FLAGS_mode == "latency" || FLAGS_mode == "LATENCY")
ov_perf_mode = CONFIG_VALUE(LATENCY);
else if (!FLAGS_mode.empty())
throw std::logic_error("Performance mode " + ov_perf_mode + " is not recognized!");
bool perf_counts = false;
// Update config per device according to command line parameters
@@ -206,6 +214,10 @@ int main(int argc, char* argv[]) {
config[device] = {};
std::map<std::string, std::string>& device_config = config.at(device);
// high-level performance modes
if (!ov_perf_mode.empty())
device_config[CONFIG_KEY(OV_PERFORMANCE_MODE)] = ov_perf_mode;
// Set performance counter
if (isFlagSetInCommandLine("pc")) {
// set to user defined value
@@ -224,6 +236,7 @@ int main(int argc, char* argv[]) {
}
perf_counts = (device_config.at(CONFIG_KEY(PERF_COUNT)) == CONFIG_VALUE(YES)) ? true : perf_counts;
// the rest are individual per-device settings (overriding the values set with perf modes)
auto setThroughputStreams = [&]() {
const std::string key = device + "_THROUGHPUT_STREAMS";
if (device_nstreams.count(device)) {
@@ -236,7 +249,7 @@ int main(int argc, char* argv[]) {
" or via configuration file.");
}
device_config[key] = device_nstreams.at(device);
} else if (!device_config.count(key) && (FLAGS_api == "async")) {
} else if (ov_perf_mode.empty() && !device_config.count(key) && (FLAGS_api == "async")) {
slog::warn << "-nstreams default value is determined automatically for " << device
<< " device. "
"Although the automatic selection usually provides a "
@@ -295,20 +308,6 @@ int main(int argc, char* argv[]) {
if (isFlagSetInCommandLine("nthreads"))
device_config[GNA_CONFIG_KEY(LIB_N_THREADS)] = std::to_string(FLAGS_nthreads);
} else {
std::vector<std::string> supported_config_keys = ie.GetMetric(device, METRIC_KEY(SUPPORTED_CONFIG_KEYS));
auto supported = [&](const std::string& key) {
return std::find(std::begin(supported_config_keys), std::end(supported_config_keys), key) != std::end(supported_config_keys);
};
if (supported(CONFIG_KEY(CPU_THREADS_NUM)) && isFlagSetInCommandLine("nthreads")) {
device_config[CONFIG_KEY(CPU_THREADS_NUM)] = std::to_string(FLAGS_nthreads);
}
if (supported(CONFIG_KEY(CPU_THROUGHPUT_STREAMS)) && isFlagSetInCommandLine("nstreams")) {
device_config[CONFIG_KEY(CPU_THROUGHPUT_STREAMS)] = FLAGS_nstreams;
}
if (supported(CONFIG_KEY(CPU_BIND_THREAD)) && isFlagSetInCommandLine("pin")) {
device_config[CONFIG_KEY(CPU_BIND_THREAD)] = FLAGS_pin;
}
}
}
@@ -422,6 +421,19 @@ int main(int argc, char* argv[]) {
slog::info << "Load network took " << duration_ms << " ms" << slog::endl;
if (statistics)
statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"load network time (ms)", duration_ms}});
if (!ov_perf_mode.empty()) {
std::cout << "OV_PERFORMANCE_MODE: " << ov_perf_mode << std::endl;
// output of the actual settings that the mode produces (debugging)
for (auto& device : devices) {
std::vector<std::string> supported_config_keys = ie.GetMetric(device,
METRIC_KEY(SUPPORTED_CONFIG_KEYS));
std::cout << "Device: " << device << std::endl;
for (auto cfg : supported_config_keys) {
std::cout << " {" << cfg << " , " << exeNetwork.GetConfig(cfg).as<std::string>() << " }" << std::endl;
}
}
}
} else {
next_step();
slog::info << "Skipping the step for compiled network" << slog::endl;
inference-engine/src/inference_engine/threading/ie_istreams_executor.cpp
+14 -10
View File
@@ -26,6 +26,19 @@ std::vector<std::string> IStreamsExecutor::Config::SupportedKeys() {
CONFIG_KEY_INTERNAL(CPU_THREADS_PER_STREAM),
};
}
int IStreamsExecutor::Config::GetDefaultNumStreams() {
const int sockets = static_cast<int>(getAvailableNUMANodes().size());
// bare minimum of streams (that evenly divides available number of core)
const int num_cores = sockets == 1 ? std::thread::hardware_concurrency() : getNumberOfCPUCores();
if (0 == num_cores % 4)
return std::max(4, num_cores / 4);
else if (0 == num_cores % 5)
return std::max(5, num_cores / 5);
else if (0 == num_cores % 3)
return std::max(3, num_cores / 3);
else // if user disables some cores say in BIOS, so we got weird #cores which is not easy to divide
return 1;
}
void IStreamsExecutor::Config::SetConfig(const std::string& key, const std::string& value) {
if (key == CONFIG_KEY(CPU_BIND_THREAD)) {
@@ -49,17 +62,8 @@ void IStreamsExecutor::Config::SetConfig(const std::string& key, const std::stri
if (value == CONFIG_VALUE(CPU_THROUGHPUT_NUMA)) {
_streams = static_cast<int>(getAvailableNUMANodes().size());
} else if (value == CONFIG_VALUE(CPU_THROUGHPUT_AUTO)) {
const int sockets = static_cast<int>(getAvailableNUMANodes().size());
// bare minimum of streams (that evenly divides available number of cores)
const int num_cores = sockets == 1 ? std::thread::hardware_concurrency() : getNumberOfCPUCores();
if (0 == num_cores % 4)
_streams = std::max(4, num_cores / 4);
else if (0 == num_cores % 5)
_streams = std::max(5, num_cores / 5);
else if (0 == num_cores % 3)
_streams = std::max(3, num_cores / 3);
else // if user disables some cores say in BIOS, so we got weird #cores which is not easy to divide
_streams = 1;
_streams = GetDefaultNumStreams();
} else {
int val_i;
try {
inference-engine/src/mkldnn_plugin/config.cpp
+16 -9
View File
@@ -27,13 +27,13 @@ Config::Config() {
#if (IE_THREAD == IE_THREAD_TBB || IE_THREAD == IE_THREAD_TBB_AUTO)
#if defined(__APPLE__) || defined(_WIN32)
// 'CORES' is not implemented for Win/MacOS; so the 'NUMA' is default
streamExecutorConfig._threadBindingType = InferenceEngine::IStreamsExecutor::NUMA;
#endif
streamExecutorConfig._threadBindingType = InferenceEngine::IStreamsExecutor::NUMA;
#endif
if (getAvailableCoresTypes().size() > 1 /*Hybrid CPU*/) {
streamExecutorConfig._threadBindingType = InferenceEngine::IStreamsExecutor::HYBRID_AWARE;
}
#endif
#endif
if (!with_cpu_x86_bfloat16())
enforceBF16 = false;
@@ -43,11 +43,10 @@ Config::Config() {
void Config::readProperties(const std::map<std::string, std::string> &prop) {
auto streamExecutorConfigKeys = streamExecutorConfig.SupportedKeys();
for (auto& kvp : prop) {
auto& key = kvp.first;
auto& val = kvp.second;
const auto streamExecutorConfigKeys = streamExecutorConfig.SupportedKeys();
for (const auto& kvp : prop) {
const auto& key = kvp.first;
const auto& val = kvp.second;
if (streamExecutorConfigKeys.end() !=
std::find(std::begin(streamExecutorConfigKeys), std::end(streamExecutorConfigKeys), key)) {
streamExecutorConfig.SetConfig(key, val);
@@ -109,7 +108,13 @@ void Config::readProperties(const std::map<std::string, std::string> &prop) {
IE_THROW() << "Wrong value for property key " << PluginConfigParams::KEY_ENFORCE_BF16
<< ". Expected only YES/NO";
}
} else {
} else if (key == PluginConfigParams::KEY_OV_PERFORMANCE_MODE) {
if (val == PluginConfigParams::LATENCY || val == PluginConfigParams::THROUGHPUT)
ovPerfMode = val;
else
IE_THROW() << "Wrong value for property key " << PluginConfigParams::KEY_OV_PERFORMANCE_MODE
<< ". Expected only " << PluginConfigParams::LATENCY << "/" << PluginConfigParams::THROUGHPUT;
} else {
IE_THROW(NotFound) << "Unsupported property " << key << " by CPU plugin";
}
_config.clear();
@@ -158,6 +163,8 @@ void Config::updateProperties() {
_config.insert({ PluginConfigParams::KEY_ENFORCE_BF16, PluginConfigParams::YES });
else
_config.insert({ PluginConfigParams::KEY_ENFORCE_BF16, PluginConfigParams::NO });
if (!ovPerfMode.empty())
_config.insert({ PluginConfigParams::KEY_OV_PERFORMANCE_MODE, ovPerfMode });
}
}
inference-engine/src/mkldnn_plugin/config.h
+1
View File
@@ -25,6 +25,7 @@ struct Config {
bool enableDynamicBatch = false;
std::string dumpToDot = "";
int batchLimit = 0;
std::string ovPerfMode = "";
InferenceEngine::IStreamsExecutor::Config streamExecutorConfig;
#if defined(__arm__) || defined(__aarch64__)
inference-engine/src/mkldnn_plugin/mkldnn_plugin.cpp
+311 -9
View File
@@ -11,6 +11,7 @@
#include <threading/ie_executor_manager.hpp>
#include <memory>
#include <ie_plugin_config.hpp>
#include <cpp_interfaces/interface/ie_internal_plugin_config.hpp>
#include <vector>
#include <tuple>
#include <unordered_set>
@@ -112,14 +113,14 @@ Engine::~Engine() {
ExecutorManager::getInstance()->clear("CPUCallbackExecutor");
}
static void Transformation(CNNNetwork& clonedNetwork, const Config& conf) {
static void Transformation(CNNNetwork& clonedNetwork, const bool _enableLPT) {
auto nGraphFunc = clonedNetwork.getFunction();
ngraph::pass::Manager manager;
manager.register_pass<ngraph::pass::InitNodeInfo>();
const bool useLpt =
(conf.lpTransformsMode == Config::LPTransformsMode::On) &&
_enableLPT &&
ngraph::pass::low_precision::LowPrecisionTransformer::isFunctionQuantized(nGraphFunc);
if (useLpt) {
manager.register_pass<ngraph::pass::DisableConvertConstantFoldingOnConstPath>(
@@ -369,8 +370,246 @@ static void Transformation(CNNNetwork& clonedNetwork, const Config& conf) {
ConvertToCPUSpecificOpset(nGraphFunc);
}
typedef std::chrono::high_resolution_clock Time;
typedef std::chrono::nanoseconds ns;
Engine::NetworkPerfStats Engine::NetworkMemBandwidthTolerance(const InferenceEngine::CNNNetwork &network) {
auto startTime = Time::now();
float L2_cache_size = mkldnn::utils::get_cache_size(2 /*level*/, true /*per core */);
float L3_cache_size = mkldnn::utils::get_cache_size(3, false);
std::cout<< "L3_cache_sizeL3_cache_size " << L3_cache_size << std::endl;
const auto nGraphFunc = network.getFunction();
ngraph::NodeVector nodes;
int total_convs = 0, mem_limited_convs = 0, compute_convs = 0, total_gemms = 0, mem_limited_gemms = 0,
total_deconvs = 0, compute_deconvs = 0, mem_limited_deconvs = 0;
auto memLimitedFactor = [&] (int size_data_moved, int datatype_size = 4) -> float { return (L2_cache_size * 1.0f/*util factor, tbd */
/ (size_data_moved * datatype_size));};
auto isLowPrecision = [&] (ngraph::element::Type type) -> bool {
return (type == ngraph::element::i8) || (type == ngraph::element::u8);
};
auto isHalfPrecision = [&] (ngraph::element::Type type) -> bool {
return (type == ngraph::element::bf16) || (type == ngraph::element::f16);
};
// auto isSuitable1x1Convolution = [](std::shared_ptr<ngraph::Node> node) {
// ngraph::Input<ngraph::Node> kernels = node->input(1);
// if (node->get_output_size() == 1 && node->output(0).get_shape().size() == 4) {
// auto shape = kernels.get_shape();
// if (shape.size() >= 2 && shape[0] == 1 && shape[1] == 1) {
// auto conv = std::dynamic_pointer_cast<ngraph::op::ConvolutionIE>(node);
// return conv && conv->get_group() == 1 && conv->get_strides()[0] == 1 && conv->get_strides()[1] == 1;
// }
// }
// return false;
// };
// auto isSuitableChildConvolution = [](const ngraph::Node* node) {
// ngraph::Input<const ngraph::Node> kernels = node->input(1);
// if (node->output(0).get_shape().size() == 4) {
// auto shape = kernels.get_shape();
// const auto conv = dynamic_cast<const ngraph::op::ConvolutionIE*>(node);
// return conv
// && shape[2] != 1 && shape[2] == conv->get_group()
// && conv->get_strides()[0] == 1 && conv->get_strides()[1] == 1
// && conv->get_dilations()[0] == 1 && conv->get_dilations()[1] == 1
// && conv->get_pads_begin()[0] == 1 && conv->get_pads_end()[0] == 1
// && conv->get_pads_begin()[1] == 1 && conv->get_pads_end()[1] == 1;
// }
// return false;
// };
float worst_case = NetworkPerfStats::memThresholdUnknown;
float worst_case_all = NetworkPerfStats::memThresholdUnknown;
// Traverse nGraph Function in topological order
for (auto & node : nGraphFunc->get_ordered_ops()) {
// todo : bias data size (always fp)
if (std::strcmp("MatMul", node->get_type_info().name) && std::strcmp("Convolution", node->get_type_info().name)
&& std::strcmp("ConvolutionBackpropData", node->get_type_info().name)) {
int inputs_data_size_bytes = 0;
for (int i = 0; i < node->get_input_size(); i++) {
auto type = node->input_value(i).get_element_type();
const bool isINT8 = isLowPrecision(type); // bf16 tbd
const bool isBF16 = isHalfPrecision(type); // bf16 tbd
const int data_type_size = isINT8 ? 1 : isBF16 ? 2 : 4;
ngraph::Input<ngraph::Node> input = node->input(i);
const auto shapeInput = input.get_shape();
const auto non_const = !get_constant_from_source(node->input_value(i));
const auto dataSizeInput = std::accumulate(shapeInput.begin(), shapeInput.end(), 1,
std::multiplies<int>());
const auto not_amortized = non_const || (dataSizeInput * data_type_size) > L3_cache_size;
inputs_data_size_bytes += not_amortized * (dataSizeInput * data_type_size);
}
// no need to track outputs, as these are inputs to some layers
const auto factor = memLimitedFactor(inputs_data_size_bytes, 1 /*already in bytes*/);
if (factor < worst_case_all) {
worst_case_all = factor;
std::cout << "TYPE: " << node->get_type_info().name << " Name: " << node->get_friendly_name()
<< " inputs_data_size_bytes " << inputs_data_size_bytes << ", factor: " << factor << std::endl;
}
continue;
}
// todo: asymmetric conv (zero-point comes via Sub/Mul)
// auto type0 = node->input_value(0).get_element_type(); //input
auto type1 = node->input_value(1).get_element_type(); //weights
const bool isINT8 = isLowPrecision(type1); // bf16 tbd
const bool isBF16 = isHalfPrecision(type1); // bf16 tbd
const int data_type_size = isINT8 ? 1 : isBF16 ? 2 : 4;
int dataSizeInput = 0, dataSizeOutput = 0;
std::cout << "Type: " << node->get_type_info().name << " Name: "
<< node->get_friendly_name();
if (!std::strcmp("MatMul", node->get_type_info().name)) {
ngraph::Input<ngraph::Node> input0 = node->input(0);
ngraph::Input<ngraph::Node> input1 = node->input(1);
ngraph::Output<ngraph::Node> output = node->output(0);
// Check that input and output shape a fully defined (not dynamic)
if (input0.get_partial_shape().is_static() && input1.get_partial_shape().is_static()
&& output.get_partial_shape().is_static()) {
const auto shapeInput0 = input0.get_shape();
const auto shapeInput1 = input1.get_shape();
const auto non_const = !get_constant_from_source(node->input_value(1));
const auto shapeOutput = output.get_shape();
const auto dataSizeInput0 = std::accumulate(shapeInput0.begin(), shapeInput0.end(), 1,
std::multiplies<int>());
const auto dataSizeInput1 = std::accumulate(shapeInput1.begin(), shapeInput1.end(), 1,
std::multiplies<int>());
dataSizeOutput = std::accumulate(shapeOutput.begin(), shapeOutput.end(), 1,
std::multiplies<int>());
const auto total_data = dataSizeInput0 + non_const*dataSizeInput1 + dataSizeOutput;
total_gemms++;
const auto factor = memLimitedFactor(total_data, data_type_size);
mem_limited_gemms += factor < NetworkPerfStats::memThresholdNotLimited;
worst_case = std::min(factor, worst_case);
std::cout << (isINT8 ? " INT8," : isBF16 ? " BF16," : " FP32")
<< ", Input0: " << dataSizeInput0
<< ", Input1: " << dataSizeInput1 << (non_const ? " non_const, " : " const")
<< ", Output: " << dataSizeOutput
<< ", total_data: " << total_data
<< " L2_cache_size: " << L2_cache_size << " FACTOR: " << factor << std::endl;
// const auto non_const0 = !get_constant_from_source(node->input_value(0));
// const auto non_const1 = !get_constant_from_source(node->input_value(1));
// const auto dataSizeInput0 = std::accumulate(shapeInput0.begin(), shapeInput0.end(), 1,
// std::multiplies<int>());
// const auto dataSizeInput1 = std::accumulate(shapeInput1.begin(), shapeInput1.end(), 1,
// std::multiplies<int>());
// dataSizeOutput = std::accumulate(shapeOutput.begin(), shapeOutput.end(), 1,
// std::multiplies<int>());
// const auto not_amortized0 = non_const0 || ((dataSizeInput0 * data_type_size) > L3_cache_size);
// const auto not_amortized1 = non_const1 || ((dataSizeInput1 * data_type_size) > L3_cache_size);
// const auto total_data = not_amortized0*dataSizeInput0 + not_amortized1*dataSizeInput1 + dataSizeOutput;
// total_gemms++;
// const auto factor = memLimitedFactor(total_data, data_type_size);
// mem_limited_gemms += factor < NetworkPerfStats::memThresholdNotLimited;
// worst_case = std::min(factor, worst_case);
// std::cout << (isINT8 ? " INT8," : isBF16 ? " BF16," : " FP32")
// << ", Input0: " << dataSizeInput0
// << (non_const0 ? " non_const" : " const") << (not_amortized0 ? ", not" : ",") << " amort "
// << ", Input1: " << dataSizeInput1
// << (non_const1 ? " non_const" : " const") << (not_amortized1 ? ", not" : ",") << " amort "
// << ", Output: " << dataSizeOutput
// << ", total_data: " << total_data
// << " L2_cache_size: " << L2_cache_size << " L3_cache_size: " << L3_cache_size
// << " FACTOR: " << factor << std::endl;
}
} else if (!std::strcmp("Convolution", node->get_type_info().name)) {
// Check that input and output shape a fully defined (not dynamic)
ngraph::Input<ngraph::Node> input = node->input(0);
ngraph::Output<ngraph::Node> output = node->output(0);
ngraph::Input<ngraph::Node> kernels = node->input(1);
const auto shape = kernels.get_shape();
total_convs++;
std::cout << " kernel is " << shape[2] << "x" << shape[3];
if (shape.size() >= 4 /* conventional 2D/3D conv */ && shape[2] >= 3 && shape[3] >= 3) {
// if (shape.size() >= 4 /* conventional 2D/3D conv */ && shape[2] >= 5 && shape[3] >= 5) {
std::cout << ", considering flops/byte amortizing the mem" << std::endl;
compute_convs++;
continue;
}
if (input.get_partial_shape().is_static() && output.get_partial_shape().is_static()) {
const auto shapeInput = input.get_shape();
const auto shapeOutput = output.get_shape();
if (shapeInput.size() > 4 /*5D*/) {
std::cout << ", considering 5D, " << std::endl;
compute_convs++;
continue;
}
dataSizeInput = std::accumulate(shapeInput.begin(), shapeInput.end(), 1,
std::multiplies<int>());
dataSizeOutput = std::accumulate(shapeOutput.begin(), shapeOutput.end(), 1,
std::multiplies<int>());
// if (mkldnn::impl::cpu::x64::mayiuse(mkldnn::impl::cpu::x64::avx2)
// || mkldnn::impl::cpu::x64::mayiuse(mkldnn::impl::cpu::x64::avx512_common)) {
// if (isSuitable1x1Convolution(node) &&
// isSuitableChildConvolution(output.get_target_inputs().begin()->get_node())) {
// if ((dataSizeInput + dataSizeOutput > L3_cache_size)) {
// std::cout << ", considering FUSED" << std::endl;
// continue;
// }
// }
// }
const auto factor = memLimitedFactor(dataSizeInput + dataSizeOutput, data_type_size);
mem_limited_convs += factor < NetworkPerfStats::memThresholdNotLimited;
worst_case = std::min(factor, worst_case);
std::cout << (isINT8 ? " INT8 " : isBF16 ? " BF16 " : " FP32")
<< ", dataSize: " << dataSizeInput + dataSizeOutput
<< ", L2_cache_size: " << L2_cache_size << " FACTOR: " << factor << std::endl;
}
} else if (!std::strcmp("ConvolutionBackpropData", node->get_type_info().name)) {
// Check that input and output shape a fully defined (not dynamic)
ngraph::Input<ngraph::Node> input = node->input(0);
ngraph::Output<ngraph::Node> output = node->output(0);
ngraph::Input<ngraph::Node> kernels = node->input(1);
const auto shape = kernels.get_shape();
total_deconvs++;
if (input.get_partial_shape().is_static() && output.get_partial_shape().is_static()) {
const auto shapeInput = input.get_shape();
const auto shapeOutput = output.get_shape();
if (shapeInput.size() > 4 /*5D*/) {
std::cout << ", considering 5D, " << std::endl;
compute_deconvs++;
continue;
}
dataSizeInput = std::accumulate(shapeInput.begin(), shapeInput.end(), 1,
std::multiplies<int>());
dataSizeOutput = std::accumulate(shapeOutput.begin(), shapeOutput.end(), 1,
std::multiplies<int>());
const auto factor = memLimitedFactor(dataSizeInput + dataSizeOutput, data_type_size);
mem_limited_deconvs += factor < NetworkPerfStats::memThresholdNotLimited;
worst_case = std::min(factor, worst_case);
std::cout << ", kernel "<< shape[2]<< "x" << shape[2]
<< (isINT8 ? " INT8," : isBF16 ? " BF16," : " FP32,")
<< ", dataSize: " << dataSizeInput + dataSizeOutput
<< ", L2_cache_size: " << L2_cache_size << " FACTOR: " << factor << std::endl;
}
}
}
std::cout << "Total convs: " << total_convs << ". Mem limited: " << mem_limited_convs << ". Compute: " << compute_convs << std::endl;
std::cout << "Total DEconvs: " << total_deconvs<< ". Mem limited: " << mem_limited_deconvs << ". Compute: " << compute_deconvs << std::endl;
// std::cout << "Total OTHER OPS: " << total_other_ops << ". Mem limited: " << mem_limited_other_ops << std::endl;
std::cout << "Total gemms: " << total_gemms<< ". Mem limited: " << mem_limited_gemms << std::endl;
NetworkPerfStats res;
res.maxMemTolerance = worst_case;
res.ratio_mem_limited_convs = total_convs ? static_cast<float>(mem_limited_convs)/total_convs : 0;
res.ratio_compute_convs = total_convs ? static_cast<float>(compute_convs)/total_convs : 0;
res.ratio_compute_deconvs = total_deconvs ? static_cast<float>(compute_deconvs)/total_deconvs : 0;
// if (!total_convs && !total_deconvs && !total_gemms) {
// std::cout << "WORST CASE ALL: " << worst_case_all << std::endl;
// res.maxMemTolerance = worst_case_all;
// } else {
std::cout << "WORST CASE: " << worst_case << std::endl;
// }
auto time = std::chrono::duration_cast<ns>(Time::now() - startTime).count() * 0.000001;
std::cout << "NetworkMemBandwidthTolerance time: " << time << " ms" << std::endl;
return res;
}
static bool hasAVX512();
InferenceEngine::IExecutableNetworkInternal::Ptr
Engine::LoadExeNetworkImpl(const InferenceEngine::CNNNetwork &network, const std::map<std::string, std::string> &config) {
Engine::LoadExeNetworkImpl(const InferenceEngine::CNNNetwork &network, const std::map<std::string, std::string> &orig_config) {
OV_ITT_SCOPED_TASK(itt::domains::MKLDNNPlugin, "Engine::LoadExeNetworkImpl");
// verification of supported input
@@ -394,23 +633,83 @@ Engine::LoadExeNetworkImpl(const InferenceEngine::CNNNetwork &network, const std
// TODO: handle input precision differently - per input and not one per network...
auto config = orig_config;
CNNNetwork clonedNetwork = InferenceEngine::details::cloneNetwork(network);
const auto& lptProp = config.find(InferenceEngine::PluginConfigInternalParams::KEY_LP_TRANSFORMS_MODE);
const bool enableLPT = (lptProp != config.end() && lptProp->second == PluginConfigParams::YES) /* enabled in the orig_config*/
|| Config::LPTransformsMode::On == engConfig.lpTransformsMode /* or already enabled */;
Transformation(clonedNetwork, enableLPT);
// Here the OV perf modes are turned into specific settings (as we need the network for better params selection)
const auto& mode = config.find(PluginConfigParams::KEY_OV_PERFORMANCE_MODE);
// the mode may have just arrived to the LoadNetwork (higher pri), or was set with the plugins' SetConfig
if (mode != config.end() || !engConfig.ovPerfMode.empty()) {
const auto mode_name = (mode != config.end()) ? mode->second : engConfig.ovPerfMode;
//checking streams (to avoid overriding what user might explicitly set in the incoming config or previously via SetConfig)
const auto streams = config.find(PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS);
if (streams == config.end() && !streamsSet) { // for EXPERIMENT: overriding the user streams settings
if (mode_name == CONFIG_VALUE(LATENCY)) {
config[PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS] = CONFIG_VALUE(CPU_THROUGHPUT_NUMA);
} else if (mode_name == CONFIG_VALUE(THROUGHPUT)) {
Engine::NetworkPerfStats NetworkToleranceForLowCache = NetworkMemBandwidthTolerance(clonedNetwork);
const auto num_cores = getNumberOfCPUCores();
const auto num_streams_default_not_ht = num_cores / 2;
const auto default_num_streams = IStreamsExecutor::Config::GetDefaultNumStreams();
// this is first heuristic in series (carefully separating int8, bf16 and float32):
// memory bandwidth limited
// compute limited
// Hybrid specific
// etc
int num_streams;
if (NetworkToleranceForLowCache.maxMemTolerance == NetworkPerfStats::memThresholdUnknown) {
if ((NetworkToleranceForLowCache.ratio_compute_convs == NetworkPerfStats::ALL)
|| (NetworkToleranceForLowCache.ratio_compute_deconvs == NetworkPerfStats::ALL)) {
std::cout << " case 1.1" <<std::endl;
num_streams = num_cores;
} else {
num_streams = default_num_streams;
std::cout << "case 0" <<std::endl;
}
} else if ((NetworkToleranceForLowCache.maxMemTolerance > NetworkPerfStats::memThresholdNotLimited)
|| (hasAVX512()
&& NetworkToleranceForLowCache.maxMemTolerance > NetworkPerfStats::memThresholdAssumeLimitedAVX512
&& NetworkToleranceForLowCache.ratio_mem_limited_convs <= NetworkPerfStats::memLimitedRatioThresholdAVX512)) {
std::cout << " case 1.0 or 1.2" <<std::endl;
num_streams = num_cores;
} else if (NetworkToleranceForLowCache.maxMemTolerance > NetworkPerfStats::memThresholdAssumeLimited) {
num_streams = std::max(default_num_streams, num_streams_default_not_ht);
std::cout << "case 2" <<std::endl;
} else {
if (NetworkToleranceForLowCache.maxMemTolerance > NetworkPerfStats::memThresholdAssumeLimitedMuch) {
num_streams = std::min(default_num_streams, num_streams_default_not_ht);
std::cout << "case 3" << std::endl;
} else {
num_streams = default_num_streams/2;
std::cout << "case 3.1" << std::endl;
}
}
config[PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS] = std::to_string(num_streams);
std::cout << "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! "
<< (NetworkToleranceForLowCache.maxMemTolerance <= NetworkPerfStats::memThresholdAssumeLimited ? "YES" : "NO")
<< ", NUM_STREAMS " << num_streams << std::endl;
}
}
}
// update the props after the perf mode translated to configs
// TODO: Clarify the behavior of SetConfig method. Skip eng_config or not?
Config conf = engConfig;
conf.readProperties(config);
if (conf.enableDynamicBatch) {
conf.batchLimit = static_cast<int>(network.getBatchSize());
}
CNNNetwork clonedNetwork = InferenceEngine::details::cloneNetwork(network);
Transformation(clonedNetwork, conf);
return std::make_shared<MKLDNNExecNetwork>(clonedNetwork, conf, extensionManager, weightsSharing);
}
void Engine::SetConfig(const std::map<std::string, std::string> &config) {
// accumulate config parameters on engine level
streamsSet = (config.find(PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS) != config.end());
engConfig.readProperties(config);
}
@@ -525,7 +824,10 @@ QueryNetworkResult Engine::QueryNetwork(const CNNNetwork& network, const std::ma
auto clonedNetwork = InferenceEngine::details::cloneNetwork(network);
auto ops = clonedNetwork.getFunction()->get_ordered_ops();
Transformation(clonedNetwork, conf);
const auto& lptProp = config.find(InferenceEngine::PluginConfigInternalParams::KEY_LP_TRANSFORMS_MODE);
const bool enableLPT = (lptProp != config.end() && lptProp->second == PluginConfigParams::YES) /* enabled in the orig_config*/
|| Config::LPTransformsMode::On == engConfig.lpTransformsMode /* or already enabled */;
Transformation(clonedNetwork, enableLPT);
std::unordered_set<std::string> supported;
std::unordered_set<std::string> unsupported;
for (auto op : ops) {
inference-engine/src/mkldnn_plugin/mkldnn_plugin.h
+19
View File
@@ -13,6 +13,7 @@
#include <memory>
#include <functional>
#include <vector>
#include <cfloat>
namespace MKLDNNPlugin {
@@ -40,6 +41,24 @@ private:
Config engConfig;
NumaNodesWeights weightsSharing;
MKLDNNExtensionManager::Ptr extensionManager = std::make_shared<MKLDNNExtensionManager>();
bool streamsSet = false;
struct NetworkPerfStats {
float maxMemTolerance = -1;
float ratio_compute_convs = 0;
float ratio_mem_limited_convs = 0;
float ratio_compute_deconvs = 0;
static constexpr float memThresholdNotLimited = 1.0f;
static constexpr float memThresholdAssumeLimited = 0.5f;
static constexpr float memThresholdAssumeLimitedAVX512 = memThresholdAssumeLimited/2;
static constexpr float memThresholdAssumeLimitedMuch = memThresholdAssumeLimited/4;
static constexpr float memThresholdUnknown = FLT_MAX;
static constexpr float memLimitedRatioThresholdAVX512 = 0.10;
static constexpr float ALL = 1.0f;
};
static NetworkPerfStats NetworkMemBandwidthTolerance(const InferenceEngine::CNNNetwork &network);
};
} // namespace MKLDNNPlugin
inference-engine/src/plugin_api/threading/ie_istreams_executor.hpp
+1
View File
@@ -79,6 +79,7 @@ public:
* @return configured values
*/
static Config MakeDefaultMultiThreaded(const Config& initial, const bool fp_intesive = true);
static int GetDefaultNumStreams(); // no network specifics considered (only CPU's caps);
std::string _name; //!< Used by `ITT` to name executor threads
int _streams = 1; //!< Number of streams.