IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Removed legacy methods SetBatch and SetBlob (#17984)

Browse Source 
* Removed legacy methods SetBatch and SetBlob

* Fixed GPU plugin build

* Remove DYN_BATCH_LIMIT from tests

* Revert some changes in GPU plugin
This commit is contained in:
Ilya Churaev 2023-06-12 22:54:23 +04:00 committed by GitHub
parent df44f92a97
commit 0743e9bfb5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
47 changed files with 18 additions and 708 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
samples/cpp/hello_query_device/README.md
View File

@ -67,8 +67,6 @@ The application prints all available devices with their supported metrics and de
[ INFO ] CPU_THREADS_NUM : 0 [ INFO ] CPU_THREADS_NUM : 0
[ INFO ] CPU_THROUGHPUT_STREAMS : 1 [ INFO ] CPU_THROUGHPUT_STREAMS : 1
[ INFO ] DUMP_EXEC_GRAPH_AS_DOT : "" [ INFO ] DUMP_EXEC_GRAPH_AS_DOT : ""
[ INFO ] DYN_BATCH_ENABLED : NO
[ INFO ] DYN_BATCH_LIMIT : 0
[ INFO ] ENFORCE_BF16 : NO [ INFO ] ENFORCE_BF16 : NO
[ INFO ] EXCLUSIVE_ASYNC_REQUESTS : NO [ INFO ] EXCLUSIVE_ASYNC_REQUESTS : NO
[ INFO ] PERFORMANCE_HINT : "" [ INFO ] PERFORMANCE_HINT : ""

2
samples/python/hello_query_device/README.md
View File

@ -62,8 +62,6 @@ For example:
[ INFO ] CPU_THREADS_NUM: 0 [ INFO ] CPU_THREADS_NUM: 0
[ INFO ] CPU_THROUGHPUT_STREAMS: 1 [ INFO ] CPU_THROUGHPUT_STREAMS: 1
[ INFO ] DUMP_EXEC_GRAPH_AS_DOT: [ INFO ] DUMP_EXEC_GRAPH_AS_DOT:
[ INFO ] DYN_BATCH_ENABLED: NO
[ INFO ] DYN_BATCH_LIMIT: 0
[ INFO ] ENFORCE_BF16: NO [ INFO ] ENFORCE_BF16: NO
[ INFO ] EXCLUSIVE_ASYNC_REQUESTS: NO [ INFO ] EXCLUSIVE_ASYNC_REQUESTS: NO
[ INFO ] PERFORMANCE_HINT: [ INFO ] PERFORMANCE_HINT:

12
src/inference/dev_api/cpp_interfaces/impl/ie_infer_async_request_thread_safe_default.hpp
View File

@ -240,11 +240,6 @@ public:
_syncRequest->SetBlob(name, data); _syncRequest->SetBlob(name, data);
} }
void SetBlob(const std::string& name, const Blob::Ptr& data, const PreProcessInfo& info) override {
CheckState();
_syncRequest->SetBlob(name, data, info);
}
void SetBlobs(const std::string& name, const std::vector<Blob::Ptr>& blobs) override { void SetBlobs(const std::string& name, const std::vector<Blob::Ptr>& blobs) override {
CheckState(); CheckState();
_syncRequest->SetBlobs(name, blobs); _syncRequest->SetBlobs(name, blobs);
@ -264,13 +259,6 @@ public:
return _syncRequest->GetPreProcess(name); return _syncRequest->GetPreProcess(name);
} }
OPENVINO_SUPPRESS_DEPRECATED_START
void SetBatch(int batch) override {
CheckState();
_syncRequest->SetBatch(batch);
};
OPENVINO_SUPPRESS_DEPRECATED_END
void SetCallback(Callback callback) override { void SetCallback(Callback callback) override {
CheckState(); CheckState();
_callback = std::move(callback); _callback = std::move(callback);

18
src/inference/dev_api/cpp_interfaces/interface/ie_iinfer_request_internal.hpp
View File

@ -131,15 +131,6 @@ public:
*/ */
virtual BatchedBlob::Ptr GetBlobs(const std::string& name); virtual BatchedBlob::Ptr GetBlobs(const std::string& name);
/**
* @brief Sets pre-process for input data
* @param name Name of input blob.
* @param data - a reference to input or output blob. The type of Blob must correspond to the network input
* precision and size.
* @param info Preprocess info for blob.
*/
virtual void SetBlob(const std::string& name, const Blob::Ptr& data, const PreProcessInfo& info);
/** /**
* @brief Gets pre-process for input data * @brief Gets pre-process for input data
* @param name Name of input blob. * @param name Name of input blob.
@ -147,14 +138,6 @@ public:
*/ */
virtual const PreProcessInfo& GetPreProcess(const std::string& name) const; virtual const PreProcessInfo& GetPreProcess(const std::string& name) const;
/**
* @brief Sets new batch size when dynamic batching is enabled in executable network that created this request.
* @deprecated
* @param batch - new batch size to be used by all the following inference calls for this request.
*/
INFERENCE_ENGINE_DEPRECATED("This method is deprecated and will be removed in 2023.1 release")
virtual void SetBatch(int batch);
/** /**
* @brief Queries memory states. * @brief Queries memory states.
* @return Returns memory states * @return Returns memory states
@ -347,7 +330,6 @@ protected:
std::vector<std::shared_ptr<const ov::Node>> _results; //!< A vector of function outputs std::vector<std::shared_ptr<const ov::Node>> _results; //!< A vector of function outputs
std::map<std::string, PreProcessDataPtr> _preProcData; //!< A map of pre-process data per input std::map<std::string, PreProcessDataPtr> _preProcData; //!< A map of pre-process data per input
std::map<std::string, BatchedBlob::Ptr> _batched_inputs; //!< A map of user passed blobs for network inputs std::map<std::string, BatchedBlob::Ptr> _batched_inputs; //!< A map of user passed blobs for network inputs
int m_curBatch = -1; //!< Current batch value used in dynamic batching
/** /**
* @brief A shared pointer to IInferRequestInternal * @brief A shared pointer to IInferRequestInternal

20
src/inference/include/ie/cpp/ie_infer_request.hpp
View File

@ -118,17 +118,6 @@ public:
*/ */
Blob::Ptr GetBlob(const std::string& name); Blob::Ptr GetBlob(const std::string& name);
/**
* @deprecated This method will be removed in 2023.1 release
* @brief Sets blob with a pre-process information
* @note Returns an error in case if data blob is output
* @param name Name of input blob.
* @param data A reference to input. The type of Blob must correspond to the network input precision and size.
* @param info Preprocess info for blob.
*/
INFERENCE_ENGINE_DEPRECATED("This method is deprecated and will be removed in 2023.1 release")
void SetBlob(const std::string& name, const Blob::Ptr& data, const PreProcessInfo& info);
/** /**
* @brief Gets pre-process for input data * @brief Gets pre-process for input data
* @param name Name of input blob. * @param name Name of input blob.
@ -175,15 +164,6 @@ public:
*/ */
void SetOutput(const BlobMap& results); void SetOutput(const BlobMap& results);
/**
* @brief Sets new batch size when dynamic batching is enabled in executable network that created this request.
* @deprecated
*
* @param batch new batch size to be used by all the following inference calls for this request.
*/
INFERENCE_ENGINE_DEPRECATED("This method is deprecated and will be removed in 2023.1 release")
void SetBatch(const int batch);
/** /**
* @brief Start inference of specified input(s) in asynchronous mode * @brief Start inference of specified input(s) in asynchronous mode
* *

27
src/inference/include/ie/ie_iinfer_request.hpp
View File

@ -85,21 +85,6 @@ public:
*/ */
virtual StatusCode GetBlob(const char* name, Blob::Ptr& data, ResponseDesc* resp) noexcept = 0; virtual StatusCode GetBlob(const char* name, Blob::Ptr& data, ResponseDesc* resp) noexcept = 0;
/**
* @deprecated This method will be removed in 2023.1 release
* @brief Sets pre-process for input data
* @param name Name of input blob.
* @param data Reference to input or output blob. The type of Blob must match the network input precision and size.
* @param info Preprocess info for blob.
* @param resp Optional: pointer to an already allocated object to contain information in case of failure
* @return Status code of the operation: OK (0) for success
*/
INFERENCE_ENGINE_DEPRECATED("This method is deprecated and will be removed in 2023.1 release")
virtual StatusCode SetBlob(const char* name,
const Blob::Ptr& data,
const PreProcessInfo& info,
ResponseDesc* resp) noexcept = 0;
/** /**
* @brief Gets pre-process for input data * @brief Gets pre-process for input data
* @param name Name of input blob. * @param name Name of input blob.
@ -200,18 +185,6 @@ public:
*/ */
virtual StatusCode SetUserData(void* data, ResponseDesc* resp) noexcept = 0; virtual StatusCode SetUserData(void* data, ResponseDesc* resp) noexcept = 0;
/**
* @brief Sets new batch size when dynamic batching is enabled in executable network that created this request.
*
* @deprecated
* @param batch_size new batch size to be used by all the following inference calls for this request.
* @param resp Optional: a pointer to an already allocated object to contain extra information of a failure (if
* occurred)
* @return Enumeration of the resulted action: InferenceEngine::OK (0) for success
*/
INFERENCE_ENGINE_DEPRECATED("This method is deprecated and will be removed in 2023.1 release")
virtual InferenceEngine::StatusCode SetBatch(int batch_size, ResponseDesc* resp) noexcept = 0;
protected: protected:
virtual ~IInferRequest() = default; virtual ~IInferRequest() = default;
}; };

24
src/inference/include/ie/ie_plugin_config.hpp
View File

@ -405,30 +405,6 @@ DECLARE_CONFIG_VALUE(CPU_THROUGHPUT_AUTO);
INFERENCE_ENGINE_1_0_DEPRECATED INFERENCE_ENGINE_1_0_DEPRECATED
DECLARE_CONFIG_KEY(PERF_COUNT); DECLARE_CONFIG_KEY(PERF_COUNT);
/**
* @brief The key defines dynamic limit of batch processing.
* @deprecated
*
* Specified value is applied to all following Infer() calls. Inference Engine processes
* min(batch_limit, original_batch_size) first pictures from input blob. For example, if input
* blob has sizes 32x3x224x224 after applying plugin.SetConfig({KEY_DYN_BATCH_LIMIT, 10})
* Inference Engine primitives processes only beginner subblobs with size 10x3x224x224.
* This value can be changed before any Infer() call to specify a new batch limit.
*
* The paired parameter value should be convertible to integer number. Acceptable values:
* -1 - Do not limit batch processing
* >0 - Direct value of limit. Batch size to process is min(new batch_limit, original_batch)
*/
INFERENCE_ENGINE_DEPRECATED("This config is deprecated and will be removed in 2023.1 release")
DECLARE_CONFIG_KEY(DYN_BATCH_LIMIT);
/**
* @brief The key checks whether dynamic batch is enabled.
* @deprecated
*/
INFERENCE_ENGINE_DEPRECATED("This config is deprecated and will be removed in 2023.1 release")
DECLARE_CONFIG_KEY(DYN_BATCH_ENABLED);
/** /**
* @brief This key directs the plugin to load a configuration file. * @brief This key directs the plugin to load a configuration file.
* *

8
src/inference/src/cpp/ie_infer_async_request_base.hpp
View File

@ -134,10 +134,6 @@ public:
TO_STATUS(_impl->SetBlob(name, data)); TO_STATUS(_impl->SetBlob(name, data));
} }
StatusCode SetBlob(const char* name, const Blob::Ptr& data, const PreProcessInfo& info, ResponseDesc* resp) noexcept override {
TO_STATUS(_impl->SetBlob(name, data, info));
}
StatusCode GetBlob(const char* name, Blob::Ptr& data, ResponseDesc* resp) noexcept override { StatusCode GetBlob(const char* name, Blob::Ptr& data, ResponseDesc* resp) noexcept override {
TO_STATUS(data = _impl->GetBlob(name)); TO_STATUS(data = _impl->GetBlob(name));
} }
@ -181,10 +177,6 @@ public:
StatusCode SetUserData(void* data, ResponseDesc* resp) noexcept override { StatusCode SetUserData(void* data, ResponseDesc* resp) noexcept override {
TO_STATUS(_impl->SetUserData(data)); TO_STATUS(_impl->SetUserData(data));
} }
StatusCode SetBatch(int batch_size, ResponseDesc* resp) noexcept override {
TO_STATUS(_impl->SetBatch(batch_size));
}
}; };
IE_SUPPRESS_DEPRECATED_END IE_SUPPRESS_DEPRECATED_END

8
src/inference/src/cpp/ie_infer_request.cpp
View File

@ -58,10 +58,6 @@ Blob::Ptr InferRequest::GetBlob(const std::string& name) {
return blobPtr; return blobPtr;
} }
void InferRequest::SetBlob(const std::string& name, const Blob::Ptr& data, const PreProcessInfo& info) {
INFER_REQ_CALL_STATEMENT(_impl->SetBlob(name, data, info);)
}
const PreProcessInfo& InferRequest::GetPreProcess(const std::string& name) const { const PreProcessInfo& InferRequest::GetPreProcess(const std::string& name) const {
INFER_REQ_CALL_STATEMENT(return _impl->GetPreProcess(name);) INFER_REQ_CALL_STATEMENT(return _impl->GetPreProcess(name);)
} }
@ -86,10 +82,6 @@ void InferRequest::SetOutput(const BlobMap& results) {
INFER_REQ_CALL_STATEMENT(for (auto&& result : results) { _impl->SetBlob(result.first, result.second); }) INFER_REQ_CALL_STATEMENT(for (auto&& result : results) { _impl->SetBlob(result.first, result.second); })
} }
void InferRequest::SetBatch(const int batch) {
INFER_REQ_CALL_STATEMENT(_impl->SetBatch(batch);)
}
void InferRequest::StartAsync() { void InferRequest::StartAsync() {
INFER_REQ_CALL_STATEMENT(_impl->StartAsync();) INFER_REQ_CALL_STATEMENT(_impl->StartAsync();)
} }

18
src/inference/src/cpp_interfaces/interface/ie_iinfer_request_internal.cpp
View File

@ -407,18 +407,6 @@ BatchedBlob::Ptr IInferRequestInternal::GetBlobs(const std::string& name) {
return nullptr; return nullptr;
} }
void IInferRequestInternal::SetBlob(const std::string& name, const Blob::Ptr& data, const PreProcessInfo& info) {
InputInfo::Ptr foundInput;
DataPtr foundOutput;
if (findInputAndOutputBlobByName(name, foundInput, foundOutput)) {
foundInput->getPreProcess() = copyPreProcess(info);
} else {
IE_THROW() << "Pre-process can't be set to output blob";
}
SetBlob(name, data);
}
const PreProcessInfo& IInferRequestInternal::GetPreProcess(const std::string& name) const { const PreProcessInfo& IInferRequestInternal::GetPreProcess(const std::string& name) const {
InputInfo::Ptr foundInput; InputInfo::Ptr foundInput;
DataPtr foundOutput; DataPtr foundOutput;
@ -429,10 +417,6 @@ const PreProcessInfo& IInferRequestInternal::GetPreProcess(const std::string& na
} }
} }
void IInferRequestInternal::SetBatch(int batch) {
IE_THROW(NotImplemented);
}
std::vector<std::shared_ptr<IVariableStateInternal>> IInferRequestInternal::QueryState() { std::vector<std::shared_ptr<IVariableStateInternal>> IInferRequestInternal::QueryState() {
IE_THROW(NotImplemented); IE_THROW(NotImplemented);
} }
@ -460,7 +444,7 @@ void IInferRequestInternal::execDataPreprocessing(InferenceEngine::BlobMap& prep
// using preconfigured resize algorithm. // using preconfigured resize algorithm.
auto it = _preProcData.find(input.first); auto it = _preProcData.find(input.first);
if (it != _preProcData.end()) { if (it != _preProcData.end()) {
it->second->execute(input.second, _networkInputs[input.first]->getPreProcess(), serial, m_curBatch); it->second->execute(input.second, _networkInputs[input.first]->getPreProcess(), serial, -1);
} }
} }
} }

10
src/inference/src/dev/converter_utils.cpp
View File

@ -541,20 +541,10 @@ public:
return std::make_shared<InferenceEngine::BatchedBlob>(blobs); return std::make_shared<InferenceEngine::BatchedBlob>(blobs);
} }
void SetBlob(const std::string& name,
const InferenceEngine::Blob::Ptr& data,
const InferenceEngine::PreProcessInfo& info) override {
OPENVINO_NOT_IMPLEMENTED;
}
const InferenceEngine::PreProcessInfo& GetPreProcess(const std::string& name) const override { const InferenceEngine::PreProcessInfo& GetPreProcess(const std::string& name) const override {
OPENVINO_NOT_IMPLEMENTED; OPENVINO_NOT_IMPLEMENTED;
} }
void SetBatch(int batch) override {
OPENVINO_NOT_IMPLEMENTED;
}
std::vector<std::shared_ptr<InferenceEngine::IVariableStateInternal>> QueryState() override { std::vector<std::shared_ptr<InferenceEngine::IVariableStateInternal>> QueryState() override {
auto res = m_request->query_state(); auto res = m_request->query_state();
std::vector<std::shared_ptr<InferenceEngine::IVariableStateInternal>> ret; std::vector<std::shared_ptr<InferenceEngine::IVariableStateInternal>> ret;

14
src/inference/tests/functional/async_infer_request_test.cpp
View File

@ -23,13 +23,6 @@ TEST(InferRequestCPPTests, throwsOnUninitializedGetBlob) {
ASSERT_THROW(req.GetBlob({}), InferenceEngine::NotAllocated); ASSERT_THROW(req.GetBlob({}), InferenceEngine::NotAllocated);
} }
TEST(InferRequestCPPTests, throwsOnUninitializedSetBlobPreproc) {
InferRequest req;
IE_SUPPRESS_DEPRECATED_START
ASSERT_THROW(req.SetBlob({}, {}, {}), InferenceEngine::NotAllocated);
IE_SUPPRESS_DEPRECATED_END
}
TEST(InferRequestCPPTests, throwsOnUninitializedGetPreProcess) { TEST(InferRequestCPPTests, throwsOnUninitializedGetPreProcess) {
InferRequest req; InferRequest req;
ASSERT_THROW(req.GetPreProcess({}), InferenceEngine::NotAllocated); ASSERT_THROW(req.GetPreProcess({}), InferenceEngine::NotAllocated);
@ -55,13 +48,6 @@ TEST(InferRequestCPPTests, throwsOnUninitializedSetOutput) {
ASSERT_THROW(req.SetOutput({{}}), InferenceEngine::NotAllocated); ASSERT_THROW(req.SetOutput({{}}), InferenceEngine::NotAllocated);
} }
TEST(InferRequestCPPTests, throwsOnUninitializedSetBatch) {
InferRequest req;
IE_SUPPRESS_DEPRECATED_START
ASSERT_THROW(req.SetBatch({}), InferenceEngine::NotAllocated);
IE_SUPPRESS_DEPRECATED_END
}
TEST(InferRequestCPPTests, throwsOnUninitializedStartAsync) { TEST(InferRequestCPPTests, throwsOnUninitializedStartAsync) {
InferRequest req; InferRequest req;
ASSERT_THROW(req.StartAsync(), InferenceEngine::NotAllocated); ASSERT_THROW(req.StartAsync(), InferenceEngine::NotAllocated);

9
src/plugins/auto/src/infer_request.cpp
View File

@ -110,15 +110,6 @@ void MultiDeviceInferRequest::SetBlob(const std::string& name, const InferenceEn
IInferRequestInternal::SetBlob(name, blob); IInferRequestInternal::SetBlob(name, blob);
} }
IE_SUPPRESS_DEPRECATED_START
void MultiDeviceInferRequest::SetBlob(const std::string& name, const Blob::Ptr& blob, const PreProcessInfo& info) {
if (_sharedRequest)
_sharedRequest->SetBlob(name, blob, info);
else
IInferRequestInternal::SetBlob(name, blob, info);
}
IE_SUPPRESS_DEPRECATED_END
InferenceEngine::Blob::Ptr MultiDeviceInferRequest::GetBlob(const std::string& name) { InferenceEngine::Blob::Ptr MultiDeviceInferRequest::GetBlob(const std::string& name) {
if (_sharedRequest) if (_sharedRequest)
return _sharedRequest->GetBlob(name); return _sharedRequest->GetBlob(name);

7
src/plugins/auto/src/infer_request.hpp
View File

@ -39,13 +39,6 @@ public:
InferenceEngine::RemoteContext::Ptr ctx = nullptr); InferenceEngine::RemoteContext::Ptr ctx = nullptr);
std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> GetPerformanceCounts() const override; std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> GetPerformanceCounts() const override;
void SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr& blob) override; void SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr& blob) override;
/**
* @deprecated This method will be removed in 2024.1 release
* @brief Sets blob with a pre-process information
*/
void SetBlob(const std::string& name,
const InferenceEngine::Blob::Ptr& blob,
const InferenceEngine::PreProcessInfo& info) override;
InferenceEngine::Blob::Ptr GetBlob(const std::string& name) override; InferenceEngine::Blob::Ptr GetBlob(const std::string& name) override;
std::vector<std::shared_ptr<InferenceEngine::IVariableStateInternal>> QueryState() override; std::vector<std::shared_ptr<InferenceEngine::IVariableStateInternal>> QueryState() override;
// Multi-Device impl specific: sets the data (blobs from the device-less requests to the specific device request) // Multi-Device impl specific: sets the data (blobs from the device-less requests to the specific device request)

8
src/plugins/hetero/infer_request.cpp
View File

@ -102,14 +102,6 @@ InferenceEngine::Blob::Ptr HeteroInferRequest::GetBlob(const std::string& name)
return itRequest->second->GetBlob(name); return itRequest->second->GetBlob(name);
} }
void HeteroInferRequest::SetBlob(const std::string& name, const Blob::Ptr& blob, const PreProcessInfo& info) {
auto itRequest = _subRequestFromBlobName.find(name);
if (itRequest == _subRequestFromBlobName.end()) {
IE_THROW() << "There is no infer requests binded to blob with name: " << name;
}
itRequest->second->SetBlob(name, blob, info);
}
const InferenceEngine::PreProcessInfo& HeteroInferRequest::GetPreProcess(const std::string& name) const { const InferenceEngine::PreProcessInfo& HeteroInferRequest::GetPreProcess(const std::string& name) const {
auto itRequest = _subRequestFromBlobName.find(name); auto itRequest = _subRequestFromBlobName.find(name);
if (itRequest == _subRequestFromBlobName.end()) { if (itRequest == _subRequestFromBlobName.end()) {

4
src/plugins/hetero/infer_request.hpp
View File

@ -44,10 +44,6 @@ public:
InferenceEngine::Blob::Ptr GetBlob(const std::string& name) override; InferenceEngine::Blob::Ptr GetBlob(const std::string& name) override;
void SetBlob(const std::string& name,
const InferenceEngine::Blob::Ptr& blob,
const InferenceEngine::PreProcessInfo& info) override;
const InferenceEngine::PreProcessInfo& GetPreProcess(const std::string& name) const override; const InferenceEngine::PreProcessInfo& GetPreProcess(const std::string& name) const override;
std::vector<std::shared_ptr<InferenceEngine::IVariableStateInternal>> QueryState() override; std::vector<std::shared_ptr<InferenceEngine::IVariableStateInternal>> QueryState() override;

27
src/plugins/intel_cpu/src/config.cpp
View File

@ -118,17 +118,6 @@ void Config::readProperties(const std::map<std::string, std::string> &prop) {
IE_THROW() << "Wrong value " << val << "for property key " << ov::hint::enable_hyper_threading.name() IE_THROW() << "Wrong value " << val << "for property key " << ov::hint::enable_hyper_threading.name()
<< ". Expected only true/false." << std::endl; << ". Expected only true/false." << std::endl;
} }
} else if (key == PluginConfigParams::KEY_DYN_BATCH_LIMIT) {
int val_i = -1;
try {
val_i = std::stoi(val);
} catch (const std::exception&) {
IE_THROW() << "Wrong value for property key " << PluginConfigParams::KEY_DYN_BATCH_LIMIT
<< ". Expected only integer numbers";
}
// zero and any negative value will be treated
// as default batch size
batchLimit = std::max(val_i, 0);
} else if (key == CPUConfigParams::KEY_CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE) { } else if (key == CPUConfigParams::KEY_CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE) {
float val_f = 0.0f; float val_f = 0.0f;
try { try {
@ -155,14 +144,6 @@ void Config::readProperties(const std::map<std::string, std::string> &prop) {
else else
IE_THROW() << "Wrong value for property key " << PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS IE_THROW() << "Wrong value for property key " << PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS
<< ". Expected only YES/NO"; << ". Expected only YES/NO";
} else if (key.compare(PluginConfigParams::KEY_DYN_BATCH_ENABLED) == 0) {
if (val.compare(PluginConfigParams::YES) == 0)
enableDynamicBatch = true;
else if (val.compare(PluginConfigParams::NO) == 0)
enableDynamicBatch = false;
else
IE_THROW() << "Wrong value for property key " << PluginConfigParams::KEY_DYN_BATCH_ENABLED
<< ". Expected only YES/NO";
IE_SUPPRESS_DEPRECATED_START IE_SUPPRESS_DEPRECATED_START
} else if (key.compare(PluginConfigParams::KEY_DUMP_EXEC_GRAPH_AS_DOT) == 0) { } else if (key.compare(PluginConfigParams::KEY_DUMP_EXEC_GRAPH_AS_DOT) == 0) {
IE_SUPPRESS_DEPRECATED_END IE_SUPPRESS_DEPRECATED_END
@ -307,14 +288,6 @@ void Config::updateProperties() {
_config.insert({ PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, PluginConfigParams::YES }); _config.insert({ PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, PluginConfigParams::YES });
else else
_config.insert({ PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, PluginConfigParams::NO }); _config.insert({ PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, PluginConfigParams::NO });
IE_SUPPRESS_DEPRECATED_START
if (enableDynamicBatch == true)
_config.insert({ PluginConfigParams::KEY_DYN_BATCH_ENABLED, PluginConfigParams::YES });
else
_config.insert({ PluginConfigParams::KEY_DYN_BATCH_ENABLED, PluginConfigParams::NO });
_config.insert({ PluginConfigParams::KEY_DYN_BATCH_LIMIT, std::to_string(batchLimit) });
IE_SUPPRESS_DEPRECATED_END
_config.insert({ PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, std::to_string(streamExecutorConfig._streams) }); _config.insert({ PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, std::to_string(streamExecutorConfig._streams) });

2
src/plugins/intel_cpu/src/config.h
View File

@ -42,11 +42,9 @@ struct Config {
bool collectPerfCounters = false; bool collectPerfCounters = false;
bool exclusiveAsyncRequests = false; bool exclusiveAsyncRequests = false;
bool enableDynamicBatch = false;
SnippetsMode snippetsMode = SnippetsMode::Enable; SnippetsMode snippetsMode = SnippetsMode::Enable;
std::string dumpToDot = {}; std::string dumpToDot = {};
std::string device_id = {}; std::string device_id = {};
int batchLimit = 0;
float fcSparseWeiDecompressionRate = 1.0f; float fcSparseWeiDecompressionRate = 1.0f;
#if defined(OPENVINO_ARCH_X86_64) #if defined(OPENVINO_ARCH_X86_64)
size_t rtCacheCapacity = 5000ul; size_t rtCacheCapacity = 5000ul;

54
src/plugins/intel_cpu/src/exec_network.cpp
View File

@ -82,13 +82,6 @@ ExecNetwork::ExecNetwork(const InferenceEngine::CNNNetwork &network,
_cfg.isNewApi = !isLegacyAPI(); _cfg.isNewApi = !isLegacyAPI();
_mutex = std::make_shared<std::mutex>(); _mutex = std::make_shared<std::mutex>();
if (_cfg.batchLimit > 1) {
// check topology for applicability
if (!CanProcessDynBatch(_network)) {
IE_THROW() << "Graph::CreateGraph: such topology cannot be compiled for dynamic batch!";
}
}
if (cfg.exclusiveAsyncRequests) { if (cfg.exclusiveAsyncRequests) {
// special case when all InferRequests are muxed into a single queue // special case when all InferRequests are muxed into a single queue
_taskExecutor = _plugin->executorManager()->getExecutor("CPU"); _taskExecutor = _plugin->executorManager()->getExecutor("CPU");
@ -378,53 +371,6 @@ InferenceEngine::Parameter ExecNetwork::GetMetric(const std::string &name) const
return GetMetricLegacy(name, graph); return GetMetricLegacy(name, graph);
} }
bool ExecNetwork::CanProcessDynBatch(const InferenceEngine::CNNNetwork &network) const {
InputsDataMap inputs = network.getInputsInfo();
if (inputs.empty())
return false;
auto function = network.getFunction();
if (function == nullptr) {
IE_THROW() << "CPU plug-in doesn't support not ngraph-based model!";
}
auto ops = function->get_ordered_ops();
for (const auto& op : ops) {
auto type = TypeFromName(op->get_type_name());
if (type == Type::Tile) {
const auto repeatsNode = std::dynamic_pointer_cast<const ngraph::opset1::Constant>(op->get_input_node_shared_ptr(1));
if (!repeatsNode)
return false;
const auto tile = std::dynamic_pointer_cast<const ngraph::opset1::Tile>(op);
if (tile && repeatsNode->cast_vector<int64_t>()[0] == 1)
continue;
}
if (type == Type::Reshape) {
if (op->get_input_shape(0)[0] == op->get_output_shape(0)[0])
continue;
}
if (type != Type::Input &&
type != Type::Output &&
type != Type::Convolution &&
type != Type::Deconvolution &&
type != Type::Lrn &&
type != Type::Pooling &&
type != Type::FullyConnected &&
type != Type::MatMul &&
type != Type::Softmax &&
type != Type::Split &&
type != Type::Concatenation &&
type != Type::Eltwise) {
return false;
}
}
return true;
}
void ExecNetwork::Export(std::ostream& modelStream) { void ExecNetwork::Export(std::ostream& modelStream) {
CNNNetworkSerializer serializer(modelStream, extensionManager); CNNNetworkSerializer serializer(modelStream, extensionManager);
serializer <<_network; serializer <<_network;

2
src/plugins/intel_cpu/src/exec_network.h
View File

@ -76,8 +76,6 @@ protected:
*/ */
GraphGuard::Lock GetGraph() const; GraphGuard::Lock GetGraph() const;
bool CanProcessDynBatch(const InferenceEngine::CNNNetwork &network) const;
bool isLegacyAPI() const; bool isLegacyAPI() const;
InferenceEngine::Parameter GetConfigLegacy(const std::string &name) const; InferenceEngine::Parameter GetConfigLegacy(const std::string &name) const;

8
src/plugins/intel_cpu/src/graph.cpp
View File

@ -983,14 +983,6 @@ void Graph::PullOutputData(BlobMap &out) {
outBloMem.SetData(intr_blob, false); outBloMem.SetData(intr_blob, false);
} else { } else {
size_t size_to_copy = intr_blob.GetDescWithType<BlockedMemoryDesc>()->getPaddedElementsCount(); size_t size_to_copy = intr_blob.GetDescWithType<BlockedMemoryDesc>()->getPaddedElementsCount();
// used only for backward compatibility with the legacy API
if (getConfig().batchLimit && dynBatch > 0) {
if (node->isDynamicNode() && !getConfig().isNewApi) {
IE_THROW(NotImplemented) << "[DS] not implemented dynamic batch for node with dynamic shape";
}
size_to_copy = std::accumulate(outDims.begin() + 1, outDims.end(), (size_t)1, std::multiplies<size_t>()) * static_cast<size_t>(dynBatch);
}
cpu_convert(intr_blob_ptr, ext_blob_ptr, srcPrec, dstPrec, size_to_copy); cpu_convert(intr_blob_ptr, ext_blob_ptr, srcPrec, dstPrec, size_to_copy);
} }

15
src/plugins/intel_cpu/src/graph.h
View File

@ -190,17 +190,6 @@ public:
return graphHasDynamicInput; return graphHasDynamicInput;
} }
/**
* @brief This call updates the dynamic batch value
*
* @note It is used for backward compatibility with legacy API only.
* @param newDynBatch
* new dynamic batch value
*/
void setDynBatch(int newDynBatch) {
dynBatch = newDynBatch;
}
protected: protected:
void VisitNode(NodePtr node, std::vector<NodePtr>& sortedNodes); void VisitNode(NodePtr node, std::vector<NodePtr>& sortedNodes);
@ -266,10 +255,6 @@ private:
GraphContext::CPtr context; GraphContext::CPtr context;
// this field stores the dynamic batch value to provide backward compatibility
// with the legacy API dyn batch behaviour
int dynBatch = -1;
void EnforceBF16(); void EnforceBF16();
}; };

31
src/plugins/intel_cpu/src/infer_request.cpp
View File

@ -354,19 +354,6 @@ void LegacyInferRequest::initBlobs() {
} }
} }
void LegacyInferRequest::SetBatch(int new_batch) {
if (!graph->getConfig().enableDynamicBatch)
IE_THROW() << "Dynamic batch is not enabled.";
if (new_batch < 1 || new_batch > graph->getConfig().batchLimit) {
IE_THROW() << "Invalid dynamic batch size " << new_batch <<
" for this request.";
}
m_curBatch = new_batch;
graph->setDynBatch(m_curBatch);
}
void LegacyInferRequest::changeDefaultPtr() { void LegacyInferRequest::changeDefaultPtr() {
// renew external pointers before infer // renew external pointers before infer
const auto &inMap = graph->inputNodesMap; const auto &inMap = graph->inputNodesMap;
@ -448,7 +435,7 @@ void LegacyInferRequest::SetBlob(const std::string& name, const InferenceEngine:
auto pBlobDesc = MemoryDescUtils::interpretAsBlobDesc(graph->getInputNodeByName(name)->getChildEdgesAtPort(0)[0]->getMemory()); auto pBlobDesc = MemoryDescUtils::interpretAsBlobDesc(graph->getInputNodeByName(name)->getChildEdgesAtPort(0)[0]->getMemory());
if (data->getTensorDesc() == pBlobDesc && if (data->getTensorDesc() == pBlobDesc &&
graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getConfig().batchLimit) { graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end()) {
externalPtr[name] = data->buffer(); externalPtr[name] = data->buffer();
} else if (externalPtr.find(name) != externalPtr.end()) { } else if (externalPtr.find(name) != externalPtr.end()) {
externalPtr.erase(name); externalPtr.erase(name);
@ -481,8 +468,7 @@ void LegacyInferRequest::SetBlob(const std::string& name, const InferenceEngine:
} }
auto pBlobDesc = MemoryDescUtils::interpretAsBlobDesc(graph->getOutputNodeByName(name)->getParentEdgesAtPort(0)[0]->getMemory()); auto pBlobDesc = MemoryDescUtils::interpretAsBlobDesc(graph->getOutputNodeByName(name)->getParentEdgesAtPort(0)[0]->getMemory());
if (data->getTensorDesc() == pBlobDesc && if (data->getTensorDesc() == pBlobDesc) {
!graph->getConfig().batchLimit) {
externalPtr[name] = data->buffer(); externalPtr[name] = data->buffer();
} else if (externalPtr.find(name) != externalPtr.end()) { } else if (externalPtr.find(name) != externalPtr.end()) {
externalPtr.erase(name); externalPtr.erase(name);
@ -527,7 +513,7 @@ InferenceEngine::Blob::Ptr LegacyInferRequest::GetBlob(const std::string& name)
_inputs[name] = make_blob_with_precision(desc); _inputs[name] = make_blob_with_precision(desc);
_inputs[name]->allocate(); _inputs[name]->allocate();
if (pBlob->getTensorDesc() == desc && if (pBlob->getTensorDesc() == desc &&
graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getConfig().batchLimit) { graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end()) {
externalPtr[name] = _inputs[name]->buffer(); externalPtr[name] = _inputs[name]->buffer();
} }
} }
@ -589,7 +575,7 @@ InferenceEngine::Blob::Ptr LegacyInferRequest::GetBlob(const std::string& name)
} }
_outputs[name] = data; _outputs[name] = data;
if (!externalPtr.count(name) && data->getTensorDesc() == pBlobDesc && !graph->getConfig().batchLimit) { if (!externalPtr.count(name) && data->getTensorDesc() == pBlobDesc) {
externalPtr[name] = data->buffer(); externalPtr[name] = data->buffer();
} }
} }
@ -706,7 +692,7 @@ void InferRequest::SetBlob(const std::string& name, const InferenceEngine::Blob:
blobDesc.getDims()); blobDesc.getDims());
} }
if (actualDesc->isCompatible(MemoryDescUtils::convertToCpuBlockedMemoryDesc(blobDesc)) && if (actualDesc->isCompatible(MemoryDescUtils::convertToCpuBlockedMemoryDesc(blobDesc)) &&
graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getConfig().batchLimit) { graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end()) {
externalPtr[name] = data->buffer(); externalPtr[name] = data->buffer();
} else if (externalPtr.find(name) != externalPtr.end()) { } else if (externalPtr.find(name) != externalPtr.end()) {
externalPtr.erase(name); externalPtr.erase(name);
@ -738,7 +724,7 @@ void InferRequest::SetBlob(const std::string& name, const InferenceEngine::Blob:
} }
const auto &desc = graph->getOutputNodeByName(name)->getParentEdgesAtPort(0)[0]->getMemory().getDesc(); const auto &desc = graph->getOutputNodeByName(name)->getParentEdgesAtPort(0)[0]->getMemory().getDesc();
if (!isDynamic && blobDesc == MemoryDescUtils::convertToTensorDesc(desc) && !graph->getConfig().batchLimit) { if (!isDynamic && blobDesc == MemoryDescUtils::convertToTensorDesc(desc)) {
externalPtr[name] = data->buffer(); externalPtr[name] = data->buffer();
} else if (externalPtr.find(name) != externalPtr.end()) { } else if (externalPtr.find(name) != externalPtr.end()) {
externalPtr.erase(name); externalPtr.erase(name);
@ -786,7 +772,7 @@ InferenceEngine::Blob::Ptr InferRequest::GetBlob(const std::string& name) {
if (!isDynamic && if (!isDynamic &&
desc == MemoryDescUtils::convertToTensorDesc(graph->getInputNodeByName(name)->getChildEdgesAtPort(0)[0]->getMemory().getDesc()) && desc == MemoryDescUtils::convertToTensorDesc(graph->getInputNodeByName(name)->getChildEdgesAtPort(0)[0]->getMemory().getDesc()) &&
graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end() && !graph->getConfig().batchLimit) { graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end()) {
externalPtr[name] = _inputs[name]->buffer(); externalPtr[name] = _inputs[name]->buffer();
} }
} else { } else {
@ -845,8 +831,7 @@ InferenceEngine::Blob::Ptr InferRequest::GetBlob(const std::string& name) {
_outputs[name] = data; _outputs[name] = data;
if (!isDynamic && !externalPtr.count(name) && if (!isDynamic && !externalPtr.count(name) &&
data->getTensorDesc() == MemoryDescUtils::convertToTensorDesc(output->second->getParentEdgesAtPort(0)[0]->getMemory().getDesc()) && data->getTensorDesc() == MemoryDescUtils::convertToTensorDesc(output->second->getParentEdgesAtPort(0)[0]->getMemory().getDesc())) {
!graph->getConfig().batchLimit) {
externalPtr[name] = data->buffer(); externalPtr[name] = data->buffer();
} }
} else { } else {

1
src/plugins/intel_cpu/src/infer_request.h
View File

@ -84,7 +84,6 @@ public:
private: private:
void PushInputData() override; void PushInputData() override;
void initBlobs() override; void initBlobs() override;
void SetBatch(int batch = -1) override;
void changeDefaultPtr() override; void changeDefaultPtr() override;
}; };

18
src/plugins/intel_cpu/src/plugin.cpp
View File

@ -418,13 +418,6 @@ static bool shouldEnforceBF16(const std::map<std::string, std::string>& modelCon
} }
static Config::SnippetsMode getSnippetsMode(const std::map<std::string, std::string>& modelConfig, const Config& engineConfig) { static Config::SnippetsMode getSnippetsMode(const std::map<std::string, std::string>& modelConfig, const Config& engineConfig) {
const auto& dynamicBatchProp = modelConfig.find(InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_ENABLED);
const bool enableDynamicBatch = (dynamicBatchProp != modelConfig.end() && dynamicBatchProp->second == PluginConfigParams::YES)
|| engineConfig.enableDynamicBatch;
if (enableDynamicBatch) // dynamic batch is not supported
return Config::SnippetsMode::Disable;
const auto& snippetsMode = modelConfig.find(InferenceEngine::PluginConfigInternalParams::KEY_SNIPPETS_MODE); const auto& snippetsMode = modelConfig.find(InferenceEngine::PluginConfigInternalParams::KEY_SNIPPETS_MODE);
if (snippetsMode == modelConfig.end()) // not set explicitly if (snippetsMode == modelConfig.end()) // not set explicitly
return Config::SnippetsMode::Enable; // enable by default return Config::SnippetsMode::Enable; // enable by default
@ -501,10 +494,6 @@ Engine::LoadExeNetworkImpl(const InferenceEngine::CNNNetwork &network, const std
Config conf = engConfig; Config conf = engConfig;
conf.readProperties(config); conf.readProperties(config);
if (conf.enableDynamicBatch) {
conf.batchLimit = static_cast<int>(network.getBatchSize());
}
if (is_cpu_map_available()) { if (is_cpu_map_available()) {
GetPerformanceStreams(conf, nGraphFunc); GetPerformanceStreams(conf, nGraphFunc);
} }
@ -744,10 +733,6 @@ QueryNetworkResult Engine::QueryNetwork(const CNNNetwork& network, const std::ma
Config conf = engConfig; Config conf = engConfig;
conf.readProperties(config); conf.readProperties(config);
if (conf.enableDynamicBatch) {
conf.batchLimit = static_cast<int>(network.getBatchSize());
}
const auto& lptProp = config.find(InferenceEngine::PluginConfigInternalParams::KEY_LP_TRANSFORMS_MODE); const auto& lptProp = config.find(InferenceEngine::PluginConfigInternalParams::KEY_LP_TRANSFORMS_MODE);
const bool enableLPT = (lptProp != config.end() && lptProp->second == PluginConfigParams::YES) /* enabled in the orig_config*/ const bool enableLPT = (lptProp != config.end() && lptProp->second == PluginConfigParams::YES) /* enabled in the orig_config*/
|| Config::LPTransformsMode::On == engConfig.lpTransformsMode /* or already enabled */; || Config::LPTransformsMode::On == engConfig.lpTransformsMode /* or already enabled */;
@ -816,9 +801,6 @@ InferenceEngine::IExecutableNetworkInternal::Ptr Engine::ImportNetwork(std::istr
} }
} }
if (conf.enableDynamicBatch) {
conf.batchLimit = static_cast<int>(cnnnetwork.getBatchSize());
}
if (is_cpu_map_available()) { if (is_cpu_map_available()) {
get_num_streams(conf.streamExecutorConfig._streams, function, conf); get_num_streams(conf.streamExecutorConfig._streams, function, conf);
} }

7
src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/infer_request/config.cpp
View File

@ -20,8 +20,7 @@ namespace {
{{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, InferenceEngine::PluginConfigParams::CPU_THROUGHPUT_NUMA}}, {{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, InferenceEngine::PluginConfigParams::CPU_THROUGHPUT_NUMA}},
{{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, "8"}}, {{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, "8"}},
{{InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, InferenceEngine::PluginConfigParams::NO}}, {{InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, InferenceEngine::PluginConfigParams::NO}},
{{InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, InferenceEngine::PluginConfigParams::YES}}, {{InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, InferenceEngine::PluginConfigParams::YES}}
{{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "10"}},
}; };
const std::vector<std::map<std::string, std::string>> MultiInConfigs = { const std::vector<std::map<std::string, std::string>> MultiInConfigs = {
@ -36,9 +35,7 @@ namespace {
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU}, {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, InferenceEngine::PluginConfigParams::NO}}, {InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, InferenceEngine::PluginConfigParams::NO}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU}, {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, InferenceEngine::PluginConfigParams::YES}}, {InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, InferenceEngine::PluginConfigParams::YES}}
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "10"}}
}; };
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestConfigTest, INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestConfigTest,

39
src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/infer_request/dynamic_batch.cpp
View File

@ -1,39 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <behavior/infer_request/dynamic_batch.hpp>
#include "common_test_utils/test_constants.hpp"
namespace ConfigurationTestsDefinitions {
namespace {
std::vector<size_t> batch_sizes = {
1,
5,
9,
16
};
std::map<std::string, std::string> additional_config = {
};
} // namespace
INSTANTIATE_TEST_SUITE_P(smoke_DynamicBatchTest_async, DynamicBatchTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_CPU),
::testing::Values(InferenceEngine::Precision::FP32),
::testing::Values(batch_sizes),
::testing::Values(true),
::testing::Values(additional_config)),
DynamicBatchTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_DynamicBatchTest_sync, DynamicBatchTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_CPU),
::testing::Values(InferenceEngine::Precision::FP32),
::testing::Values(batch_sizes),
::testing::Values(false),
::testing::Values(additional_config)),
DynamicBatchTest::getTestCaseName);
} // namespace ConfigurationTestsDefinitions

5
src/plugins/intel_cpu/tests/functional/shared_tests_instances/behavior/plugin/configuration_tests.cpp
View File

@ -60,7 +60,6 @@ namespace {
{{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, "8"}}, {{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, "8"}},
{{InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, InferenceEngine::PluginConfigParams::NO}}, {{InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, InferenceEngine::PluginConfigParams::NO}},
{{InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, InferenceEngine::PluginConfigParams::YES}}, {{InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, InferenceEngine::PluginConfigParams::YES}},
{{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "10"}},
// check that hints doesn't override customer value (now for streams and later for other config opts) // check that hints doesn't override customer value (now for streams and later for other config opts)
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT}, {{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT},
{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, "3"}}, {InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, "3"}},
@ -137,7 +136,6 @@ namespace {
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "should be int"}}, {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "should be int"}},
{{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, "OFF"}}, {{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, "OFF"}},
{{InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, "OFF"}}, {{InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, "OFF"}},
{{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "NAN"}}
}; };
const std::vector<std::map<std::string, std::string>> multiinconfigs = { const std::vector<std::map<std::string, std::string>> multiinconfigs = {
@ -168,8 +166,6 @@ namespace {
{InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, "OFF"}}, {InferenceEngine::PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS, "OFF"}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU}, {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, "OFF"}}, {InferenceEngine::PluginConfigParams::KEY_CPU_BIND_THREAD, "OFF"}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "NAN"}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU}, {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU},
{InferenceEngine::PluginConfigParams::KEY_MODEL_PRIORITY, "-1"}}, {InferenceEngine::PluginConfigParams::KEY_MODEL_PRIORITY, "-1"}},
{{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU}, {{InferenceEngine::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES , CommonTestUtils::DEVICE_CPU},
@ -238,7 +234,6 @@ namespace {
{{InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::YES}}, {{InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::YES}},
{{InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, InferenceEngine::PluginConfigParams::NO}}, {{InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, InferenceEngine::PluginConfigParams::NO}},
{{InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, InferenceEngine::PluginConfigParams::YES}}, {{InferenceEngine::PluginConfigParams::KEY_EXCLUSIVE_ASYNC_REQUESTS, InferenceEngine::PluginConfigParams::YES}},
{{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "10"}}
}; };
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, CorrectConfigCheck, INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, CorrectConfigCheck,

14
src/plugins/intel_cpu/tests/functional/shared_tests_instances/single_layer_tests/reshape.cpp
View File

@ -15,20 +15,6 @@ const std::vector<InferenceEngine::Precision> netPrecisions = {
InferenceEngine::Precision::FP16 InferenceEngine::Precision::FP16
}; };
INSTANTIATE_TEST_SUITE_P(smoke_ReshapeCheckDynBatch, ReshapeLayerTest,
::testing::Combine(
::testing::Values(true),
::testing::ValuesIn(netPrecisions),
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(std::vector<size_t>({30, 30, 30, 30})),
::testing::Values(std::vector<int64_t>({30, 30, 30, 30})),
::testing::Values(CommonTestUtils::DEVICE_CPU),
::testing::Values(std::map<std::string, std::string>({{CONFIG_KEY(DYN_BATCH_ENABLED), CONFIG_VALUE(YES)}}))),
ReshapeLayerTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_ReshapeCheck, ReshapeLayerTest, INSTANTIATE_TEST_SUITE_P(smoke_ReshapeCheck, ReshapeLayerTest,
::testing::Combine( ::testing::Combine(
::testing::Values(true), ::testing::Values(true),

3
src/plugins/intel_gpu/include/intel_gpu/plugin/infer_request_legacy.hpp
View File

@ -46,7 +46,7 @@ public:
void SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr &data) override; void SetBlob(const std::string& name, const InferenceEngine::Blob::Ptr &data) override;
void SetBlobs(const std::string& name, const std::vector<InferenceEngine::Blob::Ptr> &data) override; void SetBlobs(const std::string& name, const std::vector<InferenceEngine::Blob::Ptr> &data) override;
void SetBatch(int batch = -1) override; void SetBatch(int batch = -1);
std::vector<std::shared_ptr<InferenceEngine::IVariableStateInternal>> QueryState() override; std::vector<std::shared_ptr<InferenceEngine::IVariableStateInternal>> QueryState() override;
void SetGraph(std::shared_ptr<Graph> graph); void SetGraph(std::shared_ptr<Graph> graph);
void EnableProfiling() { m_useProfiling = true; } void EnableProfiling() { m_useProfiling = true; }
@ -108,6 +108,7 @@ private:
std::map<cldnn::primitive_id, cldnn::network_output> internal_outputs; std::map<cldnn::primitive_id, cldnn::network_output> internal_outputs;
std::vector<std::map<cldnn::primitive_id, cldnn::network_output>> internal_outputs_dynamic; std::vector<std::map<cldnn::primitive_id, cldnn::network_output>> internal_outputs_dynamic;
Graph::variable_states_map variables_states_; Graph::variable_states_map variables_states_;
int m_curBatch = -1;
}; };
} // namespace intel_gpu } // namespace intel_gpu

1
src/plugins/intel_gpu/include/intel_gpu/runtime/internal_properties.hpp
View File

@ -52,7 +52,6 @@ static constexpr Property<std::vector<std::string>, PropertyMutability::RW> cust
static constexpr Property<ImplForcingMap, PropertyMutability::RW> force_implementations{"GPU_FORCE_IMPLEMENTATIONS"}; static constexpr Property<ImplForcingMap, PropertyMutability::RW> force_implementations{"GPU_FORCE_IMPLEMENTATIONS"};
static constexpr Property<std::string, PropertyMutability::RW> config_file{"CONFIG_FILE"}; static constexpr Property<std::string, PropertyMutability::RW> config_file{"CONFIG_FILE"};
static constexpr Property<bool, PropertyMutability::RW> enable_lp_transformations{"LP_TRANSFORMS_MODE"}; static constexpr Property<bool, PropertyMutability::RW> enable_lp_transformations{"LP_TRANSFORMS_MODE"};
static constexpr Property<bool, PropertyMutability::RW> enable_dynamic_batch{"DYN_BATCH_ENABLED"};
static constexpr Property<size_t, PropertyMutability::RW> max_dynamic_batch{"DYN_BATCH_LIMIT"}; static constexpr Property<size_t, PropertyMutability::RW> max_dynamic_batch{"DYN_BATCH_LIMIT"};
static constexpr Property<bool, PropertyMutability::RW> exclusive_async_requests{"EXCLUSIVE_ASYNC_REQUESTS"}; static constexpr Property<bool, PropertyMutability::RW> exclusive_async_requests{"EXCLUSIVE_ASYNC_REQUESTS"};
static constexpr Property<bool, PropertyMutability::RW> nv12_two_inputs{"GPU_NV12_TWO_INPUTS"}; static constexpr Property<bool, PropertyMutability::RW> nv12_two_inputs{"GPU_NV12_TWO_INPUTS"};

1
src/plugins/intel_gpu/src/plugin/compiled_model.cpp
View File

@ -346,7 +346,6 @@ InferenceEngine::Parameter CompiledModel::GetMetric(const std::string &name) con
CONFIG_KEY(PERFORMANCE_HINT), CONFIG_KEY(PERFORMANCE_HINT),
CONFIG_KEY(PERFORMANCE_HINT_NUM_REQUESTS), CONFIG_KEY(PERFORMANCE_HINT_NUM_REQUESTS),
CONFIG_KEY(PERF_COUNT), CONFIG_KEY(PERF_COUNT),
CONFIG_KEY(DYN_BATCH_ENABLED),
CONFIG_KEY(CONFIG_FILE), CONFIG_KEY(CONFIG_FILE),
CONFIG_KEY(DEVICE_ID), CONFIG_KEY(DEVICE_ID),
CONFIG_KEY(EXCLUSIVE_ASYNC_REQUESTS), CONFIG_KEY(EXCLUSIVE_ASYNC_REQUESTS),

2
src/plugins/intel_gpu/src/plugin/infer_request.cpp
View File

@ -1053,7 +1053,7 @@ std::vector<std::shared_ptr<InferenceEngine::IVariableStateInternal>> InferReque
std::vector<std::shared_ptr<InferenceEngine::IVariableStateInternal>> ret{}; std::vector<std::shared_ptr<InferenceEngine::IVariableStateInternal>> ret{};
ret.reserve(variables_states_.size()); ret.reserve(variables_states_.size());
for (const auto& pair : variables_states_) for (const auto& pair : variables_states_)
ret.push_back(std::make_shared<VariableState>(pair.first, pair.second, m_graph->get_engine(), m_curBatch)); ret.push_back(std::make_shared<VariableState>(pair.first, pair.second, m_graph->get_engine(), -1));
return ret; return ret;
} }

1
src/plugins/intel_gpu/src/plugin/legacy_api_helper.cpp
View File

@ -229,7 +229,6 @@ std::vector<std::string> LegacyAPIHelper::get_supported_configs() {
CONFIG_KEY(PERFORMANCE_HINT), CONFIG_KEY(PERFORMANCE_HINT),
CONFIG_KEY(PERFORMANCE_HINT_NUM_REQUESTS), CONFIG_KEY(PERFORMANCE_HINT_NUM_REQUESTS),
CONFIG_KEY(PERF_COUNT), CONFIG_KEY(PERF_COUNT),
CONFIG_KEY(DYN_BATCH_ENABLED),
CONFIG_KEY(CONFIG_FILE), CONFIG_KEY(CONFIG_FILE),
CONFIG_KEY(DEVICE_ID), CONFIG_KEY(DEVICE_ID),
CONFIG_KEY(EXCLUSIVE_ASYNC_REQUESTS), CONFIG_KEY(EXCLUSIVE_ASYNC_REQUESTS),

13
src/plugins/intel_gpu/src/plugin/program.cpp
View File

@ -170,22 +170,11 @@ Program::Program(InferenceEngine::CNNNetwork& network, cldnn::engine& engine, co
bool dyn_shape_batch_found = false; bool dyn_shape_batch_found = false;
std::map<std::string, ngraph::PartialShape> shapes; std::map<std::string, ngraph::PartialShape> shapes;
std::map<std::string, std::pair<int64_t, int64_t>> batch_dim; std::map<std::string, std::pair<int64_t, int64_t>> batch_dim;
auto enable_dynamic_batch = m_config.get_property(ov::intel_gpu::enable_dynamic_batch);
if (enable_dynamic_batch) {
m_config.set_property(ov::intel_gpu::max_dynamic_batch(network.getBatchSize()));
// in case of legacy dynamic batch,
// we assume 4D input with 0 batch dim
auto param = func->get_parameters().front();
auto pname = getParamName(param);
shapes[pname] = param->get_output_partial_shape(0);
batch_dim[pname].first = 0;
batch_dim[pname].second = m_config.get_property(ov::intel_gpu::max_dynamic_batch);
} else {
dyn_shape_batch_found = IsDynBatchModel(func, shapes, batch_dim); dyn_shape_batch_found = IsDynBatchModel(func, shapes, batch_dim);
if (dyn_shape_batch_found) { if (dyn_shape_batch_found) {
m_config.set_property(ov::intel_gpu::max_dynamic_batch(batch_dim.begin()->second.second)); m_config.set_property(ov::intel_gpu::max_dynamic_batch(batch_dim.begin()->second.second));
} }
}
int m_bv_sz = GetMaxBatchSizeForSingleProgram(); int m_bv_sz = GetMaxBatchSizeForSingleProgram();
m_max_batch = static_cast<int>(m_config.get_property(ov::intel_gpu::max_dynamic_batch)); m_max_batch = static_cast<int>(m_config.get_property(ov::intel_gpu::max_dynamic_batch));

1
src/plugins/intel_gpu/src/runtime/execution_config.cpp
View File

@ -54,7 +54,6 @@ void ExecutionConfig::set_default() {
std::make_tuple(ov::intel_gpu::enable_loop_unrolling, true), std::make_tuple(ov::intel_gpu::enable_loop_unrolling, true),
// Legacy API properties // Legacy API properties
std::make_tuple(ov::intel_gpu::enable_dynamic_batch, false),
std::make_tuple(ov::intel_gpu::exclusive_async_requests, false), std::make_tuple(ov::intel_gpu::exclusive_async_requests, false),
std::make_tuple(ov::intel_gpu::nv12_two_inputs, false), std::make_tuple(ov::intel_gpu::nv12_two_inputs, false),
std::make_tuple(ov::intel_gpu::config_file, ""), std::make_tuple(ov::intel_gpu::config_file, ""),

41
src/plugins/intel_gpu/tests/functional/shared_tests_instances/behavior/infer_request/dynamic_batch.cpp
View File

@ -1,41 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <behavior/infer_request/dynamic_batch.hpp>
#include "common_test_utils/test_constants.hpp"
namespace ConfigurationTestsDefinitions {
namespace {
std::vector<size_t> batch_sizes = {
16,
1,
5,
9,
16
};
auto additional_config = []() {
return std::map<std::string, std::string>{};
};
} // namespace
INSTANTIATE_TEST_SUITE_P(smoke_DynamicBatchTest_async, DynamicBatchTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_GPU),
::testing::Values(InferenceEngine::Precision::FP32),
::testing::Values(batch_sizes),
::testing::Values(true),
::testing::Values(additional_config())),
DynamicBatchTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_DynamicBatchTest_sync, DynamicBatchTest,
::testing::Combine(
::testing::Values(CommonTestUtils::DEVICE_GPU),
::testing::Values(InferenceEngine::Precision::FP32),
::testing::Values(batch_sizes),
::testing::Values(false),
::testing::Values(additional_config())),
DynamicBatchTest::getTestCaseName);
} // namespace ConfigurationTestsDefinitions

15
src/plugins/intel_gpu/tests/functional/shared_tests_instances/single_layer_tests/reshape.cpp
View File

@ -15,21 +15,6 @@ const std::vector<InferenceEngine::Precision> netPrecisions = {
InferenceEngine::Precision::I64 InferenceEngine::Precision::I64
}; };
//TODO: Issue : - 28981
INSTANTIATE_TEST_SUITE_P(DISABLE_smoke_ReshapeCheckDynBatch, ReshapeLayerTest,
::testing::Combine(
::testing::Values(true),
::testing::ValuesIn(netPrecisions),
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(std::vector<size_t>({1, 16, 16, 16})),
::testing::Values(std::vector<int64_t>({1, 0, 256})),
::testing::Values(CommonTestUtils::DEVICE_GPU),
::testing::Values(std::map<std::string, std::string>({{CONFIG_KEY(DYN_BATCH_ENABLED), CONFIG_VALUE(YES)}}))),
ReshapeLayerTest::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(smoke_ReshapeCheck, ReshapeLayerTest, INSTANTIATE_TEST_SUITE_P(smoke_ReshapeCheck, ReshapeLayerTest,
::testing::Combine( ::testing::Combine(
::testing::Values(true), ::testing::Values(true),

3
src/tests/functional/plugin/conformance/test_runner/api_conformance_runner/src/behavior/plugin/configuration_tests.cpp
View File

@ -38,7 +38,6 @@ namespace {
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY}}, {{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY}},
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY}, {{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "1"}}, {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "1"}},
{{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "10"}},
// check that hints doesn't override customer value (now for streams and later for other config opts) // check that hints doesn't override customer value (now for streams and later for other config opts)
}; };
@ -88,7 +87,6 @@ INSTANTIATE_TEST_SUITE_P(ie_plugin_Hetero, CorrectConfigTests,
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "-1"}}, {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "-1"}},
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT}, {{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::THROUGHPUT},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "should be int"}}, {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "should be int"}},
{{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "NAN"}}
}; };
const std::vector<std::map<std::string, std::string>> pluginMultiInConfigs = { const std::vector<std::map<std::string, std::string>> pluginMultiInConfigs = {
@ -165,7 +163,6 @@ INSTANTIATE_TEST_SUITE_P(ie_plugin_Hetero, CorrectConfigTests,
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY}}, {{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY}},
{{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY}, {{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT, InferenceEngine::PluginConfigParams::LATENCY},
{InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "1"}}, {InferenceEngine::PluginConfigParams::KEY_PERFORMANCE_HINT_NUM_REQUESTS, "1"}},
{{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "10"}}
}; };
INSTANTIATE_TEST_SUITE_P(ie_plugin, CorrectConfigCheck, INSTANTIATE_TEST_SUITE_P(ie_plugin, CorrectConfigCheck,

44
src/tests/functional/plugin/shared/include/behavior/infer_request/dynamic_batch.hpp
View File

@ -1,44 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#pragma once
#include <cstddef>
#include <vector>
#include <string>
#include <tuple>
#include "shared_test_classes/base/layer_test_utils.hpp"
namespace ConfigurationTestsDefinitions {
typedef std::tuple<
std::string, // Device
InferenceEngine::Precision, // Network precision
std::vector<size_t>, // Batch sizes
bool, // Asynchronous execution
std::map<std::string, std::string> // Additional configuration
> dynamicBatchTestParams;
class DynamicBatchTest : virtual public LayerTestsUtils::LayerTestsCommon,
public testing::WithParamInterface<dynamicBatchTestParams> {
private:
bool run_async = false;
size_t max_batch_size = 0;
std::vector<size_t> batch_sizes;
std::vector<std::vector<InferenceEngine::Blob::Ptr>> reference_inputs;
std::vector<std::vector<InferenceEngine::Blob::Ptr>> scaled_inputs;
std::vector<std::vector<std::pair<ngraph::element::Type, std::vector<std::uint8_t>>>> reference_outputs;
std::vector<std::vector<InferenceEngine::Blob::Ptr>> actual_outputs;
std::vector<InferenceEngine::InferRequest> infer_requests;
protected:
void SetUp() override;
void Run() override;
void LoadNetwork() override;
void Infer() override;
void Validate() override;
public:
static std::string getTestCaseName(const testing::TestParamInfo<dynamicBatchTestParams> &obj);
};
} // namespace ConfigurationTestsDefinitions

5
src/tests/functional/plugin/shared/include/behavior/plugin/preprocessing.hpp
View File

@ -65,11 +65,6 @@ public:
inputs.push_back(blob); inputs.push_back(blob);
} }
if (configuration.count(InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_ENABLED) &&
configuration.count(InferenceEngine::PluginConfigParams::YES)) {
auto batchSize = executableNetwork.GetInputsInfo().begin()->second->getTensorDesc().getDims()[0] / 2;
inferRequest.SetBatch(static_cast<int>(batchSize));
}
inferRequest.Infer(); inferRequest.Infer();
} }

156
src/tests/functional/plugin/shared/src/behavior/infer_request/dynamic_batch.cpp
View File

@ -1,156 +0,0 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include <tuple>
#include <string>
#include <vector>
#include <algorithm>
#include "ie_core.hpp"
#include "ie_transformations.hpp"
#include "common_test_utils/common_utils.hpp"
#include "functional_test_utils/skip_tests_config.hpp"
#include "behavior/infer_request/dynamic_batch.hpp"
#include "ngraph_functions/subgraph_builders.hpp"
namespace ConfigurationTestsDefinitions {
std::string DynamicBatchTest::getTestCaseName(const testing::TestParamInfo<dynamicBatchTestParams> &obj) {
std::string targetDevice;
InferenceEngine::Precision netPrecision;
std::vector<size_t> batchSizes;
bool runAsync;
std::map<std::string, std::string> config;
std::tie(targetDevice, netPrecision, batchSizes, runAsync, config) = obj.param;
std::ostringstream result;
result << "netPrecision=" << netPrecision.name() << "_";
result << "BS=" << CommonTestUtils::vec2str(batchSizes) << "_";
result << std::string(runAsync ? "Async" : "Sync") << "_";
result << "targetDevice=" << targetDevice;
return result.str();
}
size_t hiddenSize;
void DynamicBatchTest::SetUp() {
InferenceEngine::Precision netPrecision;
std::map<std::string, std::string> config;
std::tie(targetDevice, netPrecision, batch_sizes, run_async, config) = this->GetParam();
configuration.insert(config.begin(), config.end());
configuration[InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_ENABLED] = InferenceEngine::PluginConfigParams::YES;
max_batch_size = *std::max_element(batch_sizes.begin(), batch_sizes.end());
function = ngraph::builder::subgraph::makeSingleConv();
}
void DynamicBatchTest::LoadNetwork() {
cnnNetwork = InferenceEngine::CNNNetwork{function};
ConfigureNetwork();
cnnNetwork.setBatchSize(max_batch_size);
executableNetwork = core->LoadNetwork(cnnNetwork, targetDevice, configuration);
}
void DynamicBatchTest::Infer() {
inferRequest = executableNetwork.CreateInferRequest();
inputs.clear();
for (int i = 0; i < batch_sizes.size(); i++) {
auto batch_size = batch_sizes[i];
cnnNetwork.setBatchSize(batch_size);
inputs.clear();
for (const auto &input : cnnNetwork.getInputsInfo()) {
const auto &info = input.second;
auto blob = GenerateInput(*info);
inputs.push_back(blob);
}
reference_inputs.push_back(inputs);
functionRefs = ngraph::clone_function(*function);
reference_outputs.push_back(CalculateRefs());
}
for (int i = 0; i < batch_sizes.size(); i++) {
infer_requests.push_back(executableNetwork.CreateInferRequest());
auto batch_size = batch_sizes[i];
auto& infer_request = infer_requests[i];
infer_request.SetBatch(batch_size);
inputs.clear();
for (const auto &input : executableNetwork.GetInputsInfo()) {
const auto &info = input.second;
auto blob = GenerateInput(*info);
infer_request.SetBlob(info->name(), blob);
inputs.push_back(blob);
}
scaled_inputs.push_back(inputs);
for (int j = 0; j < reference_inputs[i].size(); j++) {
auto& ref = reference_inputs[i][j];
auto& actual = scaled_inputs[i][j];
auto byte_num = ref->byteSize();
auto ref_ptr = ref->buffer().as<uint8_t*>();
auto actual_ptr = actual->buffer().as<uint8_t*>();
for (int k = 0; k < byte_num; k++) {
actual_ptr[k] = ref_ptr[k];
}
}
}
for (auto& infer_request : infer_requests) {
if (run_async) {
infer_request.StartAsync();
} else {
infer_request.Infer();
}
}
if (run_async) {
for (auto& infer_request : infer_requests) {
auto status = infer_request.Wait(10000);
if (status != InferenceEngine::StatusCode::OK) {
GTEST_FAIL() << "Inference request status after wait is not OK";
}
}
}
}
void DynamicBatchTest::Validate() {
for (int i = 0; i < infer_requests.size(); i++) {
auto outputs = std::vector<InferenceEngine::Blob::Ptr>{};
for (const auto &output : executableNetwork.GetOutputsInfo()) {
const auto &name = output.first;
outputs.push_back(infer_requests[i].GetBlob(name));
}
for (int j = 0; j < reference_outputs[i].size(); j++) {
if (reference_outputs[i][j].second.size() < outputs[j]->byteSize()) {
auto actual_ptr = outputs[j]->buffer().as<uint8_t*>();
for (int k = reference_outputs[i][j].second.size(); k < outputs[j]->byteSize(); k++) actual_ptr[k] = 0;
reference_outputs[i][j].second.resize(outputs[j]->byteSize());
}
}
Compare(reference_outputs[i], outputs);
}
}
void DynamicBatchTest::Run() {
SKIP_IF_CURRENT_TEST_IS_DISABLED();
LoadNetwork();
GenerateInputs();
Infer();
Validate();
}
TEST_P(DynamicBatchTest, CompareWithRefs) {
Run();
};
} // namespace ConfigurationTestsDefinitions

5
src/tests/functional/shared_test_classes/src/base/layer_test_utils.cpp
View File

@ -406,11 +406,6 @@ void LayerTestsCommon::ConfigureInferRequest() {
auto blob = inputs[i]; auto blob = inputs[i];
inferRequest.SetBlob(info->name(), blob); inferRequest.SetBlob(info->name(), blob);
} }
if (configuration.count(InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_ENABLED) &&
configuration.count(InferenceEngine::PluginConfigParams::YES)) {
auto batchSize = executableNetwork.GetInputsInfo().begin()->second->getTensorDesc().getDims()[0] / 2;
inferRequest.SetBatch(batchSize);
}
} }
void LayerTestsCommon::Infer() { void LayerTestsCommon::Infer() {

5
src/tests/functional/shared_test_classes/src/subgraph/perm_conv_perm_concat.cpp
View File

@ -101,11 +101,6 @@ void PermConvPermConcat::Run() {
inferRequest.SetBlob(info->name(), blob); inferRequest.SetBlob(info->name(), blob);
inputs.push_back(blob); inputs.push_back(blob);
} }
if (configuration.count(InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_ENABLED) &&
configuration.count(InferenceEngine::PluginConfigParams::YES)) {
auto batchSize = cnnNetwork.getInputsInfo().begin()->second->getTensorDesc().getDims()[0] / 2;
inferRequest.SetBatch(batchSize);
}
inferRequest.Infer(); inferRequest.Infer();
Validate(); Validate();

5
src/tests/functional/shared_test_classes/src/subgraph/reshape_permute_conv_permute_reshape_act.cpp
View File

@ -97,11 +97,6 @@ namespace SubgraphTestsDefinitions {
inferRequest.SetBlob(info->name(), blob); inferRequest.SetBlob(info->name(), blob);
inputs.push_back(blob); inputs.push_back(blob);
} }
if (configuration.count(InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_ENABLED) &&
configuration.count(InferenceEngine::PluginConfigParams::YES)) {
auto batchSize = cnnNetwork.getInputsInfo().begin()->second->getTensorDesc().getDims()[0] / 2;
inferRequest.SetBatch(batchSize);
}
inferRequest.Infer(); inferRequest.Infer();
threshold = 0.1; threshold = 0.1;

2
src/tests/ie_test_utils/unit_test_utils/mocks/cpp_interfaces/interface/mock_iinfer_request_internal.hpp
View File

@ -22,10 +22,8 @@ public:
MOCK_CONST_METHOD0(GetPerformanceCounts, std::map<std::string, InferenceEngine::InferenceEngineProfileInfo>()); MOCK_CONST_METHOD0(GetPerformanceCounts, std::map<std::string, InferenceEngine::InferenceEngineProfileInfo>());
MOCK_METHOD2(SetBlob, void(const std::string&, const InferenceEngine::Blob::Ptr &)); MOCK_METHOD2(SetBlob, void(const std::string&, const InferenceEngine::Blob::Ptr &));
MOCK_METHOD1(GetBlob, InferenceEngine::Blob::Ptr(const std::string&)); MOCK_METHOD1(GetBlob, InferenceEngine::Blob::Ptr(const std::string&));
MOCK_METHOD3(SetBlob, void(const std::string&, const InferenceEngine::Blob::Ptr &, const InferenceEngine::PreProcessInfo&));
MOCK_CONST_METHOD1(GetPreProcess, const InferenceEngine::PreProcessInfo&(const std::string&)); MOCK_CONST_METHOD1(GetPreProcess, const InferenceEngine::PreProcessInfo&(const std::string&));
MOCK_METHOD1(SetCallback, void(std::function<void(std::exception_ptr)>)); MOCK_METHOD1(SetCallback, void(std::function<void(std::exception_ptr)>));
MOCK_METHOD1(SetBatch, void(int));
MOCK_METHOD0(QueryState, std::vector<InferenceEngine::IVariableStateInternal::Ptr>()); MOCK_METHOD0(QueryState, std::vector<InferenceEngine::IVariableStateInternal::Ptr>());
MOCK_METHOD0(Cancel, void()); MOCK_METHOD0(Cancel, void());
MOCK_METHOD0(StartAsyncImpl, void()); MOCK_METHOD0(StartAsyncImpl, void());

3
src/tests/ie_test_utils/unit_test_utils/mocks/mock_iinfer_request.hpp
View File

@ -34,9 +34,6 @@ public:
MOCK_METHOD(StatusCode, GetPreProcess, MOCK_METHOD(StatusCode, GetPreProcess,
(const char*, const PreProcessInfo**, ResponseDesc*), (const, noexcept)); (const char*, const PreProcessInfo**, ResponseDesc*), (const, noexcept));
MOCK_METHOD(StatusCode, SetBlob, (const char*, const Blob::Ptr&, ResponseDesc*), (noexcept)); MOCK_METHOD(StatusCode, SetBlob, (const char*, const Blob::Ptr&, ResponseDesc*), (noexcept));
MOCK_METHOD(StatusCode, SetBlob,
(const char*, const Blob::Ptr&, const PreProcessInfo&, ResponseDesc*), (noexcept));
MOCK_METHOD(StatusCode, SetBatch, (int batch, ResponseDesc*), (noexcept));
MOCK_METHOD(StatusCode, Cancel, (ResponseDesc*), (noexcept)); MOCK_METHOD(StatusCode, Cancel, (ResponseDesc*), (noexcept));
}; };