From 8f1067f4f1866263f66c9328ed91ce46c9e9d08a Mon Sep 17 00:00:00 2001 From: myshevts Date: Thu, 18 Nov 2021 14:51:05 +0300 Subject: [PATCH] auto-batching functional test (with results check vs ref) and GPU instance for that --- .../functional/plugin/gpu/CMakeLists.txt | 2 +- .../auto_batching/auto_batching_tests.cpp | 18 +++ .../auto_batching/auto_batching_tests.hpp | 122 ++++++++++++++++++ 3 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 inference-engine/tests/functional/plugin/gpu/shared_tests_instances/auto_batching/auto_batching_tests.cpp create mode 100644 inference-engine/tests/functional/plugin/shared/include/auto_batching/auto_batching_tests.hpp diff --git a/inference-engine/tests/functional/plugin/gpu/CMakeLists.txt b/inference-engine/tests/functional/plugin/gpu/CMakeLists.txt index b0ae9365a27..8ce5bfd7aac 100644 --- a/inference-engine/tests/functional/plugin/gpu/CMakeLists.txt +++ b/inference-engine/tests/functional/plugin/gpu/CMakeLists.txt @@ -12,7 +12,7 @@ addIeTargetTest( INCLUDES ${CMAKE_CURRENT_SOURCE_DIR} DEPENDENCIES - clDNNPlugin + clDNNPlugin AutoBatchPlugin LINK_LIBRARIES funcSharedTests OpenCL diff --git a/inference-engine/tests/functional/plugin/gpu/shared_tests_instances/auto_batching/auto_batching_tests.cpp b/inference-engine/tests/functional/plugin/gpu/shared_tests_instances/auto_batching/auto_batching_tests.cpp new file mode 100644 index 00000000000..0721899c048 --- /dev/null +++ b/inference-engine/tests/functional/plugin/gpu/shared_tests_instances/auto_batching/auto_batching_tests.cpp @@ -0,0 +1,18 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include + +const std::vector num_streams{ 1, 2 }; +const std::vector get_vs_set{ true, false }; +const std::vector num_requests{ 1, 8, 16, 64 }; +const std::vector num_batch{ 1, 4, 8, 16, 32, 64, 128, 256 }; + +INSTANTIATE_TEST_SUITE_P(smoke_AutoBatching_GPU, AutoBatching_Test, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_GPU), + ::testing::ValuesIn(get_vs_set), + ::testing::ValuesIn(num_streams), + ::testing::ValuesIn(num_requests), + ::testing::ValuesIn(num_batch)), + AutoBatching_Test::getTestCaseName); diff --git a/inference-engine/tests/functional/plugin/shared/include/auto_batching/auto_batching_tests.hpp b/inference-engine/tests/functional/plugin/shared/include/auto_batching/auto_batching_tests.hpp new file mode 100644 index 00000000000..d1e25bde4ca --- /dev/null +++ b/inference-engine/tests/functional/plugin/shared/include/auto_batching/auto_batching_tests.hpp @@ -0,0 +1,122 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include +#include + +#include +#include +#include + +#include "ngraph_functions/subgraph_builders.hpp" +#include "functional_test_utils/blob_utils.hpp" + +using namespace ::testing; +using namespace InferenceEngine; + +using AutoBatchTwoNetsParams = std::tuple< + std::string, // device name + bool, // get or set blob + size_t, // number of streams + size_t, // number of requests + size_t>; // batch size> + + +class AutoBatching_Test : public CommonTestUtils::TestsCommon, + public testing::WithParamInterface { + void SetUp() override { + std::tie(device_name, use_get_blob, num_streams, num_requests, num_batch) = this->GetParam(); + fn_ptrs = {ngraph::builder::subgraph::makeSingleConv(), ngraph::builder::subgraph::makeMultiSingleConv()}; + }; +public: + static std::string getTestCaseName(const testing::TestParamInfo& obj) { + size_t streams, requests, batch; + bool use_get_blob; + std::string device_name; + std::tie(device_name, use_get_blob, streams, requests, batch) = obj.param; + return device_name + std::string(use_get_blob ? "_get_blob" : "_set_blob") + "_batch_size_" + std::to_string(batch) + + "_num_streams_" + std::to_string(streams) + "_num_req_" + std::to_string(requests); + } + +protected: + std::string device_name; + bool use_get_blob; + size_t num_streams; + size_t num_requests; + size_t num_batch; + std::vector> fn_ptrs; +}; + +TEST_P(AutoBatching_Test, compareAutoBatchingToBatch1) { + std::vector nets; + for (auto &fn_ptr : fn_ptrs) { + nets.push_back(CNNNetwork(fn_ptr)); + } + + auto ie = InferenceEngine::Core(); + std::vector outputs; + std::vector irs; + std::vector> ref; + std::vector outElementsCount; + + for (size_t i = 0; i < nets.size(); ++i) { + auto net = nets[i]; + + net.getInputsInfo().begin()->second->setLayout(Layout::NCHW); + net.getInputsInfo().begin()->second->setPrecision(Precision::FP32); + std::map config; + if (device_name.find("GPU") != std::string::npos) + config[CONFIG_KEY(GPU_THROUGHPUT_STREAMS)] = std::to_string(num_streams); + if (device_name.find("CPU") != std::string::npos) + config[CONFIG_KEY(CPU_THROUGHPUT_STREAMS)] = std::to_string(num_streams); + auto exec_net_ref = ie.LoadNetwork(net, std::string("BATCH:") + + device_name + "(" + std::to_string(num_batch) + ")", + config); + + for (int j = 0; j < num_requests; j++) { + outputs.push_back(net.getOutputsInfo().begin()->first); + + auto inf_req = exec_net_ref.CreateInferRequest(); + irs.push_back(inf_req); + + auto blob = FuncTestUtils::createAndFillBlob(net.getInputsInfo().begin()->second->getTensorDesc()); + if (use_get_blob) + InferenceEngine::blob_copy(blob, inf_req.GetBlob(net.getInputsInfo().begin()->first)); + else + inf_req.SetBlob(net.getInputsInfo().begin()->first, blob); + outElementsCount.push_back( + std::accumulate(begin(fn_ptrs[i]->get_output_shape(0)), end(fn_ptrs[i]->get_output_shape(0)), 1, + std::multiplies())); + const auto inBlob = inf_req.GetBlob(net.getInputsInfo().begin()->first); + const auto blobSize = inBlob->byteSize(); + const auto inBlobBuf = inBlob->cbuffer().as(); + std::vector inData(inBlobBuf, inBlobBuf + blobSize); + auto refOutData = ngraph::helpers::interpreterFunction(fn_ptrs[i], {inData}).front().second; + ref.push_back(refOutData); + } + } + + const int niter = 1; + for (int i = 0; i < niter; i++) { + for (auto ir : irs) { + ir.StartAsync(); + } + + for (auto ir : irs) { + ir.Wait(InferRequest::RESULT_READY); + } + } + + auto thr = FuncTestUtils::GetComparisonThreshold(InferenceEngine::Precision::FP32); + for (size_t i = 0; i < irs.size(); ++i) { + const auto &refBuffer = ref[i].data(); + ASSERT_EQ(outElementsCount[i], irs[i].GetBlob(outputs[i])->size()); + FuncTestUtils::compareRawBuffers(irs[i].GetBlob(outputs[i])->buffer().as(), + reinterpret_cast(refBuffer), outElementsCount[i], + outElementsCount[i], + thr); + } +} \ No newline at end of file