From 9ff8e262eeebddf9279e08aa1348dbeb1027aea0 Mon Sep 17 00:00:00 2001 From: Kamil Magierski Date: Wed, 3 Feb 2021 11:16:48 +0100 Subject: [PATCH] Dynamic Batch Test (#3936) * Dynamic Batch Test Whitespace fixes style fixes Include path fix whiteline fix Review comments implementation * MacOS build fix * Move dynamic batch test to configuration_tests * change test namespace --- .../configuration_tests/dynamic_batch.cpp | 37 ++++ .../configuration_tests/dynamic_batch.cpp | 38 ++++ .../skip_tests_config.cpp | 2 + .../configuration_tests/dynamic_batch.hpp | 44 +++++ .../src/configuration_tests/dynamic_batch.cpp | 164 ++++++++++++++++++ 5 files changed, 285 insertions(+) create mode 100644 inference-engine/tests/functional/plugin/cpu/shared_tests_instances/configuration_tests/dynamic_batch.cpp create mode 100644 inference-engine/tests/functional/plugin/gpu/shared_tests_instances/configuration_tests/dynamic_batch.cpp create mode 100644 inference-engine/tests/functional/plugin/shared/include/configuration_tests/dynamic_batch.hpp create mode 100644 inference-engine/tests/functional/plugin/shared/src/configuration_tests/dynamic_batch.cpp diff --git a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/configuration_tests/dynamic_batch.cpp b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/configuration_tests/dynamic_batch.cpp new file mode 100644 index 00000000000..34eed9884d0 --- /dev/null +++ b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/configuration_tests/dynamic_batch.cpp @@ -0,0 +1,37 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +#include +#include "common_test_utils/test_constants.hpp" + +namespace ConfigurationTestsDefinitions { +namespace { +std::vector batch_sizes = { + 1, + 5, + 9, + 16 +}; + +std::map additional_config = { +}; +} // namespace + + +INSTANTIATE_TEST_CASE_P(smoke_DynamicBatchTest_async, DynamicBatchTest, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_CPU), + ::testing::Values(InferenceEngine::Precision::FP32), + ::testing::Values(batch_sizes), + ::testing::Values(true), + ::testing::Values(additional_config)), + DynamicBatchTest::getTestCaseName); + +INSTANTIATE_TEST_CASE_P(smoke_DynamicBatchTest_sync, DynamicBatchTest, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_CPU), + ::testing::Values(InferenceEngine::Precision::FP32), + ::testing::Values(batch_sizes), + ::testing::Values(false), + ::testing::Values(additional_config)), + DynamicBatchTest::getTestCaseName); +} // namespace ConfigurationTestsDefinitions diff --git a/inference-engine/tests/functional/plugin/gpu/shared_tests_instances/configuration_tests/dynamic_batch.cpp b/inference-engine/tests/functional/plugin/gpu/shared_tests_instances/configuration_tests/dynamic_batch.cpp new file mode 100644 index 00000000000..d9390e4cee6 --- /dev/null +++ b/inference-engine/tests/functional/plugin/gpu/shared_tests_instances/configuration_tests/dynamic_batch.cpp @@ -0,0 +1,38 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +#include +#include "common_test_utils/test_constants.hpp" + +namespace ConfigurationTestsDefinitions { +namespace { +std::vector batch_sizes = { + 16, + 1, + 5, + 9, + 16 +}; + +std::map additional_config = { +}; +} // namespace + + +INSTANTIATE_TEST_CASE_P(smoke_DynamicBatchTest_async, DynamicBatchTest, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_GPU), + ::testing::Values(InferenceEngine::Precision::FP32), + ::testing::Values(batch_sizes), + ::testing::Values(true), + ::testing::Values(additional_config)), + DynamicBatchTest::getTestCaseName); + +INSTANTIATE_TEST_CASE_P(smoke_DynamicBatchTest_sync, DynamicBatchTest, + ::testing::Combine( + ::testing::Values(CommonTestUtils::DEVICE_GPU), + ::testing::Values(InferenceEngine::Precision::FP32), + ::testing::Values(batch_sizes), + ::testing::Values(false), + ::testing::Values(additional_config)), + DynamicBatchTest::getTestCaseName); +} // namespace ConfigurationTestsDefinitions diff --git a/inference-engine/tests/functional/plugin/gpu/shared_tests_instances/skip_tests_config.cpp b/inference-engine/tests/functional/plugin/gpu/shared_tests_instances/skip_tests_config.cpp index 5c0c571e9ac..c8082550318 100644 --- a/inference-engine/tests/functional/plugin/gpu/shared_tests_instances/skip_tests_config.cpp +++ b/inference-engine/tests/functional/plugin/gpu/shared_tests_instances/skip_tests_config.cpp @@ -51,5 +51,7 @@ std::vector disabledTestPatterns() { // INT8 StridedSlice not supported R"(.*(LPT/StridedSliceTransformation).*)", + // TODO: Issue: 47219 + R"(.*DynamicBatchTest.*)", }; } diff --git a/inference-engine/tests/functional/plugin/shared/include/configuration_tests/dynamic_batch.hpp b/inference-engine/tests/functional/plugin/shared/include/configuration_tests/dynamic_batch.hpp new file mode 100644 index 00000000000..973ae5b6e4b --- /dev/null +++ b/inference-engine/tests/functional/plugin/shared/include/configuration_tests/dynamic_batch.hpp @@ -0,0 +1,44 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include +#include + +#include "shared_test_classes/base/layer_test_utils.hpp" + +namespace ConfigurationTestsDefinitions { +typedef std::tuple< + std::string, // Device + InferenceEngine::Precision, // Network precision + std::vector, // Batch sizes + bool, // Asynchronous execution + std::map // Additional configuration +> dynamicBatchTestParams; + +class DynamicBatchTest : public LayerTestsUtils::LayerTestsCommon, + public testing::WithParamInterface { +private: + bool run_async = false; + size_t max_batch_size = 0; + std::vector batch_sizes; + std::vector> reference_inputs; + std::vector> scaled_inputs; + std::vector>> reference_outputs; + std::vector> actual_outputs; + std::vector infer_requests; +protected: + void SetUp() override; + void Run() override; + + void LoadNetwork(); + void Infer() override; + void Validate() override; +public: + static std::string getTestCaseName(const testing::TestParamInfo &obj); +}; +} // namespace ConfigurationTestsDefinitions diff --git a/inference-engine/tests/functional/plugin/shared/src/configuration_tests/dynamic_batch.cpp b/inference-engine/tests/functional/plugin/shared/src/configuration_tests/dynamic_batch.cpp new file mode 100644 index 00000000000..f1144bab8af --- /dev/null +++ b/inference-engine/tests/functional/plugin/shared/src/configuration_tests/dynamic_batch.cpp @@ -0,0 +1,164 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include +#include +#include +#include +#include + +#include "ie_core.hpp" + +#include "ie_transformations.hpp" +#include "common_test_utils/common_utils.hpp" +#include "functional_test_utils/blob_utils.hpp" +#include "functional_test_utils/precision_utils.hpp" +#include "functional_test_utils/plugin_cache.hpp" +#include "functional_test_utils/skip_tests_config.hpp" +#include "ngraph_functions/utils/ngraph_helpers.hpp" +#include "ngraph_functions/builders.hpp" + +#include +#include "transformations/control_flow/unroll_tensor_iterator.hpp" +#include "configuration_tests/dynamic_batch.hpp" + +#include "ngraph_functions/subgraph_builders.hpp" + +namespace ConfigurationTestsDefinitions { + + std::string DynamicBatchTest::getTestCaseName(const testing::TestParamInfo &obj) { + std::string targetDevice; + InferenceEngine::Precision netPrecision; + std::vector batchSizes; + bool runAsync; + std::map config; + std::tie(targetDevice, netPrecision, batchSizes, runAsync, config) = obj.param; + std::ostringstream result; + + result << "netPrecision=" << netPrecision.name() << "_"; + result << "BS=" << CommonTestUtils::vec2str(batchSizes) << "_"; + result << std::string(runAsync ? "Async" : "Sync") << "_"; + result << "targetDevice=" << targetDevice; + return result.str(); + } + + size_t hiddenSize; + + + void DynamicBatchTest::SetUp() { + InferenceEngine::Precision netPrecision; + std::map config; + std::tie(targetDevice, netPrecision, batch_sizes, run_async, config) = this->GetParam(); + configuration.insert(config.begin(), config.end()); + configuration[InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_ENABLED] = InferenceEngine::PluginConfigParams::YES; + + max_batch_size = *std::max_element(batch_sizes.begin(), batch_sizes.end()); + + function = ngraph::builder::subgraph::makeSingleConv(); + } + + void DynamicBatchTest::LoadNetwork() { + cnnNetwork = InferenceEngine::CNNNetwork{function}; + ConfigureNetwork(); + cnnNetwork.setBatchSize(max_batch_size); + executableNetwork = core->LoadNetwork(cnnNetwork, targetDevice, configuration); + } + + void DynamicBatchTest::Infer() { + inferRequest = executableNetwork.CreateInferRequest(); + inputs.clear(); + + for (int i = 0; i < batch_sizes.size(); i++) { + auto batch_size = batch_sizes[i]; + + cnnNetwork.setBatchSize(batch_size); + inputs.clear(); + for (const auto &input : cnnNetwork.getInputsInfo()) { + const auto &info = input.second; + auto blob = GenerateInput(*info); + inputs.push_back(blob); + } + reference_inputs.push_back(inputs); + reference_outputs.push_back(CalculateRefs()); + } + + for (int i = 0; i < batch_sizes.size(); i++) { + infer_requests.push_back(executableNetwork.CreateInferRequest()); + auto batch_size = batch_sizes[i]; + + auto& infer_request = infer_requests[i]; + infer_request.SetBatch(batch_size); + + inputs.clear(); + for (const auto &input : executableNetwork.GetInputsInfo()) { + const auto &info = input.second; + auto blob = GenerateInput(*info); + infer_request.SetBlob(info->name(), blob); + inputs.push_back(blob); + } + + scaled_inputs.push_back(inputs); + + for (int j = 0; j < reference_inputs[i].size(); j++) { + auto& ref = reference_inputs[i][j]; + auto& actual = scaled_inputs[i][j]; + + auto byte_num = ref->byteSize(); + auto ref_ptr = ref->buffer().as(); + auto actual_ptr = actual->buffer().as(); + + for (int k = 0; k < byte_num; k++) { + actual_ptr[k] = ref_ptr[k]; + } + } + } + + for (auto& infer_request : infer_requests) { + if (run_async) { + infer_request.StartAsync(); + } else { + infer_request.Infer(); + } + } + + if (run_async) { + for (auto& infer_request : infer_requests) { + auto status = infer_request.Wait(10000); + if (status != InferenceEngine::StatusCode::OK) { + GTEST_FAIL() << "Inference request status after wait is not OK"; + } + } + } + } + + void DynamicBatchTest::Validate() { + for (int i = 0; i < infer_requests.size(); i++) { + auto outputs = std::vector{}; + for (const auto &output : executableNetwork.GetOutputsInfo()) { + const auto &name = output.first; + outputs.push_back(infer_requests[i].GetBlob(name)); + } + for (int j = 0; j < reference_outputs[i].size(); j++) { + if (reference_outputs[i][j].size() < outputs[j]->byteSize()) { + auto actual_ptr = outputs[j]->buffer().as(); + for (int k = reference_outputs[i][j].size(); k < outputs[j]->byteSize(); k++) actual_ptr[k] = 0; + reference_outputs[i][j].resize(outputs[j]->byteSize()); + } + } + Compare(reference_outputs[i], outputs); + } + } + + void DynamicBatchTest::Run() { + SKIP_IF_CURRENT_TEST_IS_DISABLED(); + LoadNetwork(); + Infer(); + Validate(); + } + + TEST_P(DynamicBatchTest, CompareWithRefs) { + Run(); + }; +} // namespace ConfigurationTestsDefinitions