openvino/src/tests/functional/plugin/gpu/concurrency/gpu_concurrency_tests.cpp

// Copyright (C) 2018-2022 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//

#include <string>
#include <utility>
#include <vector>
#include <memory>

#include "openvino/runtime/core.hpp"

#include <gpu/gpu_config.hpp>
#include <common_test_utils/test_common.hpp>
#include <functional_test_utils/plugin_cache.hpp>
#include "ngraph_functions/subgraph_builders.hpp"
#include "functional_test_utils/blob_utils.hpp"
#include "openvino/core/preprocess/pre_post_process.hpp"
#include "transformations/utils/utils.hpp"

using namespace ::testing;

using ConcurrencyTestParams = std::tuple<size_t,   // number of streams
                                         size_t>;  // number of requests

class OVConcurrencyTest : public CommonTestUtils::TestsCommon,
    public testing::WithParamInterface<ConcurrencyTestParams> {
    void SetUp() override {
        std::tie(num_streams, num_requests) = this->GetParam();
        fn_ptrs = {ngraph::builder::subgraph::makeSplitMultiConvConcat(),
                   ngraph::builder::subgraph::makeMultiSingleConv()};
    };
public:
    static std::string getTestCaseName(const testing::TestParamInfo<ConcurrencyTestParams>& obj) {
        size_t streams, requests;
        std::tie(streams, requests) = obj.param;
        return "_num_streams_" + std::to_string(streams) + "_num_req_" +
            std::to_string(requests);
    }

protected:
    size_t num_streams;
    size_t num_requests;
    std::vector<std::shared_ptr<ngraph::Function>> fn_ptrs;
};

TEST_P(OVConcurrencyTest, canInferTwoExecNets) {
    auto ie = ov::Core();

    ov::ResultVector outputs;
    std::vector<ov::InferRequest> irs;
    std::vector<std::vector<uint8_t>> ref;
    std::vector<int> outElementsCount;

    for (size_t i = 0; i < fn_ptrs.size(); ++i) {
        auto fn = fn_ptrs[i];

        auto exec_net = ie.compile_model(fn_ptrs[i], CommonTestUtils::DEVICE_GPU,
                                         {{ov::ie::PluginConfigParams::KEY_GPU_THROUGHPUT_STREAMS, std::to_string(num_streams)}});

        auto input = fn_ptrs[i]->get_parameters().at(0);
        auto output = fn_ptrs[i]->get_results().at(0);

        for (int j = 0; j < num_streams * num_requests; j++) {
            outputs.push_back(output);

            auto inf_req = exec_net.create_infer_request();
            irs.push_back(inf_req);

            auto tensor = FuncTestUtils::create_and_fill_tensor(input->get_element_type(), input->get_shape());
            inf_req.set_tensor(input, tensor);

            outElementsCount.push_back(ov::shape_size(fn_ptrs[i]->get_output_shape(0)));
            const auto in_tensor = inf_req.get_tensor(input);
            const auto tensorSize = in_tensor.get_byte_size();
            const auto inBlobBuf = static_cast<uint8_t*>(in_tensor.data());
            std::vector<uint8_t> inData(inBlobBuf, inBlobBuf + tensorSize);
            auto reOutData = ngraph::helpers::interpreterFunction(fn_ptrs[i], {inData}).front().second;
            ref.push_back(reOutData);
        }
    }

    const int niter = 10;
    for (int i = 0; i < niter; i++) {
        for (auto ir : irs) {
            ir.start_async();
        }

        for (auto ir : irs) {
            ir.wait();
        }
    }

    auto thr = FuncTestUtils::GetComparisonThreshold(InferenceEngine::Precision::FP32);
    for (size_t i = 0; i < irs.size(); ++i) {
        const auto &refBuffer = ref[i].data();
        ASSERT_EQ(outElementsCount[i], irs[i].get_tensor(outputs[i]).get_size());
        FuncTestUtils::compareRawBuffers(irs[i].get_tensor(outputs[i]).data<float>(),
                                         reinterpret_cast<const float *>(refBuffer), outElementsCount[i],
                                         outElementsCount[i],
                                         thr);
    }
}

const std::vector<size_t> num_streams{ 1, 2 };
const std::vector<size_t> num_requests{ 1, 4 };

INSTANTIATE_TEST_SUITE_P(smoke_RemoteTensor, OVConcurrencyTest,
    ::testing::Combine(::testing::ValuesIn(num_streams),
        ::testing::ValuesIn(num_requests)),
    OVConcurrencyTest::getTestCaseName);

TEST(canSwapTensorsBetweenInferRequests, inputs) {
    std::vector<std::vector<uint8_t>> ref;
    std::vector<ov::Tensor> input_tensors;
    auto fn = ngraph::builder::subgraph::makeSplitMultiConvConcat();

    auto ie = ov::Core();
    auto compiled_model = ie.compile_model(fn, CommonTestUtils::DEVICE_GPU);

    const int infer_requests_num = 2;
    ov::InferRequest infer_request1 = compiled_model.create_infer_request();
    ov::InferRequest infer_request2 = compiled_model.create_infer_request();

    input_tensors.push_back(infer_request1.get_input_tensor());
    input_tensors.push_back(infer_request2.get_input_tensor());

    auto calc_ref_results = [&](const ov::Tensor& tensor){
        const auto tensor_size = tensor.get_byte_size();
        const auto in_blob_buf = static_cast<uint8_t*>(tensor.data());
        std::vector<uint8_t> inData(in_blob_buf, in_blob_buf + tensor_size);
        auto ref_out_data = ngraph::helpers::interpreterFunction(fn, {inData}).front().second;
        ref.push_back(ref_out_data);
    };

    auto compare_results = [&](ov::Tensor& result, const uint8_t* refResult) {
        auto thr = FuncTestUtils::GetComparisonThreshold(InferenceEngine::Precision::FP32);
        ASSERT_EQ(ov::shape_size(fn->get_output_shape(0)), result.get_size());
        FuncTestUtils::compareRawBuffers(result.data<float>(),
                                        reinterpret_cast<const float *>(refResult), ov::shape_size(fn->get_output_shape(0)),
                                        ov::shape_size(fn->get_output_shape(0)),
                                        thr);
    };

    for (size_t i = 0; i < infer_requests_num; i++) {
        FuncTestUtils::fill_tensor(input_tensors[i], 10, -5, 1, i);
        calc_ref_results(input_tensors[i]);
    }

    // Swap tensors between IRs
    infer_request1.set_input_tensor(input_tensors[1]);
    infer_request2.set_input_tensor(input_tensors[0]);

    const int niter_limit = 10;
    int iter1 = 0, iter2 = 0;
    infer_request1.set_callback([&](std::exception_ptr exception_ptr) mutable {
        if (exception_ptr) {
            std::rethrow_exception(exception_ptr);
        } else {
            iter1++;
            ov::Tensor output_tensor = infer_request1.get_output_tensor();
            compare_results(output_tensor, ref[iter1 % 2].data());
            if (iter1 < niter_limit) {
                infer_request1.set_input_tensor(input_tensors[(iter1 + 1) % 2]);
                infer_request1.start_async();
            }
        }
    });

    infer_request2.set_callback([&](std::exception_ptr exception_ptr) mutable {
        if (exception_ptr) {
            std::rethrow_exception(exception_ptr);
        } else {
            iter2++;
            ov::Tensor output_tensor = infer_request2.get_output_tensor();
            compare_results(output_tensor, ref[(iter2 + 1) % 2].data());
            if (iter2 < niter_limit) {
                infer_request2.set_input_tensor(input_tensors[iter2 % 2]);
                infer_request2.start_async();
            }
        }
    });

    infer_request1.start_async();
    infer_request2.start_async();

    for (size_t i = 0; i < niter_limit; i++) {
        infer_request1.wait();
        infer_request2.wait();
    }
}

TEST(canSwapTensorsBetweenInferRequests, outputs) {
    std::vector<std::vector<uint8_t>> ref;
    std::vector<ov::Tensor> input_tensors;
    std::vector<ov::Tensor> output_tensors;
    auto fn = ngraph::builder::subgraph::makeSplitMultiConvConcat();

    auto ie = ov::Core();
    auto compiled_model = ie.compile_model(fn, CommonTestUtils::DEVICE_GPU);

    const int infer_requests_num = 2;
    ov::InferRequest infer_request1 = compiled_model.create_infer_request();
    ov::InferRequest infer_request2 = compiled_model.create_infer_request();

    input_tensors.push_back(infer_request1.get_input_tensor());
    input_tensors.push_back(infer_request2.get_input_tensor());
    output_tensors.push_back(infer_request1.get_output_tensor());
    output_tensors.push_back(infer_request2.get_output_tensor());

    auto calc_ref_results = [&](const ov::Tensor& tensor){
        const auto tensor_size = tensor.get_byte_size();
        const auto in_blob_buf = static_cast<uint8_t*>(tensor.data());
        std::vector<uint8_t> inData(in_blob_buf, in_blob_buf + tensor_size);
        auto ref_out_data = ngraph::helpers::interpreterFunction(fn, {inData}).front().second;
        ref.push_back(ref_out_data);
    };

    auto compare_results = [&](ov::Tensor& result, const uint8_t* refResult) {
        auto thr = FuncTestUtils::GetComparisonThreshold(InferenceEngine::Precision::FP32);
        ASSERT_EQ(ov::shape_size(fn->get_output_shape(0)), result.get_size());
        FuncTestUtils::compareRawBuffers(result.data<float>(),
                                        reinterpret_cast<const float *>(refResult), ov::shape_size(fn->get_output_shape(0)),
                                        ov::shape_size(fn->get_output_shape(0)),
                                        thr);
    };

    for (size_t i = 0; i < infer_requests_num; i++) {
        FuncTestUtils::fill_tensor(input_tensors[i], 10, -5, 1, i);
        calc_ref_results(input_tensors[i]);
    }

    // Swap tensors between IRs
    infer_request1.set_output_tensor(output_tensors[1]);
    infer_request2.set_output_tensor(output_tensors[0]);

    const int niter_limit = 10;
    int iter1 = 0, iter2 = 0;
    infer_request1.set_callback([&](std::exception_ptr exception_ptr) mutable {
        if (exception_ptr) {
            std::rethrow_exception(exception_ptr);
        } else {
            iter1++;
            ov::Tensor output_tensor = infer_request1.get_output_tensor();
            compare_results(output_tensor, ref[iter1 % 2].data());
            if (iter1 < niter_limit) {
                infer_request1.set_output_tensor(output_tensors[(iter1 + 1) % 2]);
                infer_request1.start_async();
            }
        }
    });

    infer_request2.set_callback([&](std::exception_ptr exception_ptr) mutable {
        if (exception_ptr) {
            std::rethrow_exception(exception_ptr);
        } else {
            iter2++;
            ov::Tensor output_tensor = infer_request2.get_output_tensor();
            compare_results(output_tensor, ref[(iter2 + 1) % 2].data());
            if (iter2 < niter_limit) {
                infer_request2.set_output_tensor(output_tensors[iter2 % 2]);
                infer_request2.start_async();
            }
        }
    });

    infer_request1.start_async();
    infer_request2.start_async();

    for (size_t i = 0; i < niter_limit; i++) {
        infer_request1.wait();
        infer_request2.wait();
    }
}