[Plugins test] Add func tests for import/export support by plugins (#4847)

* [Plugins test] Add functional test for correct import/export support by plugins Test suite creates number of models with various precisions. For each model: - If plugin doesn't support IMPORT_EXPORT_SUPPORT metric - skip the test - Try LoadNetwork without cache enabled fails - skip the test - Do one inference If Load and Infer request is succeeded and plugin has import metric: - Load network with cache enabled. Infer request and compare outputs with original infer - Import network, perform inference. Compare outputs with previous ones * Fix Centos build warnings Myriad: reset executableNetwork before next load Myriad: Reduced time consumption for Myriad tests * Caching test suite - batch size parameter support
2021-03-18 13:03:18 +03:00
parent e455a69d07
commit 59707b7937
8 changed files with 406 additions and 6 deletions
@@ -0,0 +1,25 @@
+// Copyright (C) 2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "behavior/caching_tests.hpp"
+
+using namespace LayerTestsDefinitions;
+
+namespace {
+    static const std::vector<ngraph::element::Type> precisionsTemplate = {
+            ngraph::element::f32,
+    };
+
+    static const std::vector<std::size_t> batchSizesTemplate = {
+            1, 2
+    };
+
+    INSTANTIATE_TEST_CASE_P(smoke_CachingSupportCase_Template, LoadNetworkCacheTestBase,
+                            ::testing::Combine(
+                                    ::testing::ValuesIn(LoadNetworkCacheTestBase::getStandardFunctions()),
+                                    ::testing::ValuesIn(precisionsTemplate),
+                                    ::testing::ValuesIn(batchSizesTemplate),
+                                    ::testing::Values(CommonTestUtils::DEVICE_TEMPLATE)),
+                            LoadNetworkCacheTestBase::getTestCaseName);
+} // namespace
@@ -0,0 +1,32 @@
+// Copyright (C) 2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "behavior/caching_tests.hpp"
+
+using namespace LayerTestsDefinitions;
+
+namespace {
+    static const std::vector<ngraph::element::Type> precisionsCPU = {
+            ngraph::element::f32,
+            ngraph::element::f16,
+            ngraph::element::i32,
+            ngraph::element::i64,
+            ngraph::element::i8,
+            ngraph::element::u8,
+            ngraph::element::i16,
+            ngraph::element::u16,
+    };
+
+    static const std::vector<std::size_t> batchSizesCPU = {
+            1, 2
+    };
+
+    INSTANTIATE_TEST_CASE_P(smoke_CachingSupportCase_CPU, LoadNetworkCacheTestBase,
+                            ::testing::Combine(
+                                    ::testing::ValuesIn(LoadNetworkCacheTestBase::getStandardFunctions()),
+                                    ::testing::ValuesIn(precisionsCPU),
+                                    ::testing::ValuesIn(batchSizesCPU),
+                                    ::testing::Values(CommonTestUtils::DEVICE_CPU)),
+                            LoadNetworkCacheTestBase::getTestCaseName);
+} // namespace
@@ -0,0 +1,27 @@
+// Copyright (C) 2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "behavior/caching_tests.hpp"
+
+using namespace LayerTestsDefinitions;
+
+namespace {
+    static const std::vector<ngraph::element::Type> precisionsGNA = {
+            ngraph::element::f32,
+            ngraph::element::u8,
+            ngraph::element::i16,
+    };
+
+    static const std::vector<std::size_t> batchSizesGNA = {
+            1, 2
+    };
+
+    INSTANTIATE_TEST_CASE_P(smoke_CachingSupportCase_GNA, LoadNetworkCacheTestBase,
+                            ::testing::Combine(
+                                    ::testing::ValuesIn(LoadNetworkCacheTestBase::getStandardFunctions()),
+                                    ::testing::ValuesIn(precisionsGNA),
+                                    ::testing::ValuesIn(batchSizesGNA),
+                                    ::testing::Values(CommonTestUtils::DEVICE_GNA)),
+                            LoadNetworkCacheTestBase::getTestCaseName);
+} // namespace
@@ -0,0 +1,32 @@
+// Copyright (C) 2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "behavior/caching_tests.hpp"
+
+using namespace LayerTestsDefinitions;
+
+namespace {
+    static const std::vector<ngraph::element::Type> precisionsGPU = {
+            ngraph::element::f32,
+            ngraph::element::f16,
+            ngraph::element::i32,
+            ngraph::element::i64,
+            ngraph::element::i8,
+            ngraph::element::u8,
+            ngraph::element::i16,
+            ngraph::element::u16,
+    };
+
+    static const std::vector<std::size_t> batchSizesGPU = {
+            1, 2
+    };
+
+    INSTANTIATE_TEST_CASE_P(smoke_CachingSupportCase_GPU, LoadNetworkCacheTestBase,
+                            ::testing::Combine(
+                                    ::testing::ValuesIn(LoadNetworkCacheTestBase::getStandardFunctions()),
+                                    ::testing::ValuesIn(precisionsGPU),
+                                    ::testing::ValuesIn(batchSizesGPU),
+                                    ::testing::Values(CommonTestUtils::DEVICE_GPU)),
+                            LoadNetworkCacheTestBase::getTestCaseName);
+} // namespace
@@ -0,0 +1,29 @@
+// Copyright (C) 2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "behavior/caching_tests.hpp"
+
+using namespace LayerTestsDefinitions;
+
+namespace {
+    static const std::vector<ngraph::element::Type> precisionsMyriad = {
+            ngraph::element::f32,
+            ngraph::element::f16,
+            ngraph::element::i32,
+            ngraph::element::i8,
+            ngraph::element::u8,
+    };
+
+    static const std::vector<std::size_t> batchSizesMyriad = {
+            1, 2
+    };
+
+    INSTANTIATE_TEST_CASE_P(smoke_CachingSupportCase_Myriad, LoadNetworkCacheTestBase,
+                            ::testing::Combine(
+                                    ::testing::ValuesIn(LoadNetworkCacheTestBase::getStandardFunctions()),
+                                    ::testing::ValuesIn(precisionsMyriad),
+                                    ::testing::ValuesIn(batchSizesMyriad),
+                                    ::testing::Values(CommonTestUtils::DEVICE_MYRIAD)),
+                            LoadNetworkCacheTestBase::getTestCaseName);
+} // namespace
@@ -0,0 +1,46 @@
+// Copyright (C) 2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "shared_test_classes/base/layer_test_utils.hpp"
+#include "ngraph/function.hpp"
+
+#include <ie_core.hpp>
+#include <ie_common.h>
+
+using ngraphFunctionGenerator = std::function<std::shared_ptr<ngraph::Function>(ngraph::element::Type, std::size_t)>;
+using nGraphFunctionWithName = std::tuple<ngraphFunctionGenerator, std::string>;
+
+using loadNetworkCacheParams = std::tuple<
+        nGraphFunctionWithName, // ngraph function with friendly name
+        ngraph::element::Type,  // precision
+        std::size_t,            // batch size
+        std::string             // device name
+        >;
+
+namespace LayerTestsDefinitions {
+
+class LoadNetworkCacheTestBase : public testing::WithParamInterface<loadNetworkCacheParams>,
+                                 public LayerTestsUtils::LayerTestsCommon {
+    std::string           m_cacheFolderName;
+    std::string           m_functionName;
+    ngraph::element::Type m_precision;
+    size_t                m_batchSize;
+public:
+    static std::string getTestCaseName(testing::TestParamInfo<loadNetworkCacheParams> obj);
+    void SetUp() override;
+    void TearDown() override;
+    void Run() override;
+
+    bool importExportSupported(InferenceEngine::Core& ie) const;
+
+    // Default functions and precisions that can be used as test parameters
+    static std::vector<nGraphFunctionWithName> getStandardFunctions();
+};
+
+} // namespace LayerTestsDefinitions
@@ -0,0 +1,209 @@
+// Copyright (C) 2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <ie_core.hpp>
+#include <ie_common.h>
+#include <thread>
+
+#include "behavior/caching_tests.hpp"
+#include "common_test_utils/file_utils.hpp"
+#include "ngraph_functions/builders.hpp"
+#include "ngraph_functions/subgraph_builders.hpp"
+
+using namespace InferenceEngine::details;
+using namespace InferenceEngine;
+using namespace ::testing;
+using namespace std::placeholders;
+
+#define GTEST_COUT std::cout << "[          ] [ INFO ] "
+
+namespace LayerTestsDefinitions {
+
+static std::shared_ptr<ngraph::Function> simple_function_multiply(ngraph::element::Type type, size_t batchSize) {
+    // Create Parameter operation with static shape
+    auto data = std::make_shared<ngraph::opset6::Parameter>(type, ngraph::Shape{batchSize, 2});
+    data->set_friendly_name("Parameter");
+
+    auto constant = ngraph::opset6::Constant::create(type, ngraph::Shape{1}, {2});
+    constant->set_friendly_name("constant");
+    auto mul = std::make_shared<ngraph::opset6::Multiply>(data, constant);
+    mul->set_friendly_name("mul");
+
+    // Create Result operation
+    auto res = std::make_shared<ngraph::opset6::Result>(mul);
+    res->set_friendly_name("res");
+
+    // Create nGraph function
+    auto func = std::make_shared<ngraph::Function>(ngraph::ResultVector{res}, ngraph::ParameterVector{data});
+    func->set_friendly_name("function");
+    return func;
+}
+
+static std::shared_ptr<ngraph::Function> simple_function_relu(ngraph::element::Type type, size_t batchSize) {
+    // Create Parameter operation with static shape
+    auto data = std::make_shared<ngraph::opset6::Parameter>(type, ngraph::Shape{batchSize, 2});
+    data->set_friendly_name("Parameter");
+
+    auto relu = std::make_shared<ngraph::opset6::Relu>(data);
+    relu->set_friendly_name("relu");
+
+    // Create Result operation
+    auto res = std::make_shared<ngraph::opset6::Result>(relu);
+    res->set_friendly_name("res");
+
+    // Create nGraph function
+    auto func = std::make_shared<ngraph::Function>(ngraph::ResultVector{res}, ngraph::ParameterVector{data});
+    func->set_friendly_name("function");
+    return func;
+}
+
+std::vector<nGraphFunctionWithName> LoadNetworkCacheTestBase::getStandardFunctions() {
+    // Wrapper of most part of available builder functions
+    using ngraphFunctionIS = std::function<std::shared_ptr<ngraph::Function>(std::vector<size_t> inputShape,
+                                                                             ngraph::element::Type_t type)>;
+    auto inputShapeWrapper = [](ngraphFunctionIS fun, std::vector<size_t> inputShape) {
+        return [fun, inputShape](ngraph::element::Type type, std::size_t batchSize) {
+            auto shape = inputShape;
+            shape[0] = batchSize;
+            return fun(shape, type);
+        };
+    };
+
+    std::vector<nGraphFunctionWithName> res;
+    res.push_back(nGraphFunctionWithName { simple_function_multiply, "SimpleFunctionMultiply"});
+    res.push_back(nGraphFunctionWithName { simple_function_relu, "SimpleFunctionRelu"});
+    res.push_back(nGraphFunctionWithName {
+        inputShapeWrapper(ngraph::builder::subgraph::makeConvPoolRelu, {1, 1, 32, 32}),
+        "ConvPoolRelu"});
+    res.push_back(nGraphFunctionWithName {
+        inputShapeWrapper(ngraph::builder::subgraph::makeSplitConvConcat, {1, 4, 20, 20}),
+        "SplitConvConcat"});
+    res.push_back(nGraphFunctionWithName {
+        inputShapeWrapper(ngraph::builder::subgraph::makeKSOFunction, {1, 4, 20, 20}),
+        "KSOFunction"});
+    res.push_back(nGraphFunctionWithName { [](ngraph::element::Type type, size_t batchSize) {
+        return ngraph::builder::subgraph::makeTIwithLSTMcell(type, batchSize);
+    }, "TIwithLSTMcell1"});
+    res.push_back(nGraphFunctionWithName {
+        inputShapeWrapper(ngraph::builder::subgraph::makeSingleConv, {1, 3, 24, 24}),
+        "SingleConv"});
+    res.push_back(nGraphFunctionWithName {
+        inputShapeWrapper(ngraph::builder::subgraph::make2InputSubtract, {1, 3, 24, 24}),
+        "2InputSubtract"});
+    res.push_back(nGraphFunctionWithName {
+        inputShapeWrapper(ngraph::builder::subgraph::makeNestedSplitConvConcat, {1, 4, 20, 20}),
+        "NestedSplitConvConcat"});
+    res.push_back(nGraphFunctionWithName {
+        inputShapeWrapper(ngraph::builder::subgraph::makeSplitConvConcatInputInBranch, {1, 4, 20, 20}),
+        "SplitConvConcatInputInBranch"});
+    res.push_back(nGraphFunctionWithName {
+        inputShapeWrapper(ngraph::builder::subgraph::makeSplitConvConcatNestedInBranch, {1, 4, 20, 20}),
+        "SplitConvConcatNestedInBranch"});
+    res.push_back(nGraphFunctionWithName {
+        inputShapeWrapper(ngraph::builder::subgraph::makeSplitConvConcatNestedInBranchNestedOut, {1, 4, 20, 20}),
+        "SplitConvConcatNestedInBranchNestedOut"});
+    res.push_back(nGraphFunctionWithName {
+        inputShapeWrapper(ngraph::builder::subgraph::makeConvBias, {1, 3, 24, 24}),
+        "ConvBias"});
+    res.push_back(nGraphFunctionWithName {
+        inputShapeWrapper(ngraph::builder::subgraph::makeReadConcatSplitAssign, {1, 1, 2, 4}),
+        "ReadConcatSplitAssign"});
+
+    return res;
+}
+
+bool LoadNetworkCacheTestBase::importExportSupported(InferenceEngine::Core& ie) const {
+    std::vector<std::string> supportedMetricKeys = ie.GetMetric(targetDevice, METRIC_KEY(SUPPORTED_METRICS));
+    auto it = std::find(supportedMetricKeys.begin(), supportedMetricKeys.end(),
+                        METRIC_KEY(IMPORT_EXPORT_SUPPORT));
+    bool supported = (it != supportedMetricKeys.end()) &&
+                     ie.GetMetric(targetDevice, METRIC_KEY(IMPORT_EXPORT_SUPPORT));
+    return supported;
+}
+
+std::string LoadNetworkCacheTestBase::getTestCaseName(testing::TestParamInfo<loadNetworkCacheParams> obj) {
+    auto param = obj.param;
+    auto funcName = std::get<1>(std::get<0>(param));
+    auto precision = std::get<1>(param);
+    auto batchSize = std::get<2>(param);
+    auto deviceName = std::get<3>(param);
+    return funcName + "_" + ngraph::element::Type(precision).get_type_name() + "_batch" + std::to_string(batchSize) + "_" + deviceName;
+}
+
+void LoadNetworkCacheTestBase::SetUp() {
+    nGraphFunctionWithName funcPair;
+    std::tie(funcPair, m_precision, m_batchSize, targetDevice) = GetParam();
+    auto fGen = std::get<0>(funcPair);
+    m_functionName = std::get<1>(funcPair);
+    try {
+        function = fGen(m_precision, m_batchSize);
+    } catch (...) {
+        SKIP();
+    }
+
+    std::stringstream ss;
+    auto hash = std::hash<std::string>()(GetTestName());
+    ss << "testCache_" << std::to_string(hash) << "_" << std::this_thread::get_id() << "_" << GetTimestamp();
+    m_cacheFolderName = ss.str();
+}
+
+void LoadNetworkCacheTestBase::TearDown() {
+    CommonTestUtils::removeFilesWithExt(m_cacheFolderName, "blob");
+    std::remove(m_cacheFolderName.c_str());
+}
+
+void LoadNetworkCacheTestBase::Run() {
+    auto compareOutputs = [&](const std::vector<InferenceEngine::Blob::Ptr>& expected,
+                              const std::vector<InferenceEngine::Blob::Ptr>& actual) {
+        ASSERT_EQ(expected.size(), actual.size());
+        for (size_t i = 0; i < expected.size(); i++) {
+            const auto& expPtr = expected[i];
+            const auto& actPtr = actual[i];
+            ASSERT_NO_THROW(Compare(expPtr, actPtr));
+        }
+    };
+    if (!function) {
+        GTEST_COUT << "Can't create function " << m_functionName << " with precision " << m_precision.get_type_name() << std::endl;
+        SKIP();
+    }
+    if (!importExportSupported(*core)) {
+        GTEST_COUT << "Plugin doesn't support import and export - skipping test" << std::endl;
+        SKIP();
+    }
+    cnnNetwork = CNNNetwork{function};
+    ConfigureNetwork();
+    try {
+        executableNetwork = core->LoadNetwork(cnnNetwork, targetDevice, configuration);
+        GenerateInputs();
+        Infer();
+    } catch (InferenceEngineException &ex) {
+        GTEST_COUT << "Can't loadNetwork without cache for " << m_functionName << " with precision " << m_precision.get_type_name() << std::endl;
+        GTEST_COUT << "Exception [" << ex.what() << "]" << std::endl;
+        SKIP();
+    } catch (...) {
+        GTEST_COUT << "Can't loadNetwork without cache for " << m_functionName << " with precision " << m_precision.get_type_name() << std::endl;
+        SKIP(); // skip caching test if such network is not supported by device at all
+    }
+    auto originalOutputs = GetOutputs();
+
+    for (int i = 0; i < 2; i++) {
+        // Step 2: Load with cache. Export or import shall not throw
+        executableNetwork = {}; // Destroy network object
+        {
+            core->SetConfig({{CONFIG_KEY(CACHE_DIR), m_cacheFolderName}});
+            ASSERT_NO_THROW(executableNetwork = core->LoadNetwork(cnnNetwork, targetDevice, configuration));
+            GenerateInputs();
+            ASSERT_NO_THROW(Infer());
+        }
+        // cache is created and reused
+        ASSERT_EQ(CommonTestUtils::listFilesWithExt(m_cacheFolderName, "blob").size(), 1);
+        compareOutputs(originalOutputs, GetOutputs());
+    }
+}
+
+TEST_P(LoadNetworkCacheTestBase, CompareWithRefImpl) {
+    Run();
+}
+
+} // namespace LayerTestsDefinitions
@@ -122,12 +122,12 @@ inline std::shared_ptr<ngraph::Function> makeSplitMultiConvConcat(std::vector<si
    return fnPtr;
 }

-inline std::shared_ptr<ngraph::Function> makeTIwithLSTMcell(ngraph::element::Type_t ngPRC = ngraph::element::Type_t::f32) {
-    // That which we iterate over
-    const size_t N = 32; // Batch size
-    const size_t L = 10; // Sequence length
-    const size_t I = 8;  // Input size
-    const size_t H = 32; // Hidden size
+inline std::shared_ptr<ngraph::Function> makeTIwithLSTMcell(
+        ngraph::element::Type_t ngPRC = ngraph::element::Type_t::f32,
+        size_t N = 32,   // Batch size
+        size_t L = 10,   // Sequence length
+        size_t I = 8,    // Input size
+        size_t H = 32) { // Hidden size
    auto SENT = std::make_shared<ngraph::opset1::Parameter>(ngPRC, ngraph::Shape{N, L, I});

    auto H_init = std::make_shared<ngraph::opset1::Parameter>(ngPRC, ngraph::Shape{N, 1, H});