nGraph unit tests refactoring (#1495)

ngraph/test/backend/abc.in.cpp

@@ -16,17 +16,15 @@
 
 #include "gtest/gtest.h"
 #include "ngraph/ngraph.hpp"
-#include "ngraph/runtime/tensor.hpp"
-#include "runtime/backend.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, abc)
 {
@@ -36,30 +34,27 @@ NGRAPH_TEST(${BACKEND_NAME}, abc)
     auto C = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>((A + B) * C, ParameterVector{A, B, C});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
+    std::vector<float> a{1, 2, 3, 4};
+    std::vector<float> b{5, 6, 7, 8};
+    std::vector<float> c{9, 10, 11, 12};
 
-    // Create some tensors for input/output
-    shared_ptr<runtime::Tensor> a = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> b = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> c = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> result = backend->create_tensor(element::f32, shape);
+    // (a + b) * c
+    auto test_case_1 = test::TestCase<TestEngine>(f);
+    test_case_1.add_multiple_inputs<float>({a, b, c});
+    test_case_1.add_expected_output<float>(shape, {54, 80, 110, 144});
+    test_case_1.run();
 
-    copy_data(a, test::NDArray<float, 2>({{1, 2}, {3, 4}}).get_vector());
-    copy_data(b, test::NDArray<float, 2>({{5, 6}, {7, 8}}).get_vector());
-    copy_data(c, test::NDArray<float, 2>({{9, 10}, {11, 12}}).get_vector());
+    // (b + a) * c
+    auto test_case_2 = test::TestCase<TestEngine>(f);
+    test_case_2.add_multiple_inputs<float>({b, a, c});
+    test_case_2.add_expected_output<float>(shape, {54, 80, 110, 144});
+    test_case_2.run();
 
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a, b, c});
-    EXPECT_TRUE(test::all_close_f(read_vector<float>(result),
-                                  (test::NDArray<float, 2>({{54, 80}, {110, 144}})).get_vector()));
-
-    handle->call_with_validate({result}, {b, a, c});
-    EXPECT_TRUE(test::all_close_f(read_vector<float>(result),
-                                  (test::NDArray<float, 2>({{54, 80}, {110, 144}})).get_vector()));
-
-    handle->call_with_validate({result}, {a, c, b});
-    EXPECT_TRUE(test::all_close_f(read_vector<float>(result),
-                                  (test::NDArray<float, 2>({{50, 72}, {98, 128}})).get_vector()));
+    // (a + c) * b
+    auto test_case_3 = test::TestCase<TestEngine>(f);
+    test_case_3.add_multiple_inputs<float>({a, c, b});
+    test_case_3.add_expected_output<float>(shape, {50, 72, 98, 128});
+    test_case_3.run();
 }
 
 NGRAPH_TEST(${BACKEND_NAME}, abc_int64)
@@ -70,24 +65,25 @@ NGRAPH_TEST(${BACKEND_NAME}, abc_int64)
     auto C = make_shared<op::Parameter>(element::i64, shape);
     auto f = make_shared<Function>((A + B) * C, ParameterVector{A, B, C});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
+    std::vector<int64_t> a{1, 2, 3, 4};
+    std::vector<int64_t> b{5, 6, 7, 8};
+    std::vector<int64_t> c{9, 10, 11, 12};
 
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::i64, shape);
-    copy_data(a, vector<int64_t>{1, 2, 3, 4});
-    auto b = backend->create_tensor(element::i64, shape);
-    copy_data(b, vector<int64_t>{5, 6, 7, 8});
-    auto c = backend->create_tensor(element::i64, shape);
-    copy_data(c, vector<int64_t>{9, 10, 11, 12});
-    auto result = backend->create_tensor(element::i64, shape);
+    // (a + b) * c
+    auto test_case_1 = test::TestCase<TestEngine>(f);
+    test_case_1.add_multiple_inputs<int64_t>({a, b, c});
+    test_case_1.add_expected_output<int64_t>(shape, {54, 80, 110, 144});
+    test_case_1.run();
 
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a, b, c});
-    EXPECT_EQ((vector<int64_t>{54, 80, 110, 144}), read_vector<int64_t>(result));
+    // (b + a) * c
+    auto test_case_2 = test::TestCase<TestEngine>(f);
+    test_case_2.add_multiple_inputs<int64_t>({b, a, c});
+    test_case_2.add_expected_output<int64_t>(shape, {54, 80, 110, 144});
+    test_case_2.run();
 
-    handle->call_with_validate({result}, {b, a, c});
-    EXPECT_EQ((vector<int64_t>{54, 80, 110, 144}), read_vector<int64_t>(result));
-
-    handle->call_with_validate({result}, {a, c, b});
-    EXPECT_EQ((vector<int64_t>{50, 72, 98, 128}), read_vector<int64_t>(result));
+    // (a + c) * b
+    auto test_case_3 = test::TestCase<TestEngine>(f);
+    test_case_3.add_multiple_inputs<int64_t>({a, c, b});
+    test_case_3.add_expected_output<int64_t>(shape, {50, 72, 98, 128});
+    test_case_3.run();
 }

ngraph/test/backend/abs.in.cpp

@@ -32,19 +32,16 @@
 // clang-format on
 
 #include "gtest/gtest.h"
-#include "runtime/backend.hpp"
-#include "ngraph/runtime/tensor.hpp"
 #include "ngraph/ngraph.hpp"
-#include "util/all_close.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, abs)
 {
@@ -52,15 +49,8 @@ NGRAPH_TEST(${BACKEND_NAME}, abs)
     auto A = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(make_shared<op::Abs>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::f32, shape);
-    copy_data(a, vector<float>{1, -2, 0, -4.75f});
-    auto result = backend->create_tensor(element::f32, shape);
-
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
-    EXPECT_TRUE(test::all_close_f(
-        (vector<float>{1, 2, 0, 4.75f}), read_vector<float>(result), MIN_FLOAT_TOLERANCE_BITS));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<float>({1, -2, 0, -4.75f});
+    test_case.add_expected_output<float>(shape, {1, 2, 0, 4.75f});
+    test_case.run(MIN_FLOAT_TOLERANCE_BITS);
 }

ngraph/test/backend/acos.in.cpp

@@ -32,19 +32,16 @@
 // clang-format on
 
 #include "gtest/gtest.h"
-#include "runtime/backend.hpp"
-#include "ngraph/runtime/tensor.hpp"
 #include "ngraph/ngraph.hpp"
-#include "util/all_close.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, acos)
 {
@@ -52,25 +49,20 @@ NGRAPH_TEST(${BACKEND_NAME}, acos)
     auto A = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(make_shared<op::Acos>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::f32, shape);
-    vector<float> input{-1.f, -0.75f, -0.5f, -0.25f, -0.125f, 0.f, 0.125f, 0.25f, 0.5f, 0.75f, 1.f};
-    copy_data(a, input);
-    auto result = backend->create_tensor(element::f32, shape);
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
-    EXPECT_TRUE(test::all_close_f(vector<float>{3.14159265f,
-                                                2.41885841f,
-                                                2.09439510f,
-                                                1.82347658f,
-                                                1.69612416f,
-                                                1.57079633f,
-                                                1.44546850f,
-                                                1.31811607f,
-                                                1.04719755f,
-                                                0.72273425f,
-                                                0.00000000f},
-                                  read_vector<float>(result)));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<float>(
+        {-1.f, -0.75f, -0.5f, -0.25f, -0.125f, 0.f, 0.125f, 0.25f, 0.5f, 0.75f, 1.f});
+    test_case.add_expected_output<float>(shape,
+                                         {3.14159265f,
+                                          2.41885841f,
+                                          2.09439510f,
+                                          1.82347658f,
+                                          1.69612416f,
+                                          1.57079633f,
+                                          1.44546850f,
+                                          1.31811607f,
+                                          1.04719755f,
+                                          0.72273425f,
+                                          0.00000000f});
+    test_case.run();
 }

ngraph/test/backend/acosh.in.cpp

@@ -33,16 +33,15 @@
 
 #include "gtest/gtest.h"
 #include "ngraph/ngraph.hpp"
-#include "util/all_close.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, acosh)
 {
@@ -50,19 +49,15 @@ NGRAPH_TEST(${BACKEND_NAME}, acosh)
     auto A = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(make_shared<op::Acosh>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::f32, shape);
     vector<float> input{0.f, 1.f, -1.f, 2.f, -2.f, 3.f, -3.f, 4.f, 5.f, 10.f, 100.f};
-    copy_data(a, input);
-    auto result = backend->create_tensor(element::f32, shape);
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
     vector<float> expected;
     for (float f : input)
     {
         expected.push_back(std::acosh(f));
     }
-    EXPECT_TRUE(test::all_close_f(expected, read_vector<float>(result)));
+
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<float>(input);
+    test_case.add_expected_output<float>(shape, expected);
+    test_case.run();
 }

ngraph/test/backend/add.in.cpp

@@ -32,19 +32,16 @@
 // clang-format on
 
 #include "gtest/gtest.h"
-#include "runtime/backend.hpp"
-#include "ngraph/runtime/tensor.hpp"
 #include "ngraph/ngraph.hpp"
-#include "util/all_close.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, add)
 {
@@ -53,20 +50,13 @@ NGRAPH_TEST(${BACKEND_NAME}, add)
     auto B = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(make_shared<op::Add>(A, B), ParameterVector{A, B});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
+    vector<float> a{1, 2, 3, 4};
+    vector<float> b{5, 6, 7, 8};
 
-    // Create some tensors for input/output
-    shared_ptr<runtime::Tensor> a = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> b = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> result = backend->create_tensor(element::f32, shape);
-
-    copy_data(a, test::NDArray<float, 2>({{1, 2}, {3, 4}}).get_vector());
-    copy_data(b, test::NDArray<float, 2>({{5, 6}, {7, 8}}).get_vector());
-
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a, b});
-    EXPECT_TRUE(test::all_close_f(read_vector<float>(result),
-                                  (test::NDArray<float, 2>({{6, 8}, {10, 12}})).get_vector()));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_multiple_inputs<float>({a, b});
+    test_case.add_expected_output<float>(shape, {6, 8, 10, 12});
+    test_case.run();
 }
 
 NGRAPH_TEST(${BACKEND_NAME}, add_overload)
@@ -76,20 +66,13 @@ NGRAPH_TEST(${BACKEND_NAME}, add_overload)
     auto B = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(A + B, ParameterVector{A, B});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
+    vector<float> a{1, 2, 3, 4};
+    vector<float> b{5, 6, 7, 8};
 
-    // Create some tensors for input/output
-    shared_ptr<runtime::Tensor> a = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> b = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> result = backend->create_tensor(element::f32, shape);
-
-    copy_data(a, test::NDArray<float, 2>({{1, 2}, {3, 4}}).get_vector());
-    copy_data(b, test::NDArray<float, 2>({{5, 6}, {7, 8}}).get_vector());
-
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a, b});
-    EXPECT_TRUE(test::all_close_f(read_vector<float>(result),
-                                  (test::NDArray<float, 2>({{6, 8}, {10, 12}})).get_vector()));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_multiple_inputs<float>({a, b});
+    test_case.add_expected_output<float>(shape, {6, 8, 10, 12});
+    test_case.run();
 }
 
 NGRAPH_TEST(${BACKEND_NAME}, add_in_place)
@@ -104,18 +87,11 @@ NGRAPH_TEST(${BACKEND_NAME}, add_in_place)
 
     auto f = make_shared<Function>(T4, ParameterVector{A, B});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
+    vector<float> a{1, 2, 3, 4};
+    vector<float> b{5, 6, 7, 8};
 
-    // Create some tensors for input/output
-    shared_ptr<runtime::Tensor> a = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> b = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> result = backend->create_tensor(element::f32, shape);
-
-    copy_data(a, test::NDArray<float, 2>({{1, 2}, {3, 4}}).get_vector());
-    copy_data(b, test::NDArray<float, 2>({{5, 6}, {7, 8}}).get_vector());
-
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a, b});
-    EXPECT_TRUE(test::all_close_f(read_vector<float>(result),
-                                  (test::NDArray<float, 2>({{48, 64}, {80, 96}})).get_vector()));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_multiple_inputs<float>({a, b});
+    test_case.add_expected_output<float>(shape, {48, 64, 80, 96});
+    test_case.run();
 }

ngraph/test/backend/aliased_output.in.cpp

@@ -16,17 +16,15 @@
 
 #include "gtest/gtest.h"
 #include "ngraph/ngraph.hpp"
-#include "ngraph/runtime/tensor.hpp"
-#include "runtime/backend.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, aliased_output)
 {
@@ -38,32 +36,20 @@ NGRAPH_TEST(${BACKEND_NAME}, aliased_output)
     auto E = op::Constant::create(element::f32, shape, {1, 2, 3, 4});
     auto f = make_shared<Function>(NodeVector{C, C, D, D, C, E, E}, ParameterVector{A, B});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    shared_ptr<runtime::Tensor> a = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> b = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> out1 = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> out2 = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> out3 = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> out4 = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> out5 = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> out6 = backend->create_tensor(element::f32, shape);
-    shared_ptr<runtime::Tensor> out7 = backend->create_tensor(element::f32, shape);
-
-    copy_data(a, vector<float>{0, 1, 2, 3});
-    copy_data(b, vector<float>{1, 2, 3, 4});
+    vector<float> a{0, 1, 2, 3};
+    vector<float> b{1, 2, 3, 4};
     vector<float> expectedC{1, 3, 5, 7};
     vector<float> expectedD{0, 2, 6, 12};
     vector<float> expectedE{1, 2, 3, 4};
 
-    auto handle = backend->compile(f);
-    handle->call_with_validate({out1, out2, out3, out4, out5, out6, out7}, {a, b});
-    EXPECT_TRUE(test::all_close_f(expectedC, read_vector<float>(out1), MIN_FLOAT_TOLERANCE_BITS));
-    EXPECT_TRUE(test::all_close_f(expectedC, read_vector<float>(out2), MIN_FLOAT_TOLERANCE_BITS));
-    EXPECT_TRUE(test::all_close_f(expectedD, read_vector<float>(out3), MIN_FLOAT_TOLERANCE_BITS));
-    EXPECT_TRUE(test::all_close_f(expectedD, read_vector<float>(out4), MIN_FLOAT_TOLERANCE_BITS));
-    EXPECT_TRUE(test::all_close_f(expectedC, read_vector<float>(out5), MIN_FLOAT_TOLERANCE_BITS));
-    EXPECT_TRUE(test::all_close_f(expectedE, read_vector<float>(out6), MIN_FLOAT_TOLERANCE_BITS));
-    EXPECT_TRUE(test::all_close_f(expectedE, read_vector<float>(out7), MIN_FLOAT_TOLERANCE_BITS));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_multiple_inputs<float>({a, b});
+    test_case.add_expected_output<float>(shape, expectedC);
+    test_case.add_expected_output<float>(shape, expectedC);
+    test_case.add_expected_output<float>(shape, expectedD);
+    test_case.add_expected_output<float>(shape, expectedD);
+    test_case.add_expected_output<float>(shape, expectedC);
+    test_case.add_expected_output<float>(shape, expectedE);
+    test_case.add_expected_output<float>(shape, expectedE);
+    test_case.run(MIN_FLOAT_TOLERANCE_BITS);
 }

ngraph/test/backend/asin.in.cpp

@@ -33,18 +33,15 @@
 
 #include "gtest/gtest.h"
 #include "ngraph/ngraph.hpp"
-#include "runtime/backend.hpp"
-#include "ngraph/runtime/tensor.hpp"
-#include "util/all_close.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, asin)
 {
@@ -52,25 +49,20 @@ NGRAPH_TEST(${BACKEND_NAME}, asin)
     auto A = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(make_shared<op::Asin>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::f32, shape);
-    vector<float> input{-1.f, -0.75f, -0.5f, -0.25f, -0.125f, 0.f, 0.125f, 0.25f, 0.5f, 0.75f, 1.f};
-    copy_data(a, input);
-    auto result = backend->create_tensor(element::f32, shape);
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
-    EXPECT_TRUE(test::all_close_f(vector<float>{-1.57079633f,
-                                                -0.84806208f,
-                                                -0.52359878f,
-                                                -0.25268026f,
-                                                -0.12532783f,
-                                                0.00000000f,
-                                                0.12532783f,
-                                                0.25268026f,
-                                                0.52359878f,
-                                                0.84806208f,
-                                                1.57079633f},
-                                  read_vector<float>(result)));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<float>(
+        {-1.f, -0.75f, -0.5f, -0.25f, -0.125f, 0.f, 0.125f, 0.25f, 0.5f, 0.75f, 1.f});
+    test_case.add_expected_output<float>(shape,
+                                         {-1.57079633f,
+                                          -0.84806208f,
+                                          -0.52359878f,
+                                          -0.25268026f,
+                                          -0.12532783f,
+                                          0.00000000f,
+                                          0.12532783f,
+                                          0.25268026f,
+                                          0.52359878f,
+                                          0.84806208f,
+                                          1.57079633f});
+    test_case.run();
 }

ngraph/test/backend/asinh.in.cpp

@@ -33,16 +33,15 @@
 
 #include "gtest/gtest.h"
 #include "ngraph/ngraph.hpp"
-#include "util/all_close.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, asinh)
 {
@@ -50,19 +49,15 @@ NGRAPH_TEST(${BACKEND_NAME}, asinh)
     auto A = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(make_shared<op::Asinh>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::f32, shape);
     vector<float> input{0.f, 1.f, -1.f, 2.f, -2.f, 3.f, -3.f, 4.f, 5.f, 10.f, 100.f};
-    copy_data(a, input);
-    auto result = backend->create_tensor(element::f32, shape);
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
     vector<float> expected;
     for (float f : input)
     {
         expected.push_back(std::asinh(f));
     }
-    EXPECT_TRUE(test::all_close_f(expected, read_vector<float>(result)));
+
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<float>(input);
+    test_case.add_expected_output<float>(shape, expected);
+    test_case.run();
 }

ngraph/test/backend/atan.in.cpp

@@ -33,18 +33,15 @@
 
 #include "gtest/gtest.h"
 #include "ngraph/ngraph.hpp"
-#include "runtime/backend.hpp"
-#include "ngraph/runtime/tensor.hpp"
-#include "util/all_close.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, atan)
 {
@@ -52,25 +49,19 @@ NGRAPH_TEST(${BACKEND_NAME}, atan)
     auto A = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(make_shared<op::Atan>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::f32, shape);
-    vector<float> input{-4.f, -2.f, -1.f, -0.5f, -0.25f, 0.f, 0.25f, 0.5f, 1.f, 2.f, 4.f};
-    copy_data(a, input);
-    auto result = backend->create_tensor(element::f32, shape);
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
-    EXPECT_TRUE(test::all_close_f(vector<float>{-1.32581766f,
-                                                -1.10714872f,
-                                                -0.78539816f,
-                                                -0.46364761f,
-                                                -0.24497866f,
-                                                0.00000000f,
-                                                0.24497866f,
-                                                0.46364761f,
-                                                0.78539816f,
-                                                1.10714872f,
-                                                1.32581766f},
-                                  read_vector<float>(result)));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<float>({-4.f, -2.f, -1.f, -0.5f, -0.25f, 0.f, 0.25f, 0.5f, 1.f, 2.f, 4.f});
+    test_case.add_expected_output<float>(shape,
+                                         {-1.32581766f,
+                                          -1.10714872f,
+                                          -0.78539816f,
+                                          -0.46364761f,
+                                          -0.24497866f,
+                                          0.00000000f,
+                                          0.24497866f,
+                                          0.46364761f,
+                                          0.78539816f,
+                                          1.10714872f,
+                                          1.32581766f});
+    test_case.run();
 }

ngraph/test/backend/atanh.in.cpp

@@ -33,16 +33,15 @@
 
 #include "gtest/gtest.h"
 #include "ngraph/ngraph.hpp"
-#include "util/all_close.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, atanh)
 {
@@ -50,19 +49,15 @@ NGRAPH_TEST(${BACKEND_NAME}, atanh)
     auto A = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(make_shared<op::Atanh>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::f32, shape);
     vector<float> input{0.f, 1.f, -1.f, 2.f, -2.f, 3.f, -3.f, 4.f, 5.f, 10.f, 100.f};
-    copy_data(a, input);
-    auto result = backend->create_tensor(element::f32, shape);
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
     vector<float> expected;
     for (float f : input)
     {
         expected.push_back(std::atanh(f));
     }
-    EXPECT_TRUE(test::all_close_f(expected, read_vector<float>(result)));
+
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<float>(input);
+    test_case.add_expected_output<float>(shape, expected);
+    test_case.run();
 }

ngraph/test/backend/ceiling.in.cpp

@@ -32,19 +32,16 @@
 // clang-format on
 
 #include "gtest/gtest.h"
-#include "runtime/backend.hpp"
-#include "ngraph/runtime/tensor.hpp"
 #include "ngraph/ngraph.hpp"
-#include "util/all_close.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, ceiling)
 {
@@ -52,18 +49,10 @@ NGRAPH_TEST(${BACKEND_NAME}, ceiling)
     auto A = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(make_shared<op::Ceiling>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::f32, shape);
-    copy_data(a, vector<float>{-2.5f, -2.0f, 0.3f, 4.8f});
-    auto result = backend->create_tensor(element::f32, shape);
-
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
-    EXPECT_TRUE(test::all_close_f((vector<float>{-2.0f, -2.0f, 1.0f, 5.0f}),
-                                  read_vector<float>(result),
-                                  MIN_FLOAT_TOLERANCE_BITS));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<float>({-2.5f, -2.0f, 0.3f, 4.8f});
+    test_case.add_expected_output<float>(shape, {-2.0f, -2.0f, 1.0f, 5.0f});
+    test_case.run(MIN_FLOAT_TOLERANCE_BITS);
 }
 
 NGRAPH_TEST(${BACKEND_NAME}, ceiling_int64)
@@ -73,14 +62,10 @@ NGRAPH_TEST(${BACKEND_NAME}, ceiling_int64)
     auto A = make_shared<op::Parameter>(element::i64, shape);
     auto f = make_shared<Function>(make_shared<op::Ceiling>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    auto a = backend->create_tensor(element::i64, shape);
     vector<int64_t> expected{0, 1, 0x4000000000000001};
-    copy_data(a, expected);
-    auto result = backend->create_tensor(element::i64, shape);
 
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
-    EXPECT_EQ(expected, read_vector<int64_t>(result));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<int64_t>(expected);
+    test_case.add_expected_output<int64_t>(shape, expected);
+    test_case.run();
 }

ngraph/test/backend/cos.in.cpp

@@ -32,19 +32,16 @@
 // clang-format on
 
 #include "gtest/gtest.h"
-#include "runtime/backend.hpp"
-#include "ngraph/runtime/tensor.hpp"
 #include "ngraph/ngraph.hpp"
-#include "util/all_close.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, cos)
 {
@@ -52,25 +49,19 @@ NGRAPH_TEST(${BACKEND_NAME}, cos)
     auto A = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(make_shared<op::Cos>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::f32, shape);
-    vector<float> input{0.f, 0.25f, -0.25f, 0.5f, -0.5f, 1.f, -1.f, 2.f, -2.f, 4.f, -4.f};
-    copy_data(a, input);
-    auto result = backend->create_tensor(element::f32, shape);
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
-    EXPECT_TRUE(test::all_close_f(vector<float>{1.00000000f,
-                                                0.96891242f,
-                                                0.96891242f,
-                                                0.87758256f,
-                                                0.87758256f,
-                                                0.54030231f,
-                                                0.54030231f,
-                                                -0.41614684f,
-                                                -0.41614684f,
-                                                -0.65364362f,
-                                                -0.65364362f},
-                                  read_vector<float>(result)));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<float>({0.f, 0.25f, -0.25f, 0.5f, -0.5f, 1.f, -1.f, 2.f, -2.f, 4.f, -4.f});
+    test_case.add_expected_output<float>(shape,
+                                         {1.00000000f,
+                                          0.96891242f,
+                                          0.96891242f,
+                                          0.87758256f,
+                                          0.87758256f,
+                                          0.54030231f,
+                                          0.54030231f,
+                                          -0.41614684f,
+                                          -0.41614684f,
+                                          -0.65364362f,
+                                          -0.65364362f});
+    test_case.run();
 }

ngraph/test/backend/cosh.in.cpp

@@ -32,19 +32,16 @@
 // clang-format on
 
 #include "gtest/gtest.h"
-#include "runtime/backend.hpp"
-#include "ngraph/runtime/tensor.hpp"
 #include "ngraph/ngraph.hpp"
-#include "util/all_close.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, cosh)
 {
@@ -52,18 +49,15 @@ NGRAPH_TEST(${BACKEND_NAME}, cosh)
     auto A = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(make_shared<op::Cosh>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::f32, shape);
     vector<float> input{1.0f, 0.0f, -0.0f, -1.0f, 5.0f, -5.0f};
-    copy_data(a, input);
-    auto result = backend->create_tensor(element::f32, shape);
+    vector<float> expected;
+    for (float f : input)
+    {
+        expected.push_back(coshf(f));
+    }
 
-    std::transform(
-        input.begin(), input.end(), input.begin(), [](float x) -> float { return coshf(x); });
-
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
-    EXPECT_TRUE(test::all_close_f(input, read_vector<float>(result)));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<float>(input);
+    test_case.add_expected_output<float>(shape, expected);
+    test_case.run();
 }

ngraph/test/backend/erf.in.cpp

@@ -32,19 +32,16 @@
 // clang-format on
 
 #include "gtest/gtest.h"
-#include "runtime/backend.hpp"
-#include "ngraph/runtime/tensor.hpp"
 #include "ngraph/ngraph.hpp"
-#include "util/all_close.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, erf)
 {
@@ -52,22 +49,16 @@ NGRAPH_TEST(${BACKEND_NAME}, erf)
     auto A = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(make_shared<op::Erf>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::f32, shape);
-    copy_data(a, vector<float>{-4.0f, -3.0f, -2.0f, -1.0f, 0.0f, 1.0f, 2.0f, 3.0f});
-    auto result = backend->create_tensor(element::f32, shape);
-
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
-    EXPECT_TRUE(test::all_close_f(vector<float>{erf(-4.0f),
-                                                erf(-3.0f),
-                                                erf(-2.0f),
-                                                erf(-1.0f),
-                                                erf(0.0f),
-                                                erf(1.0f),
-                                                erf(2.0f),
-                                                erf(3.0f)},
-                                  read_vector<float>(result)));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<float>({-4.0f, -3.0f, -2.0f, -1.0f, 0.0f, 1.0f, 2.0f, 3.0f});
+    test_case.add_expected_output<float>(shape,
+                                         {erf(-4.0f),
+                                          erf(-3.0f),
+                                          erf(-2.0f),
+                                          erf(-1.0f),
+                                          erf(0.0f),
+                                          erf(1.0f),
+                                          erf(2.0f),
+                                          erf(3.0f)});
+    test_case.run();
 }

ngraph/test/backend/exp.in.cpp

@@ -32,19 +32,16 @@
 // clang-format on
 
 #include "gtest/gtest.h"
-#include "runtime/backend.hpp"
-#include "ngraph/runtime/tensor.hpp"
 #include "ngraph/ngraph.hpp"
-#include "util/all_close.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, exp)
 {
@@ -52,16 +49,9 @@ NGRAPH_TEST(${BACKEND_NAME}, exp)
     auto A = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(make_shared<op::Exp>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::f32, shape);
-    copy_data(a, vector<float>{-4, -3, -2, -1, 0, 1, 2, 3});
-    auto result = backend->create_tensor(element::f32, shape);
-
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
-    EXPECT_TRUE(test::all_close_f(
-        vector<float>{expf(-4), expf(-3), expf(-2), expf(-1), expf(0), expf(1), expf(2), expf(3)},
-        read_vector<float>(result)));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<float>({-4, -3, -2, -1, 0, 1, 2, 3});
+    test_case.add_expected_output<float>(
+        shape, {expf(-4), expf(-3), expf(-2), expf(-1), expf(0), expf(1), expf(2), expf(3)});
+    test_case.run();
 }

ngraph/test/backend/floor.in.cpp

@@ -32,19 +32,16 @@
 // clang-format on
 
 #include "gtest/gtest.h"
-#include "runtime/backend.hpp"
-#include "ngraph/runtime/tensor.hpp"
 #include "ngraph/ngraph.hpp"
-#include "util/all_close.hpp"
-#include "util/all_close_f.hpp"
-#include "util/ndarray.hpp"
+#include "util/engine/test_engines.hpp"
+#include "util/test_case.hpp"
 #include "util/test_control.hpp"
-#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
 static string s_manifest = "${MANIFEST}";
+using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME});
 
 NGRAPH_TEST(${BACKEND_NAME}, floor)
 {
@@ -52,18 +49,10 @@ NGRAPH_TEST(${BACKEND_NAME}, floor)
     auto A = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(make_shared<op::Floor>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::f32, shape);
-    copy_data(a, vector<float>{-2.5f, -2.0f, 0.3f, 4.8f});
-    auto result = backend->create_tensor(element::f32, shape);
-
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
-    EXPECT_TRUE(test::all_close_f((vector<float>{-3.0f, -2.0f, 0.0f, 4.0f}),
-                                  read_vector<float>(result),
-                                  MIN_FLOAT_TOLERANCE_BITS));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<float>({-2.5f, -2.0f, 0.3f, 4.8f});
+    test_case.add_expected_output<float>(shape, {-3.0f, -2.0f, 0.0f, 4.0f});
+    test_case.run();
 }
 
 NGRAPH_TEST(${BACKEND_NAME}, floor_int32)
@@ -72,17 +61,10 @@ NGRAPH_TEST(${BACKEND_NAME}, floor_int32)
     auto A = make_shared<op::Parameter>(element::i32, shape);
     auto f = make_shared<Function>(make_shared<op::Floor>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::i32, shape);
-    copy_data(a, vector<int32_t>{-2, -136314888, 0x40000010, 0x40000001});
-    auto result = backend->create_tensor(element::i32, shape);
-
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
-    EXPECT_EQ((vector<int32_t>{-2, -136314888, 0x40000010, 0x40000001}),
-              read_vector<int32_t>(result));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<int32_t>({-2, -136314888, 0x40000010, 0x40000001});
+    test_case.add_expected_output<int32_t>(shape, {-2, -136314888, 0x40000010, 0x40000001});
+    test_case.run();
 }
 
 NGRAPH_TEST(${BACKEND_NAME}, floor_int64)
@@ -92,14 +74,8 @@ NGRAPH_TEST(${BACKEND_NAME}, floor_int64)
     auto A = make_shared<op::Parameter>(element::i64, shape);
     auto f = make_shared<Function>(make_shared<op::Floor>(A), ParameterVector{A});
 
-    auto backend = runtime::Backend::create("${BACKEND_NAME}");
-
-    auto a = backend->create_tensor(element::i64, shape);
-    vector<int64_t> expected{0, 1, 0x4000000000000001};
-    copy_data(a, expected);
-    auto result = backend->create_tensor(element::i64, shape);
-
-    auto handle = backend->compile(f);
-    handle->call_with_validate({result}, {a});
-    EXPECT_EQ(expected, read_vector<int64_t>(result));
+    auto test_case = test::TestCase<TestEngine>(f);
+    test_case.add_input<int64_t>({0, 1, 0x4000000000000001});
+    test_case.add_expected_output<int64_t>(shape, {0, 1, 0x4000000000000001});
+    test_case.run();
 }