[IE TESTS] Allow to save Conformance report with unique names (#4844)

* [IE TESTS] Lock report file (between processes and threads)

* Fix condition

* t

* Fix CI

* Fix ci

* win

* Fix win

* Fix reading logic

* Fix rights

* d

* Fixes

* d

* ddd

* s

* dd

* Allow to write reports by unique name

* tp

* fix ci

* remove extra lines

inference-engine/tests/functional/plugin/conformance/test_runner/include/gflag_config.hpp

@@ -9,19 +9,25 @@
 
 static const char help_message[] = "Print a usage message.";
 static const char disable_test_config_message[] = "Optional. Ignore tests skipping rules and run all the test (except those which are skipped with DISABLED "
-                                                  "prefix)";
+                                                  "prefix). Default value is true";
-static const char extend_report_config_message[] = "Optional. Extend operation coverage report without overwriting the device results.";
+static const char extend_report_config_message[] = "Optional. Extend operation coverage report without overwriting the device results."
+                                                   "Mutually exclusive with --report_unique_name. Default value is false";
 static const char target_device_message[] = "Required. Specify the target device for Conformance Test Suite "
                                             "(the list of available devices is shown below). Default value is CPU. "
                                             "Use \"-d HETERO:<comma-separated_devices_list>\" format to specify HETERO plugin. "
                                             "The application looks for a suitable plugin for the specified device.";
 static const char input_folders_message[] = "Required. Paths to the input folders with IRs. Delimiter is `,` symbol.";
+static const char output_folder_message[] = "Optional. Paths to the output folder to save report.  Default value is \".\"";
+static const char report_unique_name_message[] = "Optional. Allow to save report with unique name (report_pid_timestamp.xml). "
+                                                 "Mutually exclusive with --extend_report. Default value is false";
 
 DEFINE_bool(h, false, help_message);
 DEFINE_string(device, "CPU", target_device_message);
 DEFINE_string(input_folders, ".", input_folders_message);
+DEFINE_string(output_folder, ".", output_folder_message);
 DEFINE_bool(disable_test_config, true, disable_test_config_message);
-DEFINE_bool(extend_report, true, extend_report_config_message);
+DEFINE_bool(extend_report, false, extend_report_config_message);
+DEFINE_bool(report_unique_name, false, report_unique_name_message);
 
 /**
 * @brief This function shows a help message
@@ -34,6 +40,8 @@ static void showUsage() {
     std::cout << "    -h                               " << help_message << std::endl;
     std::cout << "    --disable_test_config            " << disable_test_config_message << std::endl;
     std::cout << "    --extend_report                  " << extend_report_config_message << std::endl;
+    std::cout << "    --report_unique_name             " << extend_report_config_message << std::endl;
     std::cout << "    --device                         " << target_device_message << std::endl;
     std::cout << "    --input_folders \"<paths>\"        " << input_folders_message << std::endl;
+    std::cout << "    --output_folder \"<path>\"         " << output_folder_message << std::endl;
 }

inference-engine/tests/functional/plugin/conformance/test_runner/src/main.cpp

@@ -22,13 +22,16 @@ static std::vector<std::string> splitStringByDelimiter(std::string str, const st
 
 int main(int argc, char* argv[]) {
     FuncTestUtils::SkipTestsConfig::disable_tests_skipping = true;
-    LayerTestsUtils::extendReport = true;
+    LayerTestsUtils::extendReport = false;
+    LayerTestsUtils::saveReportWithUniqueName = false;
+    LayerTestsUtils::outputFolder = {"."};
+
     // Workaround for Gtest + Gflag
     std::vector<char*> argv_gflags_vec;
     int argc_gflags = 0;
     for (int i = 0; i < argc; ++i) {
         std::string arg(argv[i]);
-        if (arg.find("gtest") == std::string::npos) {
+        if (arg.find("--gtest") == std::string::npos) {
             argv_gflags_vec.emplace_back(argv[i]);
             argc_gflags++;
         }
@@ -41,12 +44,22 @@ int main(int argc, char* argv[]) {
         showUsage();
         return 0;
     }
+    if (FLAGS_extend_report && FLAGS_report_unique_name) {
+        std::cout << "Using mutually exclusive arguments: --extend_report and --report_unique_name" << std::endl;
+        return -1;
+    }
+
     if (!FLAGS_disable_test_config) {
         FuncTestUtils::SkipTestsConfig::disable_tests_skipping = false;
     }
-    if (!FLAGS_extend_report) {
+    if (FLAGS_extend_report) {
-        LayerTestsUtils::extendReport = false;
+        LayerTestsUtils::extendReport = true;
     }
+    if (FLAGS_report_unique_name) {
+        LayerTestsUtils::saveReportWithUniqueName = true;
+    }
+    LayerTestsUtils::outputFolder = {FLAGS_output_folder};
+
     // ---------------------------Initialization of Gtest env -----------------------------------------------
     ConformanceTests::targetDevice = FLAGS_device.c_str();
     ConformanceTests::IRFolderPaths = splitStringByDelimiter(FLAGS_input_folders);

inference-engine/tests/functional/plugin/shared/include/behavior/cpp_holders.hpp

@@ -15,7 +15,6 @@
 #include <common_test_utils/test_constants.hpp>
 #include <cpp/ie_cnn_network.h>
 #include "gtest/gtest.h"
-#include "common_test_utils/common_utils.hpp"
 #include "common_test_utils/test_common.hpp"
 #include "functional_test_utils/skip_tests_config.hpp"
 #include "functional_test_utils/precision_utils.hpp"

inference-engine/tests/functional/plugin/shared/src/main.cpp

@@ -9,6 +9,8 @@
 int main(int argc, char* argv[]) {
     FuncTestUtils::SkipTestsConfig::disable_tests_skipping = false;
     LayerTestsUtils::extendReport = false;
+    LayerTestsUtils::saveReportWithUniqueName = false;
+    LayerTestsUtils::outputFolder = {"."};
     bool print_custom_help = false;
     for (int i = 0; i < argc; ++i) {
         if (std::string(argv[i]) == "--disable_tests_skipping") {
@@ -17,17 +19,34 @@ int main(int argc, char* argv[]) {
             LayerTestsUtils::extendReport = true;
         } else if (std::string(argv[i]) == "--help") {
             print_custom_help = true;
+        } else if (std::string(argv[i]).find("--output_folder") != std::string::npos) {
+            LayerTestsUtils::outputFolder = {std::string(argv[i]).substr(std::string("--output_folder").length() + 1)};
+        } else if (std::string(argv[i]).find("--report_unique_name") != std::string::npos) {
+            LayerTestsUtils::saveReportWithUniqueName = true;
         }
     }
+
     if (print_custom_help) {
         std::cout << "Custom command line argument:" << std::endl;
         std::cout << "  --disable_tests_skipping" << std::endl;
         std::cout << "       Ignore tests skipping rules and run all the test" << std::endl;
         std::cout << "       (except those which are skipped with DISABLED prefix)" << std::endl;
         std::cout << "  --extend_report" << std::endl;
-        std::cout << "       Extend operation coverage report without overwriting the device results" << std::endl;
+        std::cout << "       Extend operation coverage report without overwriting the device results. " <<
+                     "Mutually exclusive with --report_unique_name" << std::endl;
+        std::cout << "  --output_folder" << std::endl;
+        std::cout << "       Folder path to save the report. Example is --output_folder=/home/user/report_folder" << std::endl;
+        std::cout << "  --report_unique_name" << std::endl;
+        std::cout << "       Allow to save report with unique name (report_pid_timestamp.xml). " <<
+                     "Mutually exclusive with --extend_report." << std::endl;
         std::cout << std::endl;
     }
+
+    if (LayerTestsUtils::saveReportWithUniqueName && LayerTestsUtils::extendReport) {
+        std::cout << "Using mutually exclusive arguments: --extend_report and --report_unique_name" << std::endl;
+        return -1;
+    }
+
     ::testing::InitGoogleTest(&argc, argv);
     ::testing::AddGlobalTestEnvironment(new LayerTestsUtils::TestEnvironment);
     auto retcode = RUN_ALL_TESTS();

inference-engine/tests/functional/shared_test_classes/include/shared_test_classes/base/layer_test_utils.hpp

@@ -32,6 +32,8 @@
 namespace LayerTestsUtils {
 
 extern bool extendReport;
+extern bool saveReportWithUniqueName;
+extern std::vector<std::string> outputFolder;
 
 // filename length limitation due to Windows constraints (max 256 characters)
 constexpr std::size_t maxFileNameLength = 140;

inference-engine/tests/functional/shared_test_classes/src/base/layer_test_utils.cpp

@@ -4,6 +4,9 @@
 
 #include <fstream>
 #include <signal.h>
+#ifdef _WIN32
+#include <process.h>
+#endif
 
 #include <transformations/serialize.hpp>
 #include <transformations/op_conversions/convert_batch_to_space.hpp>
@@ -11,6 +14,7 @@
 #include <ngraph/opsets/opset.hpp>
 #include <pugixml.hpp>
 #include <common_test_utils/file_utils.hpp>
+#include <thread>
 
 #include "ngraph/variant.hpp"
 #include "shared_test_classes/base/layer_test_utils.hpp"
@@ -20,6 +24,8 @@ namespace LayerTestsUtils {
 
 bool isReported = false;
 bool extendReport = true;
+bool saveReportWithUniqueName = false;
+std::vector<std::string> outputFolder = {"."};
 
 Summary *Summary::p_instance = nullptr;
 SummaryDestroyer Summary::destroyer;
@@ -107,6 +113,23 @@ void TestEnvironment::saveReport() {
     if (isReported) {
         return;
     }
+    if (outputFolder.size() > 1) {
+        throw std::runtime_error("Num of output folders should be 1");
+    }
+
+    std::string filename = CommonTestUtils::REPORT_FILENAME;
+    if (saveReportWithUniqueName) {
+        auto processId = std::to_string(getpid());
+        filename += "_" + processId + "_" + std::string(CommonTestUtils::GetTimestamp());
+    }
+    filename += CommonTestUtils::REPORT_EXTENSION;
+
+    if (!CommonTestUtils::directoryExists(outputFolder.front())) {
+        CommonTestUtils::createDirectoryRecursive(outputFolder.front());
+    }
+
+    std::string outputFilePath = outputFolder.front() + std::string(CommonTestUtils::FileSeparator) + filename;
+
     std::vector<ngraph::OpSet> opsets;
     opsets.push_back(ngraph::get_opset1());
     opsets.push_back(ngraph::get_opset2());
@@ -127,7 +150,7 @@ void TestEnvironment::saveReport() {
     pugi::xml_document doc;
 
     std::ifstream file;
-    file.open(CommonTestUtils::REPORT_FILENAME);
+    file.open(outputFilePath);
 
     time_t rawtime;
     struct tm *timeinfo;
@@ -141,7 +164,7 @@ void TestEnvironment::saveReport() {
 
     pugi::xml_node root;
     if (file) {
-        doc.load_file(CommonTestUtils::REPORT_FILENAME);
+        doc.load_file(outputFilePath.c_str());
         root = doc.child("report");
         //Ugly but shorter than to write predicate for find_atrribute() to update existing one
         root.remove_attribute("timestamp");
@@ -203,13 +226,14 @@ void TestEnvironment::saveReport() {
             }
         }
     }
-
+    bool result = doc.save_file(outputFilePath.c_str());
-    bool result = doc.save_file(CommonTestUtils::REPORT_FILENAME);
     if (!result) {
-        std::cout << "Failed to write report to " << CommonTestUtils::REPORT_FILENAME << "!" << std::endl;
+        std::string errMessage = "Failed to write report to " + outputFilePath;
+        throw std::runtime_error(errMessage);
     } else {
         isReported = true;
     }
+    file.close();
 }
 
 void TestEnvironment::TearDown() {

inference-engine/tests/functional/shared_test_classes/src/read_ir/generate_inputs.cpp

@@ -608,7 +608,6 @@ InferenceEngine::Blob::Ptr generate(const ngraph::op::v5::LSTMSequence node,
 InferenceEngine::Blob::Ptr generate(const ngraph::op::v5::NonMaxSuppression node,
                                     const InferenceEngine::InputInfo& info,
                                     size_t port) {
-    std::cout << "lklkllll" << std::endl;
     if (port == 1) {
         InferenceEngine::Blob::Ptr blob;
         blob = make_blob_with_precision(info.getTensorDesc());

inference-engine/tests/ie_test_utils/common_test_utils/common_utils.hpp

@@ -10,6 +10,7 @@
 #include <iterator>
 #include <vector>
 #include <set>
+#include <chrono>
 
 #include <cpp/ie_cnn_network.h>
 #include <legacy/details/ie_cnn_network_iterator.hpp>
@@ -116,4 +117,11 @@ inline T getTotal(const std::vector<T>& shape) {
     return shape.empty() ? 0 : std::accumulate(shape.cbegin(), shape.cend(), static_cast<T>(1), std::multiplies<T>());
 }
 
+inline std::string GetTimestamp() {
+    auto now = std::chrono::system_clock::now();
+    auto epoch = now.time_since_epoch();
+    auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(epoch);
+    return std::to_string(ns.count());
+}
+
 }  // namespace CommonTestUtils

inference-engine/tests/ie_test_utils/common_test_utils/file_utils.hpp

@@ -61,7 +61,7 @@ inline void createFile(const std::string& filename, const std::string& content)
 
 inline void removeFile(const std::string& path) {
     if (!path.empty()) {
-        remove(path.c_str());
+        std::remove(path.c_str());
     }
 }
 

inference-engine/tests/ie_test_utils/common_test_utils/test_common.cpp

@@ -3,6 +3,7 @@
 //
 
 #include "test_common.hpp"
+#include "common_utils.hpp"
 
 #include <threading/ie_executor_manager.hpp>
 
@@ -69,10 +70,7 @@ TestsCommon::TestsCommon() {
 }
 
 std::string TestsCommon::GetTimestamp() {
-    auto now = std::chrono::system_clock::now();
+    return CommonTestUtils::GetTimestamp();
-    auto epoch = now.time_since_epoch();
-    auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(epoch);
-    return std::to_string(ns.count());
 }
 
 std::string TestsCommon::GetTestName() const {

inference-engine/tests/ie_test_utils/common_test_utils/test_common.hpp

@@ -13,11 +13,9 @@ namespace CommonTestUtils {
 class TestsCommon : virtual public ::testing::Test {
 protected:
     TestsCommon();
-
     ~TestsCommon() override;
 
     static std::string GetTimestamp();
-
     std::string GetTestName() const;
 };
 

inference-engine/tests/ie_test_utils/common_test_utils/test_constants.hpp

@@ -17,7 +17,8 @@ const char DEVICE_MULTI[] = "MULTI";
 const char DEVICE_TEMPLATE[] = "TEMPLATE";
 const char DEVICE_HETERO[] = "HETERO";
 
-const char REPORT_FILENAME[] = "report.xml";
+const char REPORT_FILENAME[] = "report";
+const char REPORT_EXTENSION[] = ".xml";
 
 #ifdef _WIN32
     #ifdef __MINGW32__

inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py

@@ -0,0 +1,105 @@
+# Copyright (C) 2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+import xml.etree.ElementTree as ET
+from jinja2 import Environment, FileSystemLoader
+import argparse
+import os
+from datetime import datetime
+import logging
+import glob
+
+logging.basicConfig()
+logger = logging.getLogger('XmlMerger')
+logger.setLevel(logging.INFO)
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser()
+
+    input_folders_help = "Paths to folders with reports to merge"
+    output_folders_help = "Path to folder to save report"
+
+    parser.add_argument("-i", "--input_folders", help=input_folders_help, nargs="*", required=True)
+    parser.add_argument("-o", "--output_folder", help=output_folders_help, default="")
+    return parser.parse_args()
+
+
+def update_passrates(results: ET.SubElement):
+    for device in results:
+        for op in device:
+            passed_tests = 0
+            total_tests = 0
+            for attrib in op.attrib:
+                if attrib == "passrate":
+                    continue
+                if attrib == "passed":
+                    passed_tests = int(op.attrib.get(attrib))
+                total_tests += int(op.attrib.get(attrib))
+            passrate = float(passed_tests * 100 / total_tests) if passed_tests < total_tests else 100
+            op.set("passrate", str(round(passrate, 1)))
+
+
+def aggregate_test_results(results: ET.SubElement, xml_reports: list):
+    timestamp = None
+    for xml in xml_reports:
+        logger.info(f" Processing: {xml}")
+        xml_root = ET.parse(xml).getroot()
+        xml_timestamp = xml_root.get("timestamp")
+        if (timestamp is None) or (xml_timestamp < timestamp):
+            timestamp = xml_timestamp
+        for device in xml_root.find("results"):
+            device_results = results.find(device.tag)
+            if device_results is None:
+                results.append(device)
+            else:
+                device_results_report = xml_root.find("results").find(device.tag)
+                for op in device_results_report:
+                    if device_results.find(op.tag) is not None:
+                        entry = device_results.find(op.tag)
+                        for attr_name in device_results.find(op.tag).attrib:
+                            xml_value = int(op.attrib.get(attr_name))
+                            aggregated_value = int(entry.attrib.get(attr_name))
+                            device_results.find(entry.tag).set(attr_name, str(xml_value + aggregated_value))
+                    else:
+                        device_results.append(op)
+    return timestamp
+
+
+def merge_xml(input_folder_paths: list, output_folder_paths: str):
+    logger.info(f" Processing is finished")
+
+    summary = ET.Element("report")
+    results = ET.SubElement(summary, "results")
+    ops_list = ET.SubElement(summary, "ops_list")
+
+    for folder_path in input_folder_paths:
+        if not os.path.exists(folder_path):
+            logger.error(f" {folder_path} is not exist!")
+            continue
+        if not os.path.isdir(folder_path):
+            logger.error(f" {folder_path} is not a directory!")
+            continue
+
+        xml_reports = glob.glob(os.path.join(folder_path, 'report*.xml'))
+
+        xml_root = ET.parse(xml_reports[0]).getroot()
+        for op in xml_root.find("ops_list"):
+            if ops_list.find(op.tag) is None:
+                ET.SubElement(ops_list, op.tag)
+
+        timestamp = aggregate_test_results(results, xml_reports)
+        update_passrates(results)
+        summary.set("timestamp", timestamp)
+        logger.info(f" Processing is finished")
+
+        out_file_path = os.path.join(output_folder_paths, "report.xml")
+        with open(out_file_path, "w") as xml_file:
+            xml_file.write(ET.tostring(summary).decode('utf8'))
+            logger.info(f" Final report is saved to file: '{out_file_path}'")
+
+
+if __name__ == "__main__":
+    arguments = parse_arguments()
+    merge_xml(arguments.input_folders, arguments.output_folder)

inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py

@@ -1,3 +1,7 @@
+# Copyright (C) 2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
 import xml.etree.ElementTree as ET
 from jinja2 import Environment, FileSystemLoader
 import argparse
@@ -95,7 +99,6 @@ verified_operations = [
     'TopK-1',
     'TopK-3'
 ]
-
 pass_rate_avg = dict()
 general_pass_rate = dict()
 general_test_count = dict()
@@ -160,20 +163,6 @@ for device in root.find("results"):
     general_pass_rate[device.tag] = general_passed_tests[device.tag] * 100 / general_test_count[device.tag]
     general_pass_rate[device.tag] = round(float(general_pass_rate[device.tag]), 1)
 
-    if "Constant-0" in root.find("results"):
-        general_test_count[device.tag] -=  (
-            int(results[device.tag]["Constant-0"]["passed"]) + int(results[device.tag]["Constant-0"]["failed"]) +
-            int(results[device.tag]["Constant-0"]["crashed"]) + int(results[device.tag]["Constant-0"]["skipped"]))
-    if "Parameter-0" in root.find("results"):
-        general_test_count[device.tag] -= (
-                int(results[device.tag]["Parameter-0"]["passed"]) + int(results[device.tag]["Parameter-0"]["failed"]) +
-                int(results[device.tag]["Parameter-0"]["crashed"]) + int(results[device.tag]["Parameter-0"]["skipped"]))
-    if "Result-0" in root.find("results"):
-        general_test_count[device.tag] -= (
-                int(results[device.tag]["Result-0"]["passed"]) + int(results[device.tag]["Result-0"]["failed"]) +
-                int(results[device.tag]["Result-0"]["crashed"]) + int(results[device.tag]["Result-0"]["skipped"]))
-
-
 devices = results.keys()
 
 file_loader = FileSystemLoader('template')
@@ -182,7 +171,6 @@ template = env.get_template('report_template.html')
 
 res = template.render(ordered_ops=ordered_ops, devices=devices, results=results, timestamp=timestamp,
                       general_pass_rate=general_pass_rate, pass_rate_avg=pass_rate_avg,
-                      general_test_count=general_test_count,
                       verified_operations=verified_operations)
 
 with open(os.path.join(args.out, "report.html"), "w") as f:

inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html

@@ -69,12 +69,6 @@
         {% for d in devices -%}
         <td class="table-secondary" >{{general_pass_rate[d]}}%</td>
         {% endfor %}
-    </tr>
-        <tr>
-        <th class="table-secondary" scope="row">Operation test number (without Parameter-0, Constant-0 and Result-0 ops):</th>
-        {% for d in devices -%}
-        <td class="table-secondary" >{{general_test_count[d]}}</td>
-        {% endfor %}
     </tr>
     </tbody>
     <tbody>