diff --git a/inference-engine/tests/functional/plugin/conformance/test_runner/include/gflag_config.hpp b/inference-engine/tests/functional/plugin/conformance/test_runner/include/gflag_config.hpp index 76d443d2636..a5c184768cc 100644 --- a/inference-engine/tests/functional/plugin/conformance/test_runner/include/gflag_config.hpp +++ b/inference-engine/tests/functional/plugin/conformance/test_runner/include/gflag_config.hpp @@ -9,19 +9,25 @@ static const char help_message[] = "Print a usage message."; static const char disable_test_config_message[] = "Optional. Ignore tests skipping rules and run all the test (except those which are skipped with DISABLED " - "prefix)"; -static const char extend_report_config_message[] = "Optional. Extend operation coverage report without overwriting the device results."; + "prefix). Default value is true"; +static const char extend_report_config_message[] = "Optional. Extend operation coverage report without overwriting the device results." + "Mutually exclusive with --report_unique_name. Default value is false"; static const char target_device_message[] = "Required. Specify the target device for Conformance Test Suite " "(the list of available devices is shown below). Default value is CPU. " "Use \"-d HETERO:\" format to specify HETERO plugin. " "The application looks for a suitable plugin for the specified device."; static const char input_folders_message[] = "Required. Paths to the input folders with IRs. Delimiter is `,` symbol."; +static const char output_folder_message[] = "Optional. Paths to the output folder to save report. Default value is \".\""; +static const char report_unique_name_message[] = "Optional. Allow to save report with unique name (report_pid_timestamp.xml). " + "Mutually exclusive with --extend_report. Default value is false"; DEFINE_bool(h, false, help_message); DEFINE_string(device, "CPU", target_device_message); DEFINE_string(input_folders, ".", input_folders_message); +DEFINE_string(output_folder, ".", output_folder_message); DEFINE_bool(disable_test_config, true, disable_test_config_message); -DEFINE_bool(extend_report, true, extend_report_config_message); +DEFINE_bool(extend_report, false, extend_report_config_message); +DEFINE_bool(report_unique_name, false, report_unique_name_message); /** * @brief This function shows a help message @@ -34,6 +40,8 @@ static void showUsage() { std::cout << " -h " << help_message << std::endl; std::cout << " --disable_test_config " << disable_test_config_message << std::endl; std::cout << " --extend_report " << extend_report_config_message << std::endl; + std::cout << " --report_unique_name " << extend_report_config_message << std::endl; std::cout << " --device " << target_device_message << std::endl; std::cout << " --input_folders \"\" " << input_folders_message << std::endl; + std::cout << " --output_folder \"\" " << output_folder_message << std::endl; } \ No newline at end of file diff --git a/inference-engine/tests/functional/plugin/conformance/test_runner/src/main.cpp b/inference-engine/tests/functional/plugin/conformance/test_runner/src/main.cpp index f188ad2a895..5a4a2877ff1 100644 --- a/inference-engine/tests/functional/plugin/conformance/test_runner/src/main.cpp +++ b/inference-engine/tests/functional/plugin/conformance/test_runner/src/main.cpp @@ -22,13 +22,16 @@ static std::vector splitStringByDelimiter(std::string str, const st int main(int argc, char* argv[]) { FuncTestUtils::SkipTestsConfig::disable_tests_skipping = true; - LayerTestsUtils::extendReport = true; + LayerTestsUtils::extendReport = false; + LayerTestsUtils::saveReportWithUniqueName = false; + LayerTestsUtils::outputFolder = {"."}; + // Workaround for Gtest + Gflag std::vector argv_gflags_vec; int argc_gflags = 0; for (int i = 0; i < argc; ++i) { std::string arg(argv[i]); - if (arg.find("gtest") == std::string::npos) { + if (arg.find("--gtest") == std::string::npos) { argv_gflags_vec.emplace_back(argv[i]); argc_gflags++; } @@ -41,12 +44,22 @@ int main(int argc, char* argv[]) { showUsage(); return 0; } + if (FLAGS_extend_report && FLAGS_report_unique_name) { + std::cout << "Using mutually exclusive arguments: --extend_report and --report_unique_name" << std::endl; + return -1; + } + if (!FLAGS_disable_test_config) { FuncTestUtils::SkipTestsConfig::disable_tests_skipping = false; } - if (!FLAGS_extend_report) { - LayerTestsUtils::extendReport = false; + if (FLAGS_extend_report) { + LayerTestsUtils::extendReport = true; } + if (FLAGS_report_unique_name) { + LayerTestsUtils::saveReportWithUniqueName = true; + } + LayerTestsUtils::outputFolder = {FLAGS_output_folder}; + // ---------------------------Initialization of Gtest env ----------------------------------------------- ConformanceTests::targetDevice = FLAGS_device.c_str(); ConformanceTests::IRFolderPaths = splitStringByDelimiter(FLAGS_input_folders); diff --git a/inference-engine/tests/functional/plugin/shared/include/behavior/cpp_holders.hpp b/inference-engine/tests/functional/plugin/shared/include/behavior/cpp_holders.hpp index d0bc8563064..2f106079538 100644 --- a/inference-engine/tests/functional/plugin/shared/include/behavior/cpp_holders.hpp +++ b/inference-engine/tests/functional/plugin/shared/include/behavior/cpp_holders.hpp @@ -15,7 +15,6 @@ #include #include #include "gtest/gtest.h" -#include "common_test_utils/common_utils.hpp" #include "common_test_utils/test_common.hpp" #include "functional_test_utils/skip_tests_config.hpp" #include "functional_test_utils/precision_utils.hpp" diff --git a/inference-engine/tests/functional/plugin/shared/src/main.cpp b/inference-engine/tests/functional/plugin/shared/src/main.cpp index 3525fcae004..26bf5ce6542 100644 --- a/inference-engine/tests/functional/plugin/shared/src/main.cpp +++ b/inference-engine/tests/functional/plugin/shared/src/main.cpp @@ -9,6 +9,8 @@ int main(int argc, char* argv[]) { FuncTestUtils::SkipTestsConfig::disable_tests_skipping = false; LayerTestsUtils::extendReport = false; + LayerTestsUtils::saveReportWithUniqueName = false; + LayerTestsUtils::outputFolder = {"."}; bool print_custom_help = false; for (int i = 0; i < argc; ++i) { if (std::string(argv[i]) == "--disable_tests_skipping") { @@ -17,17 +19,34 @@ int main(int argc, char* argv[]) { LayerTestsUtils::extendReport = true; } else if (std::string(argv[i]) == "--help") { print_custom_help = true; + } else if (std::string(argv[i]).find("--output_folder") != std::string::npos) { + LayerTestsUtils::outputFolder = {std::string(argv[i]).substr(std::string("--output_folder").length() + 1)}; + } else if (std::string(argv[i]).find("--report_unique_name") != std::string::npos) { + LayerTestsUtils::saveReportWithUniqueName = true; } } + if (print_custom_help) { std::cout << "Custom command line argument:" << std::endl; std::cout << " --disable_tests_skipping" << std::endl; std::cout << " Ignore tests skipping rules and run all the test" << std::endl; std::cout << " (except those which are skipped with DISABLED prefix)" << std::endl; std::cout << " --extend_report" << std::endl; - std::cout << " Extend operation coverage report without overwriting the device results" << std::endl; + std::cout << " Extend operation coverage report without overwriting the device results. " << + "Mutually exclusive with --report_unique_name" << std::endl; + std::cout << " --output_folder" << std::endl; + std::cout << " Folder path to save the report. Example is --output_folder=/home/user/report_folder" << std::endl; + std::cout << " --report_unique_name" << std::endl; + std::cout << " Allow to save report with unique name (report_pid_timestamp.xml). " << + "Mutually exclusive with --extend_report." << std::endl; std::cout << std::endl; } + + if (LayerTestsUtils::saveReportWithUniqueName && LayerTestsUtils::extendReport) { + std::cout << "Using mutually exclusive arguments: --extend_report and --report_unique_name" << std::endl; + return -1; + } + ::testing::InitGoogleTest(&argc, argv); ::testing::AddGlobalTestEnvironment(new LayerTestsUtils::TestEnvironment); auto retcode = RUN_ALL_TESTS(); diff --git a/inference-engine/tests/functional/shared_test_classes/include/shared_test_classes/base/layer_test_utils.hpp b/inference-engine/tests/functional/shared_test_classes/include/shared_test_classes/base/layer_test_utils.hpp index f6a26112ea5..a6230a3fee1 100644 --- a/inference-engine/tests/functional/shared_test_classes/include/shared_test_classes/base/layer_test_utils.hpp +++ b/inference-engine/tests/functional/shared_test_classes/include/shared_test_classes/base/layer_test_utils.hpp @@ -32,6 +32,8 @@ namespace LayerTestsUtils { extern bool extendReport; +extern bool saveReportWithUniqueName; +extern std::vector outputFolder; // filename length limitation due to Windows constraints (max 256 characters) constexpr std::size_t maxFileNameLength = 140; diff --git a/inference-engine/tests/functional/shared_test_classes/src/base/layer_test_utils.cpp b/inference-engine/tests/functional/shared_test_classes/src/base/layer_test_utils.cpp index 33dfc8843e6..50a7e3c3026 100644 --- a/inference-engine/tests/functional/shared_test_classes/src/base/layer_test_utils.cpp +++ b/inference-engine/tests/functional/shared_test_classes/src/base/layer_test_utils.cpp @@ -4,6 +4,9 @@ #include #include +#ifdef _WIN32 +#include +#endif #include #include @@ -11,6 +14,7 @@ #include #include #include +#include #include "ngraph/variant.hpp" #include "shared_test_classes/base/layer_test_utils.hpp" @@ -20,6 +24,8 @@ namespace LayerTestsUtils { bool isReported = false; bool extendReport = true; +bool saveReportWithUniqueName = false; +std::vector outputFolder = {"."}; Summary *Summary::p_instance = nullptr; SummaryDestroyer Summary::destroyer; @@ -107,6 +113,23 @@ void TestEnvironment::saveReport() { if (isReported) { return; } + if (outputFolder.size() > 1) { + throw std::runtime_error("Num of output folders should be 1"); + } + + std::string filename = CommonTestUtils::REPORT_FILENAME; + if (saveReportWithUniqueName) { + auto processId = std::to_string(getpid()); + filename += "_" + processId + "_" + std::string(CommonTestUtils::GetTimestamp()); + } + filename += CommonTestUtils::REPORT_EXTENSION; + + if (!CommonTestUtils::directoryExists(outputFolder.front())) { + CommonTestUtils::createDirectoryRecursive(outputFolder.front()); + } + + std::string outputFilePath = outputFolder.front() + std::string(CommonTestUtils::FileSeparator) + filename; + std::vector opsets; opsets.push_back(ngraph::get_opset1()); opsets.push_back(ngraph::get_opset2()); @@ -127,7 +150,7 @@ void TestEnvironment::saveReport() { pugi::xml_document doc; std::ifstream file; - file.open(CommonTestUtils::REPORT_FILENAME); + file.open(outputFilePath); time_t rawtime; struct tm *timeinfo; @@ -141,7 +164,7 @@ void TestEnvironment::saveReport() { pugi::xml_node root; if (file) { - doc.load_file(CommonTestUtils::REPORT_FILENAME); + doc.load_file(outputFilePath.c_str()); root = doc.child("report"); //Ugly but shorter than to write predicate for find_atrribute() to update existing one root.remove_attribute("timestamp"); @@ -203,13 +226,14 @@ void TestEnvironment::saveReport() { } } } - - bool result = doc.save_file(CommonTestUtils::REPORT_FILENAME); + bool result = doc.save_file(outputFilePath.c_str()); if (!result) { - std::cout << "Failed to write report to " << CommonTestUtils::REPORT_FILENAME << "!" << std::endl; + std::string errMessage = "Failed to write report to " + outputFilePath; + throw std::runtime_error(errMessage); } else { isReported = true; } + file.close(); } void TestEnvironment::TearDown() { diff --git a/inference-engine/tests/functional/shared_test_classes/src/read_ir/generate_inputs.cpp b/inference-engine/tests/functional/shared_test_classes/src/read_ir/generate_inputs.cpp index 816abb6046f..28cc0ff696e 100644 --- a/inference-engine/tests/functional/shared_test_classes/src/read_ir/generate_inputs.cpp +++ b/inference-engine/tests/functional/shared_test_classes/src/read_ir/generate_inputs.cpp @@ -608,7 +608,6 @@ InferenceEngine::Blob::Ptr generate(const ngraph::op::v5::LSTMSequence node, InferenceEngine::Blob::Ptr generate(const ngraph::op::v5::NonMaxSuppression node, const InferenceEngine::InputInfo& info, size_t port) { - std::cout << "lklkllll" << std::endl; if (port == 1) { InferenceEngine::Blob::Ptr blob; blob = make_blob_with_precision(info.getTensorDesc()); diff --git a/inference-engine/tests/ie_test_utils/common_test_utils/common_utils.hpp b/inference-engine/tests/ie_test_utils/common_test_utils/common_utils.hpp index 4cbb243070c..391ada9bf77 100644 --- a/inference-engine/tests/ie_test_utils/common_test_utils/common_utils.hpp +++ b/inference-engine/tests/ie_test_utils/common_test_utils/common_utils.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -116,4 +117,11 @@ inline T getTotal(const std::vector& shape) { return shape.empty() ? 0 : std::accumulate(shape.cbegin(), shape.cend(), static_cast(1), std::multiplies()); } +inline std::string GetTimestamp() { + auto now = std::chrono::system_clock::now(); + auto epoch = now.time_since_epoch(); + auto ns = std::chrono::duration_cast(epoch); + return std::to_string(ns.count()); +} + } // namespace CommonTestUtils diff --git a/inference-engine/tests/ie_test_utils/common_test_utils/file_utils.hpp b/inference-engine/tests/ie_test_utils/common_test_utils/file_utils.hpp index 7da035bb7dc..1e5534c7d8f 100644 --- a/inference-engine/tests/ie_test_utils/common_test_utils/file_utils.hpp +++ b/inference-engine/tests/ie_test_utils/common_test_utils/file_utils.hpp @@ -61,7 +61,7 @@ inline void createFile(const std::string& filename, const std::string& content) inline void removeFile(const std::string& path) { if (!path.empty()) { - remove(path.c_str()); + std::remove(path.c_str()); } } diff --git a/inference-engine/tests/ie_test_utils/common_test_utils/test_common.cpp b/inference-engine/tests/ie_test_utils/common_test_utils/test_common.cpp index b2dc8a169f0..02f5f5dca9e 100644 --- a/inference-engine/tests/ie_test_utils/common_test_utils/test_common.cpp +++ b/inference-engine/tests/ie_test_utils/common_test_utils/test_common.cpp @@ -3,6 +3,7 @@ // #include "test_common.hpp" +#include "common_utils.hpp" #include @@ -69,10 +70,7 @@ TestsCommon::TestsCommon() { } std::string TestsCommon::GetTimestamp() { - auto now = std::chrono::system_clock::now(); - auto epoch = now.time_since_epoch(); - auto ns = std::chrono::duration_cast(epoch); - return std::to_string(ns.count()); + return CommonTestUtils::GetTimestamp(); } std::string TestsCommon::GetTestName() const { diff --git a/inference-engine/tests/ie_test_utils/common_test_utils/test_common.hpp b/inference-engine/tests/ie_test_utils/common_test_utils/test_common.hpp index f74704a911d..2c4dcf49067 100644 --- a/inference-engine/tests/ie_test_utils/common_test_utils/test_common.hpp +++ b/inference-engine/tests/ie_test_utils/common_test_utils/test_common.hpp @@ -13,11 +13,9 @@ namespace CommonTestUtils { class TestsCommon : virtual public ::testing::Test { protected: TestsCommon(); - ~TestsCommon() override; static std::string GetTimestamp(); - std::string GetTestName() const; }; diff --git a/inference-engine/tests/ie_test_utils/common_test_utils/test_constants.hpp b/inference-engine/tests/ie_test_utils/common_test_utils/test_constants.hpp index 5bb9ee8f9cc..327439cfa65 100644 --- a/inference-engine/tests/ie_test_utils/common_test_utils/test_constants.hpp +++ b/inference-engine/tests/ie_test_utils/common_test_utils/test_constants.hpp @@ -17,7 +17,8 @@ const char DEVICE_MULTI[] = "MULTI"; const char DEVICE_TEMPLATE[] = "TEMPLATE"; const char DEVICE_HETERO[] = "HETERO"; -const char REPORT_FILENAME[] = "report.xml"; +const char REPORT_FILENAME[] = "report"; +const char REPORT_EXTENSION[] = ".xml"; #ifdef _WIN32 #ifdef __MINGW32__ diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py new file mode 100644 index 00000000000..a9f1d6fc2ad --- /dev/null +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py @@ -0,0 +1,105 @@ +# Copyright (C) 2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import xml.etree.ElementTree as ET +from jinja2 import Environment, FileSystemLoader +import argparse +import os +from datetime import datetime +import logging +import glob + +logging.basicConfig() +logger = logging.getLogger('XmlMerger') +logger.setLevel(logging.INFO) + + +def parse_arguments(): + parser = argparse.ArgumentParser() + + input_folders_help = "Paths to folders with reports to merge" + output_folders_help = "Path to folder to save report" + + parser.add_argument("-i", "--input_folders", help=input_folders_help, nargs="*", required=True) + parser.add_argument("-o", "--output_folder", help=output_folders_help, default="") + return parser.parse_args() + + +def update_passrates(results: ET.SubElement): + for device in results: + for op in device: + passed_tests = 0 + total_tests = 0 + for attrib in op.attrib: + if attrib == "passrate": + continue + if attrib == "passed": + passed_tests = int(op.attrib.get(attrib)) + total_tests += int(op.attrib.get(attrib)) + passrate = float(passed_tests * 100 / total_tests) if passed_tests < total_tests else 100 + op.set("passrate", str(round(passrate, 1))) + + +def aggregate_test_results(results: ET.SubElement, xml_reports: list): + timestamp = None + for xml in xml_reports: + logger.info(f" Processing: {xml}") + xml_root = ET.parse(xml).getroot() + xml_timestamp = xml_root.get("timestamp") + if (timestamp is None) or (xml_timestamp < timestamp): + timestamp = xml_timestamp + for device in xml_root.find("results"): + device_results = results.find(device.tag) + if device_results is None: + results.append(device) + else: + device_results_report = xml_root.find("results").find(device.tag) + for op in device_results_report: + if device_results.find(op.tag) is not None: + entry = device_results.find(op.tag) + for attr_name in device_results.find(op.tag).attrib: + xml_value = int(op.attrib.get(attr_name)) + aggregated_value = int(entry.attrib.get(attr_name)) + device_results.find(entry.tag).set(attr_name, str(xml_value + aggregated_value)) + else: + device_results.append(op) + return timestamp + + +def merge_xml(input_folder_paths: list, output_folder_paths: str): + logger.info(f" Processing is finished") + + summary = ET.Element("report") + results = ET.SubElement(summary, "results") + ops_list = ET.SubElement(summary, "ops_list") + + for folder_path in input_folder_paths: + if not os.path.exists(folder_path): + logger.error(f" {folder_path} is not exist!") + continue + if not os.path.isdir(folder_path): + logger.error(f" {folder_path} is not a directory!") + continue + + xml_reports = glob.glob(os.path.join(folder_path, 'report*.xml')) + + xml_root = ET.parse(xml_reports[0]).getroot() + for op in xml_root.find("ops_list"): + if ops_list.find(op.tag) is None: + ET.SubElement(ops_list, op.tag) + + timestamp = aggregate_test_results(results, xml_reports) + update_passrates(results) + summary.set("timestamp", timestamp) + logger.info(f" Processing is finished") + + out_file_path = os.path.join(output_folder_paths, "report.xml") + with open(out_file_path, "w") as xml_file: + xml_file.write(ET.tostring(summary).decode('utf8')) + logger.info(f" Final report is saved to file: '{out_file_path}'") + + +if __name__ == "__main__": + arguments = parse_arguments() + merge_xml(arguments.input_folders, arguments.output_folder) diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py index 04098baf313..0499f6d4e8e 100644 --- a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py @@ -1,3 +1,7 @@ +# Copyright (C) 2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + import xml.etree.ElementTree as ET from jinja2 import Environment, FileSystemLoader import argparse @@ -95,7 +99,6 @@ verified_operations = [ 'TopK-1', 'TopK-3' ] - pass_rate_avg = dict() general_pass_rate = dict() general_test_count = dict() @@ -160,20 +163,6 @@ for device in root.find("results"): general_pass_rate[device.tag] = general_passed_tests[device.tag] * 100 / general_test_count[device.tag] general_pass_rate[device.tag] = round(float(general_pass_rate[device.tag]), 1) - if "Constant-0" in root.find("results"): - general_test_count[device.tag] -= ( - int(results[device.tag]["Constant-0"]["passed"]) + int(results[device.tag]["Constant-0"]["failed"]) + - int(results[device.tag]["Constant-0"]["crashed"]) + int(results[device.tag]["Constant-0"]["skipped"])) - if "Parameter-0" in root.find("results"): - general_test_count[device.tag] -= ( - int(results[device.tag]["Parameter-0"]["passed"]) + int(results[device.tag]["Parameter-0"]["failed"]) + - int(results[device.tag]["Parameter-0"]["crashed"]) + int(results[device.tag]["Parameter-0"]["skipped"])) - if "Result-0" in root.find("results"): - general_test_count[device.tag] -= ( - int(results[device.tag]["Result-0"]["passed"]) + int(results[device.tag]["Result-0"]["failed"]) + - int(results[device.tag]["Result-0"]["crashed"]) + int(results[device.tag]["Result-0"]["skipped"])) - - devices = results.keys() file_loader = FileSystemLoader('template') @@ -182,7 +171,6 @@ template = env.get_template('report_template.html') res = template.render(ordered_ops=ordered_ops, devices=devices, results=results, timestamp=timestamp, general_pass_rate=general_pass_rate, pass_rate_avg=pass_rate_avg, - general_test_count=general_test_count, verified_operations=verified_operations) with open(os.path.join(args.out, "report.html"), "w") as f: diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html index c8701783329..f31a987f6c8 100644 --- a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html @@ -69,12 +69,6 @@ {% for d in devices -%} {{general_pass_rate[d]}}% {% endfor %} - - - Operation test number (without Parameter-0, Constant-0 and Result-0 ops): - {% for d in devices -%} - {{general_test_count[d]}} - {% endfor %}