From 47dec42ca050a7f5adbd13dd53a4045d970b537a Mon Sep 17 00:00:00 2001 From: Irina Efode Date: Tue, 26 Sep 2023 14:24:56 +0400 Subject: [PATCH] [CONFORMANCE] Provide correct numbers for rel_passed && rel_all (#20035) --- .../src/read_ir/read_ir.cpp | 3 ++ .../summary/op_summary.hpp | 5 ++- .../functional_test_utils/summary/summary.hpp | 12 ------ .../src/summary/op_summary.cpp | 40 +++++++++---------- 4 files changed, 26 insertions(+), 34 deletions(-) diff --git a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir.cpp b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir.cpp index d9f161eb632..5f411e5aa37 100644 --- a/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir.cpp +++ b/src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir.cpp @@ -214,6 +214,9 @@ uint64_t clip(uint64_t n, uint64_t lower, uint64_t upper) { } void ReadIRTest::SetUp() { + // todo: find the optimal way to find TEST_P instances + // inference + query_model + import_export + summary.setDowngradeCoefficient(3); std::pair model_pair; std::tie(model_pair, targetDevice, configuration) = this->GetParam(); std::tie(path_to_model, path_to_cache) = model_pair; diff --git a/src/tests/test_utils/functional_test_utils/include/functional_test_utils/summary/op_summary.hpp b/src/tests/test_utils/functional_test_utils/include/functional_test_utils/summary/op_summary.hpp index 4dc2d08b2d2..cc97fb45cd6 100644 --- a/src/tests/test_utils/functional_test_utils/include/functional_test_utils/summary/op_summary.hpp +++ b/src/tests/test_utils/functional_test_utils/include/functional_test_utils/summary/op_summary.hpp @@ -30,16 +30,19 @@ private: static OpSummary* p_instance; static bool extractBody; std::map opsStats = {}; + unsigned short int downgrade_coefficient; std::string getOpVersion(const std::string& version); protected: - OpSummary(); + OpSummary(unsigned short int downgrade_coefficient = 1); + static OpSummary& createInstance(unsigned short int downgrade_coefficient = 1); static OpSummaryDestroyer destroyer; friend class OpSummaryDestroyer; public: static OpSummary& getInstance(); + static void setDowngradeCoefficient(unsigned short int downgrade_coefficient = 1); std::map getOPsStats() { return opsStats; diff --git a/src/tests/test_utils/functional_test_utils/include/functional_test_utils/summary/summary.hpp b/src/tests/test_utils/functional_test_utils/include/functional_test_utils/summary/summary.hpp index 71ea95a791b..a832aea0d49 100644 --- a/src/tests/test_utils/functional_test_utils/include/functional_test_utils/summary/summary.hpp +++ b/src/tests/test_utils/functional_test_utils/include/functional_test_utils/summary/summary.hpp @@ -66,18 +66,6 @@ public: std::string getDeviceName() const; - // #define IE_TEST_DEBUG - -#ifdef IE_TEST_DEBUG - void saveDebugReport(const char* className, - const char* opName, - unsigned long passed, - unsigned long failed, - unsigned long skipped, - unsigned long crashed, - unsigned long hanged); -#endif // IE_TEST_DEBUG - virtual void saveReport() {} void setReportFilename(const std::string& val); diff --git a/src/tests/test_utils/functional_test_utils/src/summary/op_summary.cpp b/src/tests/test_utils/functional_test_utils/src/summary/op_summary.cpp index 08a0d9c0a24..b895d14ab26 100644 --- a/src/tests/test_utils/functional_test_utils/src/summary/op_summary.cpp +++ b/src/tests/test_utils/functional_test_utils/src/summary/op_summary.cpp @@ -27,18 +27,30 @@ void OpSummaryDestroyer::initialize(OpSummary* p) { p_instance = p; } -OpSummary::OpSummary() { +OpSummary::OpSummary(unsigned short int in_downgrade_coefficient) { reportFilename = ov::test::utils::OP_REPORT_FILENAME; + downgrade_coefficient = in_downgrade_coefficient; } -OpSummary& OpSummary::getInstance() { +OpSummary& OpSummary::createInstance(unsigned short int in_downgrade_coefficient) { if (!p_instance) { - p_instance = new OpSummary(); + p_instance = new OpSummary(in_downgrade_coefficient); destroyer.initialize(p_instance); } return *p_instance; } +OpSummary& OpSummary::getInstance() { + return createInstance(); +} + +void OpSummary::setDowngradeCoefficient(unsigned short int in_downgrade_coefficient) { + if (p_instance && p_instance->downgrade_coefficient != in_downgrade_coefficient) { + p_instance->downgrade_coefficient = in_downgrade_coefficient; + } + auto& summary_instance = createInstance(in_downgrade_coefficient); +} + void OpSummary::updateOPsStats(const ov::NodeTypeInfo& op, const PassRate::Statuses& status, double rel_influence_coef) { @@ -218,23 +230,6 @@ void OpSummary::updateOPsImplStatus(const std::shared_ptr& model, con } } -#ifdef IE_TEST_DEBUG -void Summary::saveDebugReport(const char* className, - const char* opName, - unsigned long passed, - unsigned long failed, - unsigned long skipped, - unsigned long crashed, - unsigned long hanged) { - std::string outputFilePath = "./part_report.txt"; - std::ofstream file; - file.open(outputFilePath, std::ios_base::app); - file << className << ' ' << opName << ' ' << passed << ' ' << failed << ' ' << skipped << ' ' << crashed << ' ' - << hanged << '\n'; - file.close(); -} -#endif // IE_TEST_DEBUG - void OpSummary::saveReport() { if (isReported) { return; @@ -316,7 +311,10 @@ void OpSummary::saveReport() { pugi::xml_node resultsNode = root.child("results"); pugi::xml_node currentDeviceNode = resultsNode.append_child(summary.deviceName.c_str()); std::unordered_set opList; - for (const auto& it : stats) { + for (auto& it : stats) { + it.second.rel_passed /= downgrade_coefficient; + it.second.rel_all /= downgrade_coefficient; + std::string name = std::string(it.first.name) + "-" + getOpVersion(it.first.version_id); opList.insert(name); pugi::xml_node entry = currentDeviceNode.append_child(name.c_str());