IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CONFORMANCE] Change downgrade coefficient logic for conformance reports (#21475)

Browse Source 
* [CONFORMANCE] Change downgrade coefficient logic for conformance reports

* Remove extra reports
This commit is contained in:
Irina Efode 2023-12-06 13:02:04 +04:00 committed by GitHub
parent d217847714
commit bbfe22732f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 12 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/tests/functional/plugin/conformance/test_runner/op_conformance_runner/src/read_ir/read_ir.cpp
View File

@ -122,9 +122,6 @@ uint64_t clip(uint64_t n, uint64_t lower, uint64_t upper) {
}
void ReadIRTest::SetUp() {
// todo: find the optimal way to find TEST_P instances
// inference + query_model + import_export
summary.setDowngradeCoefficient(3);
std::pair<std::string, std::string> model_pair;
std::tie(model_pair, targetDevice, configuration) = this->GetParam();
std::tie(path_to_model, path_to_ref_tensor) = model_pair;

6
src/tests/test_utils/functional_test_utils/include/functional_test_utils/summary/op_summary.hpp
View File

@ -30,19 +30,17 @@ private:
static OpSummary* p_instance;
static bool extractBody;
std::map<ov::NodeTypeInfo, PassRate> opsStats = {};
unsigned short int downgrade_coefficient;
std::string get_opset_number(const std::string& opset_full_name);
protected:
OpSummary(unsigned short int downgrade_coefficient = 1);
static OpSummary& createInstance(unsigned short int downgrade_coefficient = 1);
OpSummary();
static OpSummary& createInstance();
static OpSummaryDestroyer destroyer;
friend class OpSummaryDestroyer;
public:
static OpSummary& getInstance();
static void setDowngradeCoefficient(unsigned short int downgrade_coefficient = 1);
std::map<ov::NodeTypeInfo, PassRate> getOPsStats() {
return opsStats;

12
src/tests/test_utils/functional_test_utils/layer_tests_summary/utils/stat_update_utils.py
View File

@ -21,7 +21,8 @@ def update_passrates(results: ET.SubElement, rel_weights={}):
passed_tests = 0
total_tests = 0
rel_passed_tests = None
rel_all_tests = None
rel_all_tests_expected = None
rel_all_tests_actual = None
for attrib in op.attrib:
if attrib == "passrate" or attrib == "relative_passrate":
continue
@ -34,13 +35,14 @@ def update_passrates(results: ET.SubElement, rel_weights={}):
continue
elif attrib == "relative_all":
if op.tag in rel_weights.keys():
rel_all_tests = rel_weights[op.tag]
else:
rel_all_tests = float(op.attrib.get(attrib))
rel_all_tests_expected = rel_weights[op.tag]
rel_all_tests_actual = float(op.attrib.get(attrib))
continue
total_tests += int(float(op.attrib.get(attrib)))
passrate = float(passed_tests * 100 / total_tests) if total_tests != 0 else 0
rel_passrate = float(rel_passed_tests * 100 / rel_all_tests) if rel_all_tests != None and rel_all_tests != 0 else 0
rel_all_tests = rel_all_tests_actual if rel_all_tests_expected is None else rel_all_tests_expected
k = 1 if rel_all_tests_expected is None else round(rel_all_tests_actual / rel_all_tests_expected)
rel_passrate = float(rel_passed_tests * 100 / (k * rel_all_tests)) if rel_all_tests != None and rel_all_tests != 0 else 0
op.set("passrate", "%.2f"%passrate)
if rel_all_tests != None and rel_passed_tests != None:
op.set("relative_passrate", "%.2f"%rel_passrate)

17
src/tests/test_utils/functional_test_utils/src/summary/op_summary.cpp
View File

@ -28,14 +28,13 @@ void OpSummaryDestroyer::initialize(OpSummary* p) {
p_instance = p;
}
OpSummary::OpSummary(unsigned short int in_downgrade_coefficient) {
OpSummary::OpSummary() {
reportFilename = ov::test::utils::OP_REPORT_FILENAME;
downgrade_coefficient = in_downgrade_coefficient;
}
OpSummary& OpSummary::createInstance(unsigned short int in_downgrade_coefficient) {
OpSummary& OpSummary::createInstance() {
if (!p_instance) {
p_instance = new OpSummary(in_downgrade_coefficient);
p_instance = new OpSummary();
destroyer.initialize(p_instance);
}
return *p_instance;
@ -45,13 +44,6 @@ OpSummary& OpSummary::getInstance() {
return createInstance();
}
void OpSummary::setDowngradeCoefficient(unsigned short int in_downgrade_coefficient) {
if (p_instance && p_instance->downgrade_coefficient != in_downgrade_coefficient) {
p_instance->downgrade_coefficient = in_downgrade_coefficient;
}
auto& summary_instance = createInstance(in_downgrade_coefficient);
}
void OpSummary::updateOPsStats(const ov::NodeTypeInfo& op,
const PassRate::Statuses& status,
double rel_influence_coef) {
@ -313,9 +305,6 @@ void OpSummary::saveReport() {
pugi::xml_node currentDeviceNode = resultsNode.append_child(summary.deviceName.c_str());
std::unordered_set<std::string> opList;
for (auto& it : stats) {
it.second.rel_passed /= downgrade_coefficient;
it.second.rel_all /= downgrade_coefficient;
std::string name = functional::get_node_version(it.first);
opList.insert(name);
pugi::xml_node entry = currentDeviceNode.append_child(name.c_str());