[apiConformance] Update reports to avoid sw plugins over all devices (#20222)

This commit is contained in:
Sofya Balandina authored and GitHub committed 2023-11-16 17:51:34 +04:00
1 parent 908fa51742
commit b6f1ffceca
6 files changed
+154 -193

No files matched your search

@@ -96,7 +96,6 @@ class HighlightTableCreator():
self.devices = expected_devices if expected_devices else []
self.expected_devices = expected_devices
self.sw_plugins = set()
self.expected_test_modes = expected_test_mode
@@ -223,12 +222,11 @@ class HighlightTableCreator():
except ET.ParseError:
logger.error(f'Error parsing {xml_path}')
else:
logger.info("Opset info collecting is finished successfully")
self.update_real_devices(ops_devices)
def build_sw_plugin_name(self, sw_plugin, device):
return 'HW PLUGIN' if str(sw_plugin).upper() == str(device).upper() else sw_plugin
def collect_api_info(self):
logger.info("API info collecting is started")
@@ -243,13 +241,13 @@ class HighlightTableCreator():
api_devices.add(device.tag)
for test_type in xml_root.findall(f"results/{device.tag}/*"):
self.api_info.setdefault(test_type.tag, {})
for sw_plugin in xml_root.findall(f"results/{device.tag}/{test_type.tag}/*"):
sw_plugin_name = self.build_sw_plugin_name(sw_plugin.tag, device.tag)
self.sw_plugins.add(sw_plugin_name)
self.api_info[test_type.tag].setdefault(sw_plugin_name, {device.tag: {}})
self.api_info[test_type.tag][sw_plugin_name][device.tag] = {'passrate': float(sw_plugin.get('passrate', 0)), 'diff': 0,
'rel_passrate': float(sw_plugin.get('relative_passrate', 0)), "rel_diff": 0,
'title': 'Passrate on optional API scope', 'rel_title': 'Passrate on mandatory API scope'}
for plugin_info in xml_root.findall(f"results/{device.tag}/{test_type.tag}/*"):
if plugin_info.tag != device.tag:
continue
self.api_info[test_type.tag].setdefault(device.tag, {})
self.api_info[test_type.tag][device.tag] = {'passrate': float(plugin_info.get('passrate', 0)), 'diff': 0,
'rel_passrate': float(plugin_info.get('relative_passrate', 0)), "rel_diff": 0,
'title': 'Passrate on optional API scope', 'rel_title': 'Passrate on mandatory API scope'}
except ET.ParseError:
logger.error(f'Error parsing {xml_path}')
@@ -263,34 +261,32 @@ class HighlightTableCreator():
for test_type in xml_root.findall(f"results/{device.tag}/*"):
self.api_info.setdefault(test_type.tag, {})
for sw_plugin in xml_root.findall(f"results/{device.tag}/{test_type.tag}/*"):
sw_plugin_name = self.build_sw_plugin_name(sw_plugin.tag, device.tag)
self.api_info[test_type.tag].setdefault(sw_plugin_name, {device.tag: {'passrate': 0, 'diff': 0, "rel_passrate": 0, "rel_diff": 0}})
self.api_info[test_type.tag][sw_plugin_name][device.tag]['diff'] = round(self.api_info[test_type.tag][sw_plugin_name][device.tag].get('passrate', 0) -\
float(sw_plugin.get('passrate')), 0)
self.api_info[test_type.tag][sw_plugin_name][device.tag]['rel_diff'] = round(self.api_info[test_type.tag][sw_plugin_name][device.tag].get('rel_passrate', 0) -\
float(sw_plugin.get('relative_passrate')), 0)
for plugin_info in xml_root.findall(f"results/{device.tag}/{test_type.tag}/*"):
if plugin_info.tag != device.tag:
continue
self.api_info[test_type.tag].setdefault(device.tag, {'passrate': 0, 'diff': 0, "rel_passrate": 0, "rel_diff": 0})
self.api_info[test_type.tag][device.tag]['diff'] = round(self.api_info[test_type.tag][device.tag].get('passrate', 0) -\
float(plugin_info.get('passrate')), 0)
self.api_info[test_type.tag][device.tag]['rel_diff'] = round(self.api_info[test_type.tag][device.tag].get('rel_passrate', 0) -\
float(plugin_info.get('relative_passrate')), 0)
if self.api_info[test_type.tag][sw_plugin_name][device.tag]['diff'] != 0:
self.api_info[test_type.tag][sw_plugin_name][device.tag]['title'] += f",{' increased' if self.api_info[test_type.tag][sw_plugin_name][device.tag]['diff'] > 0 else ' decreased'} of " +\
f"{self.api_info[test_type.tag][sw_plugin_name][device.tag]['diff']}% from previous run"
if self.api_info[test_type.tag][sw_plugin_name][device.tag]['rel_diff'] != 0:
self.api_info[test_type.tag][sw_plugin_name][device.tag]['rel_title'] += f",{' increased' if self.api_info[test_type.tag][sw_plugin_name][device.tag]['rel_diff'] > 0 else ' decreased'} of " +\
f"{self.api_info[test_type.tag][sw_plugin_name][device.tag]['rel_diff']}% from previous run"
if self.api_info[test_type.tag][device.tag]['diff'] != 0:
self.api_info[test_type.tag][device.tag]['title'] += f",{' increased' if self.api_info[test_type.tag][device.tag]['diff'] > 0 else ' decreased'} of " +\
f"{self.api_info[test_type.tag][device.tag]['diff']}% from previous run"
if self.api_info[test_type.tag][device.tag]['rel_diff'] != 0:
self.api_info[test_type.tag][device.tag]['rel_title'] += f",{' increased' if self.api_info[test_type.tag][device.tag]['rel_diff'] > 0 else ' decreased'} of " +\
f"{self.api_info[test_type.tag][device.tag]['rel_diff']}% from previous run"
except ET.ParseError:
logger.error(f'Error parsing {xml_path}')
else:
logger.info("API info collecting is finished successfully")
self.update_real_devices(api_devices)
def create_html(self, output_folder=None, output_filename=None, report_tag="",
report_version="", current_commit="", prev_commit=""):
sw_plugins = list(self.sw_plugins)
sw_plugins.sort()
if 'HW PLUGIN' in sw_plugins:
sw_plugins.remove('HW PLUGIN')
sw_plugins.insert(0, 'HW PLUGIN')
logger.info("Creating html report is started")
script_dir = Path(__file__).parent.absolute()
file_loader = FileSystemLoader(script_dir.joinpath('template').as_posix())
env = Environment(loader=file_loader)
@@ -301,7 +297,6 @@ class HighlightTableCreator():
general_pass_rate=self.general_pass_rate,
expected_test_mode=self.ops_info.keys(),
api_info=self.api_info,
sw_plugins=sw_plugins,
report_tag=report_tag,
report_version=report_version,
current_commit=current_commit,
@@ -316,6 +311,8 @@ class HighlightTableCreator():
with open(report_path.as_posix(), "w") as f:
f.write(res_summary)
logger.info("Creating html report is finished successfully")
if __name__ == "__main__":
@@ -338,7 +338,7 @@ def create_api_summary(xml_paths: list, output_folder: str, expected_devices:lis
timestamp = None
api_info = {}
sw_plugins = set()
sw_plugins = ['MULTI', 'BATCH', 'AUTO', 'HETERO']
api_devices = set(expected_devices) if expected_devices else set()
logger.info("Statistic collecting is started")
@@ -358,35 +358,29 @@ def create_api_summary(xml_paths: list, output_folder: str, expected_devices:lis
api_devices.add(device.tag)
for test_type in xml_root.findall(f"results/{device.tag}/*"):
api_info.setdefault(test_type.tag, {})
for sw_plugin in xml_root.findall(f"results/{device.tag}/{test_type.tag}/*"):
sw_plugin_name = 'HW_PLUGIN' if str(sw_plugin.tag).upper() == str(device.tag).upper() else sw_plugin.tag
sw_plugins.add(sw_plugin_name)
api_info[test_type.tag].setdefault(sw_plugin_name, {device.tag: {}})
api_info[test_type.tag][sw_plugin_name][device.tag] = {'passrate': float(sw_plugin.get('passrate', 0)),
'relative_passrate': float(sw_plugin.get('relative_passrate', 0)),
'relative_all': int(float(sw_plugin.get('relative_all', 0))),
'relative_passed': int(float(sw_plugin.get('relative_passed', 0))),
'passed': int(sw_plugin.get('passed', 0)),
'failed': int(sw_plugin.get('failed', 0)),
'crashed': int(sw_plugin.get('crashed', 0)),
'skipped': int(sw_plugin.get('skipped', 0)),
'hanged': int(sw_plugin.get('hanged', 0)),
'test_amout': int(sw_plugin.get('passed', 0)) +\
int(sw_plugin.get('failed', 0)) +\
int(sw_plugin.get('crashed', 0)) +\
int(sw_plugin.get('skipped', 0)) +\
int(sw_plugin.get('hanged', 0))}
for plugin_info in xml_root.findall(f"results/{device.tag}/{test_type.tag}/*"):
if str(plugin_info.tag).upper() != str(device.tag).upper() and\
(plugin_info.tag in sw_plugins and str(plugin_info.tag).upper() != 'TEMPLATE'):
continue
api_info[test_type.tag].setdefault(device.tag, {'test_amout': 0})
for key in ['passed', 'failed', 'crashed', 'skipped', 'hanged', 'relative_all', 'relative_passed']:
val = int(float(plugin_info.get(key, 0)))
api_info[test_type.tag][device.tag]['test_amout'] += val if 'relative' not in key else 0
val += api_info[test_type.tag][device.tag].setdefault(key, 0)
api_info[test_type.tag][device.tag][key] = val
if api_info[test_type.tag][device.tag]['relative_all'] > 0:
api_info[test_type.tag][device.tag]['relative_passrate'] = round(api_info[test_type.tag][device.tag]['relative_passed'] * 100 /\
api_info[test_type.tag][device.tag]['relative_all'], 2)
if api_info[test_type.tag][device.tag]['test_amout'] > 0:
api_info[test_type.tag][device.tag]['passrate'] = round(api_info[test_type.tag][device.tag]['passed'] * 100 /\
api_info[test_type.tag][device.tag]['test_amout'], 2)
except ET.ParseError:
logger.error(f'Error parsing {xml_path}')
logger.info("Statistic collecting is completed")
sw_plugins = list(sw_plugins)
sw_plugins.sort()
if 'HW_PLUGIN' in sw_plugins:
sw_plugins.remove('HW_PLUGIN')
sw_plugins.insert(0, 'HW_PLUGIN')
logger.info("File with report creating is started")
script_dir = Path(__file__).parent.absolute()
file_loader = FileSystemLoader(script_dir.joinpath('template').as_posix())
@@ -395,7 +389,6 @@ def create_api_summary(xml_paths: list, output_folder: str, expected_devices:lis
res_summary = template.render(devices=api_devices,
api_info=api_info,
sw_plugins=sw_plugins,
timestamp=timestamp,
report_tag=report_tag,
report_version=report_version)
@@ -5,7 +5,6 @@ FILTERS = {
DATA_ATTRIBUTES = {
FILTER: "data-filter",
SW_PLUGIN: 'data-sw_plugin',
DEVICE: 'data-device',
PASSED: 'data-passed_tests',
FAILED: 'data-failed',
@@ -25,19 +24,11 @@ SCOPE_TYPE = {
}
allDevices = []
swPlugins = []
$(document).ready(function () {
$("#devices").chosen({max_selected_options: 6});
Array.from($("#devices")[0].options).map((element) => allDevices.push(element.value))
$("#report #statistic td").filter(function () {
sw_plugin = $(this).attr(DATA_ATTRIBUTES.SW_PLUGIN)
if (sw_plugin && !swPlugins.includes(sw_plugin)) {
swPlugins.push(sw_plugin)
}
});
update_statistic();
});
@@ -62,40 +53,38 @@ function get_background_color(float_passrate) {
function update_statistic() {
for (device of allDevices) {
for (sw_plugin of swPlugins) {
let passed = 0
let amount = 0
let scope = $('#scope_toggle').text()
let passed = 0
let amount = 0
let scope = $('#scope_toggle').text()
$("#report #data td:not(:hidden)").filter(function () {
if ($(this).attr(DATA_ATTRIBUTES.DEVICE) === device && $(this).attr(DATA_ATTRIBUTES.SW_PLUGIN) === sw_plugin) {
if ($(this).attr(DATA_ATTRIBUTES.PASSED) && $(this).attr(DATA_ATTRIBUTES.TESTS_AMOUNT) &&
scope == SCOPE_TYPE.OPTIONAL) {
passed += parseInt($(this).attr(DATA_ATTRIBUTES.PASSED))
amount += parseInt($(this).attr(DATA_ATTRIBUTES.TESTS_AMOUNT))
}
if ($(this).attr(DATA_ATTRIBUTES.REL_PASSED) && $(this).attr(DATA_ATTRIBUTES.REL_TESTS_AMOUNT) &&
scope == SCOPE_TYPE.MANDATORY) {
passed += parseInt($(this).attr(DATA_ATTRIBUTES.REL_PASSED))
amount += parseInt($(this).attr(DATA_ATTRIBUTES.REL_TESTS_AMOUNT))
}
$("#report #data td:not(:hidden)").filter(function () {
if ($(this).attr(DATA_ATTRIBUTES.DEVICE) === device) {
if ($(this).attr(DATA_ATTRIBUTES.PASSED) && $(this).attr(DATA_ATTRIBUTES.TESTS_AMOUNT) &&
scope == SCOPE_TYPE.OPTIONAL) {
passed += parseInt($(this).attr(DATA_ATTRIBUTES.PASSED))
amount += parseInt($(this).attr(DATA_ATTRIBUTES.TESTS_AMOUNT))
}
});
if ($(this).attr(DATA_ATTRIBUTES.REL_PASSED) && $(this).attr(DATA_ATTRIBUTES.REL_TESTS_AMOUNT) &&
scope == SCOPE_TYPE.MANDATORY) {
passed += parseInt($(this).attr(DATA_ATTRIBUTES.REL_PASSED))
amount += parseInt($(this).attr(DATA_ATTRIBUTES.REL_TESTS_AMOUNT))
}
}
});
let passrate = ''
let background_color = "rgba(255, 255, 255, 0.2)"
if (!amount) {
passrate = "---";
} else {
let float_passrate = passed * 100 / amount
passrate = (float_passrate).toFixed(2) + ' %';
background_color = get_background_color(float_passrate)
}
let id_general = '#' + device + '_' + sw_plugin + '_statistic'
if ($(id_general).length) {
$(id_general).text(passrate)
$(id_general).css("background-color", background_color)
}
let passrate = ''
let background_color = "rgba(255, 255, 255, 0.2)"
if (!amount) {
passrate = "---";
} else {
let float_passrate = passed * 100 / amount
passrate = (float_passrate).toFixed(2) + ' %';
background_color = get_background_color(float_passrate)
}
let id_general = '#' + device + '_statistic'
if ($(id_general).length) {
$(id_general).text(passrate)
$(id_general).css("background-color", background_color)
}
}
}
@@ -105,7 +105,6 @@
<thead>
<tr>
<th class="table-dark" style="text-align: left; vertical-align: middle !important;">Entity</th>
<th class="diagonal"><span class="th-devices">Devices</span><span class="th-sw-plugin">Plugins</span></th>
{% for device in devices %}
<th class="table-dark" colspan="2" style="text-align: center; vertical-align: middle !important;">{{ device }}</th>
{% endfor %}
@@ -113,38 +112,35 @@
</thead>
<tbody style="text-align: center;">
{% for test_type in api_info %}
<td class="table-primary" style="text-align: left;" rowspan="{{ api_info[test_type].keys()|length + 1 }}" style="vertical-align: middle !important;">{{ test_type }}</td>
{% for sw_plugin in sw_plugins %}
<tr>
<td style="text-align: left;">{{sw_plugin}}</td>
{% for device in devices %}
{% if device in api_info[test_type][sw_plugin] %}
<td title="{{api_info[test_type][sw_plugin][device]['title']}}">
<span>Total:</span>
<span>{{ api_info[test_type][sw_plugin][device]['passrate'] }}</span>
{% if api_info[test_type][sw_plugin][device]['diff'] > 0 %}
(<span class="text-success font-weight-bold">+{{ api_info[test_type][sw_plugin][device]['diff'] }}</span>)
{% elif api_info[test_type][sw_plugin][device]['diff'] < 0 %}
(<span class="text-danger font-weight-bold">{{ api_info[test_type][sw_plugin][device]['diff'] }}</span>)
{% endif %}
%
</td>
<td title="{{api_info[test_type][sw_plugin][device]['rel_title']}}">
<span>Rel:</span>
<span>{{ api_info[test_type][sw_plugin][device]['rel_passrate'] }}</span>
{% if api_info[test_type][sw_plugin][device]['rel_diff'] > 0 %}
(<span class="text-success font-weight-bold">+{{ api_info[test_type][sw_plugin][device]['rel_diff'] }}</span>)
{% elif api_info[test_type][sw_plugin][device]['rel_diff'] < 0 %}
(<span class="text-danger font-weight-bold">{{ api_info[test_type][sw_plugin][device]['rel_diff'] }}</span>)
{% endif %}
%
</td>
{% else %}
<td colspan="2" style="text-align: center;" title="API conformanace wasn't run on {{device}} {{sw_plugin}}">NOT RUN</td>
{% endif %}
{% endfor %}
</tr>
{% endfor %}
<tr>
<td class="table-primary" style="text-align: left;" style="vertical-align: middle !important;">{{ test_type }}</td>
{% for device in devices %}
{% if device in api_info[test_type] %}
<td title="{{api_info[test_type][device]['title']}}">
<span>Total:</span>
<span>{{ api_info[test_type][device]['passrate'] }}</span>
{% if api_info[test_type][device]['diff'] > 0 %}
(<span class="text-success font-weight-bold">+{{ api_info[test_type][device]['diff'] }}</span>)
{% elif api_info[test_type][device]['diff'] < 0 %}
(<span class="text-danger font-weight-bold">{{ api_info[test_type][device]['diff'] }}</span>)
{% endif %}
%
</td>
<td title="{{api_info[test_type][device]['rel_title']}}">
<span>Rel:</span>
<span>{{ api_info[test_type][device]['rel_passrate'] }}</span>
{% if api_info[test_type][device]['rel_diff'] > 0 %}
(<span class="text-success font-weight-bold">+{{ api_info[test_type][device]['rel_diff'] }}</span>)
{% elif api_info[test_type][device]['rel_diff'] < 0 %}
(<span class="text-danger font-weight-bold">{{ api_info[test_type][device]['rel_diff'] }}</span>)
{% endif %}
%
</td>
{% else %}
<td colspan="2" style="text-align: center;" title="API conformanace wasn't run on {{device}} {{sw_plugin}}">NOT RUN</td>
{% endif %}
{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>
@@ -84,71 +84,64 @@
<thead>
<tr>
<th class="table-dark entity">Entity</th>
<th class="diagonal"><span class="th-devices">Devices</span><span class="th-sw-plugin">Plugins</span></th>
{% for device in devices %}
<td class="table-dark device" style="text-align: center;" data-device="{{device}}">{{ device }}</td>
{% endfor %}
</tr>
</thead>
<tbody id="statistic">
<td class="table-primary" style="font-weight: bold; vertical-align: middle !important;" rowspan="{{ sw_plugins|length + 1 }}">Passrate</td>
{% for sw_plugin in sw_plugins %}
<tr>
<td>{{sw_plugin}}</td>
{% for device in devices %}
<td
style="text-align: center;"
data-device="{{device}}"
data-sw_plugin="{{sw_plugin}}"
id="{{device}}_{{sw_plugin}}_statistic"
>
&#8634;
</td>
{% endfor %}
</tr>
{% endfor %}
<tr>
<td class="table-primary" style="font-weight: bold; vertical-align: middle !important;">Passrate</td>
{% for device in devices %}
<td
style="text-align: center;"
data-device="{{device}}"
id="{{device}}_statistic"
>
&#8634;
</td>
{% endfor %}
</tr>
</tbody>
<tbody id="data">
{% for test_type in api_info %}
<td data-test-type="{{test_type}}" class="table-primary" rowspan="{{ api_info[test_type].keys()|length + 1 }}" style="vertical-align: middle !important;">{{ test_type }}</td>
{% for sw_plugin in sw_plugins %}
<tr>
<td data-test-type="{{test_type}}">{{sw_plugin}}</td>
{% for device in devices %}
{% if device in api_info[test_type][sw_plugin] %}
<td data-test-type="{{test_type}}"
data-device="{{device}}"
data-sw_plugin="{{sw_plugin}}"
style="text-align: center;"
data-passed_tests="{{api_info[test_type][sw_plugin][device]['passed']}}"
data-all_tests="{{api_info[test_type][sw_plugin][device]['test_amout']}}"
data-rel_passed_tests="{{api_info[test_type][sw_plugin][device]['relative_passed']}}"
data-rel_all_tests="{{api_info[test_type][sw_plugin][device]['relative_all']}}"
data-relative_passrate="{{api_info[test_type][sw_plugin][device]['relative_passrate']}}"
data-passrate="{{api_info[test_type][sw_plugin][device]['passrate']}}"
data-failed="{{api_info[test_type][sw_plugin][device]['failed']}}"
data-skipped="{{api_info[test_type][sw_plugin][device]['skipped']}}"
data-crashed="{{api_info[test_type][sw_plugin][device]['crashed']}}"
data-hanged="{{api_info[test_type][sw_plugin][device]['hanged']}}"
>
<span>{{api_info[test_type][sw_plugin][device]['relative_passrate']}} %</span>
<div style="width: 150px; margin: auto;">
<span class="green" title="Passed">P:{{ api_info[test_type][sw_plugin][device]['relative_passed'] }}</span>
<span class="red" title="Failed">F:{{ api_info[test_type][sw_plugin][device]['relative_all'] - api_info[test_type][sw_plugin][device]['relative_passed'] }}</span>
<span class="grey" title="Skipped" style="display: none;">---</span>
<span class="dark" title="Crashed" style="display: none;">---</span>
<span class="grey-red" title="Hanged" style="display: none;">---</span>
</div>
</td>
{% else %}
<td data-test-type="{{test_type}}" data-device="{{device}}" data-sw_plugin="{{sw_plugin}}" style="text-align: center;">
<span>NOT RUN</span></td>
{% endif %}
{% endfor %}
</tr>
{% endfor %}
<tr>
<td data-test-type="{{test_type}}" class="table-primary" style="vertical-align: middle !important;">{{ test_type }}</td>
{% for device in devices %}
{% if device in api_info[test_type] %}
<td data-test-type="{{test_type}}"
data-device="{{device}}"
style="text-align: center;"
data-passed_tests="{{api_info[test_type][device]['passed']}}"
data-all_tests="{{api_info[test_type][device]['test_amout']}}"
data-rel_passed_tests="{{api_info[test_type][device]['relative_passed']}}"
data-rel_all_tests="{{api_info[test_type][device]['relative_all']}}"
data-relative_passrate="{{api_info[test_type][device]['relative_passrate']}}"
data-passrate="{{api_info[test_type][device]['passrate']}}"
data-failed="{{api_info[test_type][device]['failed']}}"
data-skipped="{{api_info[test_type][device]['skipped']}}"
data-crashed="{{api_info[test_type][device]['crashed']}}"
data-hanged="{{api_info[test_type][device]['hanged']}}"
>
{% if api_info[test_type][device]['relative_all'] == 0 %}
<span>---</span>
{% else %}
<span>{{api_info[test_type][device]['relative_passrate']}} %</span>
{% endif %}
<div style="width: 150px; margin: auto;">
<span class="green" title="Passed">P:{{ api_info[test_type][device]['relative_passed'] }}</span>
<span class="red" title="Failed">F:{{ api_info[test_type][device]['relative_all'] - api_info[test_type][device]['relative_passed'] }}</span>
<span class="grey" title="Skipped" style="display: none;">---</span>
<span class="dark" title="Crashed" style="display: none;">---</span>
<span class="grey-red" title="Hanged" style="display: none;">---</span>
</div>
</td>
{% else %}
<td data-test-type="{{test_type}}" data-device="{{device}}" style="text-align: center;">
<span>NOT RUN</span></td>
{% endif %}
{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>
@@ -698,13 +698,6 @@ h2 {
opacity: 0.5;
}
.diagonal {
position: relative;
background: linear-gradient(to right top, #000000 0%,#000000 46.9%,#ffffff 47%,#ffffff 51%,#000000 51.1%,#000000 100%) !important;
background-color: transparent;
background-color: none !important;
}
.th-sw-plugin {
color: white;
}