From 09b24a19328eba98a2442cd807d3549fbd27f11d Mon Sep 17 00:00:00 2001 From: Anastasia Kuporosova Date: Tue, 6 Dec 2022 11:46:26 +0100 Subject: [PATCH] Fix XML vulnerabilities found by bandit (#14415) * Fix vulnerabilities found by bandir * fix vulnerabilities in tests dir * fix for mo * some changes --- .../layer_tests_summary/merge_xmls.py | 21 +++++++---- .../rename_conformance_ir.py | 2 +- .../layer_tests_summary/requirements.txt | 1 + .../layer_tests_summary/run_conformance.py | 2 +- .../layer_tests_summary/summarize.py | 21 +++++++---- src/tests_deprecated/functional/vpu/tester.py | 35 +++++++++++-------- tests/layer_tests/common/layer_test_class.py | 2 +- tests/layer_tests/common/utils/parsers.py | 4 +-- tests/stress_tests/scripts/get_testdata.py | 2 +- tests/stress_tests/scripts/memcheck_upload.py | 2 +- tests/stress_tests/scripts/requirements.txt | 3 +- .../mo/back/ie_ir_ver_2/emitter_test.py | 9 ++++- 12 files changed, 67 insertions(+), 37 deletions(-) diff --git a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py index 57811dd8f80..ea13b0e08d9 100644 --- a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py +++ b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/merge_xmls.py @@ -5,10 +5,17 @@ import argparse import os import glob -import xml.etree.ElementTree as ET +import defusedxml.ElementTree as ET +from defusedxml import defuse_stdlib from utils import utils +# defuse_stdlib provide patched version of xml.etree.ElementTree which allows to use objects from xml.etree.ElementTree +# in a safe manner without including unsafe xml.etree.ElementTree +ET_defused = defuse_stdlib()[ET] +Element = ET_defused.Element +SubElement = ET_defused.SubElement + logger = utils.get_logger('XmlMerger') def parse_arguments(): @@ -27,7 +34,7 @@ def parse_arguments(): return parser.parse_args() -def update_result_node(xml_node: ET.SubElement, aggregated_res: ET.SubElement): +def update_result_node(xml_node: SubElement, aggregated_res: SubElement): for attr_name in xml_node.attrib: if attr_name == "passrate": continue @@ -44,7 +51,7 @@ def update_result_node(xml_node: ET.SubElement, aggregated_res: ET.SubElement): aggregated_res.set(attr_name, str(xml_value + aggregated_value)) -def aggregate_test_results(aggregated_results: ET.SubElement, xml_reports: list, report_type: str): +def aggregate_test_results(aggregated_results: SubElement, xml_reports: list, report_type: str): aggregated_timestamp = None for xml in xml_reports: logger.info(f" Processing: {xml}") @@ -83,8 +90,8 @@ def aggregate_test_results(aggregated_results: ET.SubElement, xml_reports: list, def merge_xml(input_folder_paths: list, output_folder_paths: str, output_filename: str, report_type: str): logger.info(f" Processing is finished") - summary = ET.Element("report") - results = ET.SubElement(summary, "results") + summary = Element("report") + results = SubElement(summary, "results") entity_name = None if report_type == "OP": entity_name = "ops_list" @@ -93,7 +100,7 @@ def merge_xml(input_folder_paths: list, output_folder_paths: str, output_filenam else: raise Exception(f"Error to create aggregated report. Incorrect report type: {report_type}") - entity_list = ET.SubElement(summary, entity_name) + entity_list = SubElement(summary, entity_name) for folder_path in input_folder_paths: if not os.path.exists(folder_path): @@ -121,7 +128,7 @@ def merge_xml(input_folder_paths: list, output_folder_paths: str, output_filenam logger.error(f'{folder_path} does not contain the correct xml files') for entity in xml_root.find(entity_name): if entity_list.find(entity.tag) is None: - ET.SubElement(entity_list, entity.tag) + SubElement(entity_list, entity.tag) timestamp = aggregate_test_results(results, xml_reports, report_type) if report_type == "OP": utils.update_passrates(results) diff --git a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/rename_conformance_ir.py b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/rename_conformance_ir.py index a0feaae33df..ef4c9222fb8 100644 --- a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/rename_conformance_ir.py +++ b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/rename_conformance_ir.py @@ -1,7 +1,7 @@ # Copyright (C) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import xml.etree.ElementTree as ET +import defusedxml.ElementTree as ET from argparse import ArgumentParser from pathlib import Path diff --git a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/requirements.txt b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/requirements.txt index 809685a62ad..6436ef4aa86 100644 --- a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/requirements.txt +++ b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/requirements.txt @@ -1,2 +1,3 @@ jinja2==3.1.2 gitpython +defusedxml>=0.7.1 diff --git a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/run_conformance.py b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/run_conformance.py index 3e0ad49cedd..f5c14453335 100644 --- a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/run_conformance.py +++ b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/run_conformance.py @@ -12,7 +12,7 @@ from merge_xmls import merge_xml from pathlib import Path, PurePath from sys import version, platform -import xml.etree.ElementTree as ET +import defusedxml.ElementTree as ET import os diff --git a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py index dc2e21f477c..0b6c4ac0382 100644 --- a/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py +++ b/src/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py @@ -4,12 +4,19 @@ import argparse import os import csv -import xml.etree.ElementTree as ET +import defusedxml.ElementTree as ET +from defusedxml import defuse_stdlib from jinja2 import Environment, FileSystemLoader from utils import utils +# defuse_stdlib provide patched version of xml.etree.ElementTree which allows to use objects from xml.etree.ElementTree +# in a safe manner without including unsafe xml.etree.ElementTree +ET_defused = defuse_stdlib()[ET] +Element = ET_defused.Element +SubElement = ET_defused.SubElement + NOT_RUN = "NOT RUN" NA = "N/A" @@ -50,10 +57,10 @@ def parse_arguments(): def merge_xmls(xml_paths: list): logger.info("Merging XML files is started") - summary = ET.Element("report") + summary = Element("report") timestamp = None - summary_results = ET.SubElement(summary, "results") - ops_list = ET.SubElement(summary, "ops_list") + summary_results = SubElement(summary, "results") + ops_list = SubElement(summary, "ops_list") for xml_path in xml_paths: try: xml_root = ET.parse(xml_path).getroot() @@ -67,7 +74,7 @@ def merge_xmls(xml_paths: list): for op in xml_root.find("ops_list"): if ops_list.find(op.tag) is None: - ET.SubElement(ops_list, op.tag) + SubElement(ops_list, op.tag) for device in xml_root.find("results"): device_results = summary_results.find(device.tag) @@ -103,7 +110,7 @@ def merge_xmls(xml_paths: list): return summary -def collect_statistic(root: ET.Element, is_conformance_mode: bool): +def collect_statistic(root: Element, is_conformance_mode: bool): logger.info("Statistic collecting is started") trusted_ops = dict() pass_rate_avg = dict() @@ -218,7 +225,7 @@ def serialize_to_csv(report_filename: str, output_dir: os.path, op_list: list, d logger.info(f'Final CSV report is saved to {csv_filename}') -def create_summary(summary_root: ET.Element, output_folder: os.path, expected_devices:list, report_tag: str, report_version: str, +def create_summary(summary_root: Element, output_folder: os.path, expected_devices:list, report_tag: str, report_version: str, is_conformance_mode: bool, is_serialize_to_csv: bool, output_filename='report'): if is_conformance_mode: utils.update_conformance_test_counters(summary_root, logger) diff --git a/src/tests_deprecated/functional/vpu/tester.py b/src/tests_deprecated/functional/vpu/tester.py index 5a59f1b8b95..0f9950b737e 100644 --- a/src/tests_deprecated/functional/vpu/tester.py +++ b/src/tests_deprecated/functional/vpu/tester.py @@ -9,9 +9,16 @@ from argparse import ArgumentParser from typing import Union -import xml.etree.cElementTree as et -import xml.dom.minidom as dom +import defusedxml.ElementTree as ET +import defusedxml.minidom as dom +from defusedxml import defuse_stdlib +# defuse_stdlib provide patched version of xml.etree.ElementTree which allows to use objects from xml.etree.ElementTree +# in a safe manner without including unsafe xml.etree.ElementTree +ET_defused = defuse_stdlib()[ET] +Element = ET_defused.Element +dom_defused = defuse_stdlib()[dom] +Document = dom_defused.Document def get_path(entry: Union[str, Path], is_directory=False, check_exists=True, file_or_directory=False): try: @@ -75,7 +82,7 @@ def build_argument_parser(): def tostring(layer, num_tabs=0): - return '\t' * num_tabs + str(et.tostring(layer, encoding='utf-8').decode()) + return '\t' * num_tabs + str(ET.tostring(layer, encoding='utf-8').decode()) def tags(node): @@ -86,17 +93,17 @@ def tags(node): def make_input(layer, port_id): - output = et.Element('output') + output = Element('output') output_port = next(port for port in tags(layer)['output'] if int(port.attrib['id']) == int(port_id)) output.append(output_port) precision_to_element_type = {'FP16': 'f16', 'I32': 'i32', 'FP32': 'f32'} - data = et.Element('data', attrib={ + data = Element('data', attrib={ 'element_type': precision_to_element_type[output_port.attrib['precision']], 'shape': ','.join([dim.text for dim in output_port])} ) - input_layer = et.Element('layer', attrib={ + input_layer = Element('layer', attrib={ 'id': layer.attrib['id'], 'name': layer.attrib['name'], 'type': 'Parameter', @@ -109,11 +116,11 @@ def make_input(layer, port_id): def make_output(layer, port_id): - input_section = et.Element('input') + input_section = Element('input') input_port = next(port for port in tags(layer)['input'] if int(port.attrib['id']) == int(port_id)) input_section.append(input_port) - output_layer = et.Element('layer', attrib={ + output_layer = Element('layer', attrib={ 'id': layer.attrib['id'], 'name': layer.attrib['name'], 'type': 'Result', @@ -158,20 +165,20 @@ def extract_weights(source, layers, destination): def prettify(element, indent=0): - header = dom.Document().toxml() + header = Document().toxml() string = dom.parseString(tostring(element)).toprettyxml()[len(header) + 1:] return '\n'.join(['\t' * indent + line for line in string.split('\n') if line.strip()]) def dump(input_model, elements, output_model): - root = et.parse(str(input_model)).getroot() - net = et.Element('net', attrib={'name': root.attrib['name'], 'version': root.attrib['version']}) + root = ET.parse(str(input_model)).getroot() + net = Element('net', attrib={'name': root.attrib['name'], 'version': root.attrib['version']}) - layers = et.Element('layers') + layers = Element('layers') for layer in elements['layers']: layers.append(layer) - edges = et.Element('edges') + edges = Element('edges') for edge in elements['edges']: edges.append(edge) @@ -190,7 +197,7 @@ def main(): print('Error: only IR version 10 or newer is supported, IRv{} has been given'.format(model.version)) sys.exit(-1) - layers, edges, _ = et.parse(str(arguments.model)).getroot() + layers, edges, _ = ET.parse(str(arguments.model)).getroot() layers_identifiers = {} for layer in layers: layers_identifiers[int(layer.attrib['id'])] = layer diff --git a/tests/layer_tests/common/layer_test_class.py b/tests/layer_tests/common/layer_test_class.py index 33de8b5a6c3..14d070ada2b 100644 --- a/tests/layer_tests/common/layer_test_class.py +++ b/tests/layer_tests/common/layer_test_class.py @@ -5,7 +5,7 @@ import itertools import os import re import warnings -import xml.etree.ElementTree as ET +import defusedxml.ElementTree as ET from pathlib import Path import numpy as np diff --git a/tests/layer_tests/common/utils/parsers.py b/tests/layer_tests/common/utils/parsers.py index 2a9e5ac9a23..0bda70e43b3 100644 --- a/tests/layer_tests/common/utils/parsers.py +++ b/tests/layer_tests/common/utils/parsers.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import os -import xml.etree.ElementTree +import defusedxml.ElementTree as ET def mapping_parser(file): @@ -13,7 +13,7 @@ def mapping_parser(file): """ mapping_dict = {} if os.path.splitext(file)[1] == '.mapping' and os.path.isfile(file): - xml_tree = xml.etree.ElementTree.parse(file) + xml_tree = ET.parse(file) xml_root = xml_tree.getroot() for child in xml_root: framework_info = child.find('.//framework') diff --git a/tests/stress_tests/scripts/get_testdata.py b/tests/stress_tests/scripts/get_testdata.py index e5e770d5a7d..89d175af6dc 100755 --- a/tests/stress_tests/scripts/get_testdata.py +++ b/tests/stress_tests/scripts/get_testdata.py @@ -18,7 +18,7 @@ import sys from distutils.dir_util import copy_tree from inspect import getsourcefile from pathlib import Path -from xml.etree import ElementTree as ET +import defusedxml.ElementTree as ET log.basicConfig(format="{file}: [ %(levelname)s ] %(message)s".format(file=os.path.basename(__file__)), level=log.INFO, stream=sys.stdout) diff --git a/tests/stress_tests/scripts/memcheck_upload.py b/tests/stress_tests/scripts/memcheck_upload.py index e36f273bfd8..144df020426 100644 --- a/tests/stress_tests/scripts/memcheck_upload.py +++ b/tests/stress_tests/scripts/memcheck_upload.py @@ -17,7 +17,7 @@ import logging import os import re import sys -import xml.etree.ElementTree as ET +import defusedxml.ElementTree as ET from glob import glob from inspect import getsourcefile from types import SimpleNamespace diff --git a/tests/stress_tests/scripts/requirements.txt b/tests/stress_tests/scripts/requirements.txt index 2c692f61da4..f8913753397 100644 --- a/tests/stress_tests/scripts/requirements.txt +++ b/tests/stress_tests/scripts/requirements.txt @@ -5,4 +5,5 @@ fastjsonschema~=2.15.1 pandas>=1.3.5 h5py>=3.1.0 scipy~=1.7; python_version == '3.7' -scipy>=1.8; python_version >= '3.8' \ No newline at end of file +scipy>=1.8; python_version >= '3.8' +defusedxml>=0.7.1 diff --git a/tools/mo/unit_tests/mo/back/ie_ir_ver_2/emitter_test.py b/tools/mo/unit_tests/mo/back/ie_ir_ver_2/emitter_test.py index 178d105f0fe..1ebfd9871e1 100644 --- a/tools/mo/unit_tests/mo/back/ie_ir_ver_2/emitter_test.py +++ b/tools/mo/unit_tests/mo/back/ie_ir_ver_2/emitter_test.py @@ -3,7 +3,8 @@ import unittest from unittest.mock import MagicMock -from xml.etree.ElementTree import Element, tostring +import defusedxml.ElementTree as ET +from defusedxml import defuse_stdlib import numpy as np @@ -13,6 +14,12 @@ from openvino.tools.mo.utils.error import Error from openvino.tools.mo.utils.runtime_info import RTInfo, OldAPIMapOrder, OldAPIMapElementType from unit_tests.utils.graph import build_graph, result, regular_op +# defuse_stdlib provide patched version of xml.etree.ElementTree which allows to use objects from xml.etree.ElementTree +# in a safe manner without including unsafe xml.etree.ElementTree +ET_defused = defuse_stdlib()[ET] +Element = ET_defused.Element +tostring = ET_defused.tostring + expected_result = b'2105050'