IntenseWebs/openvino
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add json validate (#6449)

Browse Source 
* Add json validate

* Fix json schema

* Fix schema loader

* Add unit test

* Update bom file

* Update all requarments

* Update dev requarments

* Update requrments

* Update path to schema

* Update schema

* Add some unit tests

* Move schema to root dir

* Update schema path in bom file

* Fix unit test

* Fix bom

* Change path to schema

* update setup

* Fix setup

* Fix mo args test

* Refactoring some code

* Refactoring according to review

* Update sort imports

* Remove id attribute from schema

* Refactoring validator

* Fix according to review

* Move schema from json to dict. Update unit tests.

* Fix BOM file

* Update bom file
This commit is contained in:
iliya mironov 2021-07-22 21:12:44 +03:00 committed by GitHub
parent 8ba64b14c5
commit e1bffcc36c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 204 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
model-optimizer/automation/package_BOM.txt
View File

@ -1070,6 +1070,7 @@ mo/utils/ir_reader/extenders/topk_extender.py
mo/utils/ir_reader/extenders/variadic_split_extender.py
mo/utils/ir_reader/layer_to_class.py
mo/utils/ir_reader/restore_graph.py
mo/utils/json_schema.py
mo/utils/logger.py
mo/utils/model_analysis.py
mo/utils/pipeline_config.py

34
model-optimizer/mo/utils/custom_replacement_config.py
View File

@ -1,6 +1,7 @@
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import fastjsonschema as json_validate
import json
import logging as log
import os
@ -9,7 +10,8 @@ from re import compile, match
from mo.graph.graph import Node, Graph
from mo.utils.error import Error
from mo.utils.graph import nodes_matching_name_pattern, sub_graph_between_nodes
from mo.utils.utils import refer_to_faq_msg
from mo.utils.json_schema import schema_dict
from mo.utils.utils import get_mo_root_dir, refer_to_faq_msg
class CustomReplacementDescriptor(object):
@ -297,12 +299,12 @@ class CustomReplacementDescriptorScope(CustomReplacementDescriptor):
log.debug("Node {} doesn't have output edges. Consider it output".format(node_name))
output_tensors.add((generate_pattern_for_node(graph, pattern, node_name), 0))
if not self.has('inputs'):
if not self.has('inputs') or len(self._replacement_desc['inputs']) == 0:
self._replacement_desc['inputs'] = [[{'node': desc[0], 'port': desc[1]} for desc in inp]
for inp in sorted(input_nodes_mapping.values())]
log.debug('Updated inputs of sub-graph for instance "{}"'.format(self.instances))
if not self.has('outputs'):
if not self.has('outputs') or len(self._replacement_desc['outputs']) == 0:
self._replacement_desc['outputs'] = [{'node': node, 'port': port} for node, port in sorted(output_tensors)]
log.debug('Updated outputs of sub-graph for instance "{}"'.format(self.instances))
@ -342,13 +344,8 @@ def parse_custom_replacement_config_file(file_name: str):
if not os.path.exists(file_name):
raise Error("Custom replacements configuration file '{}' does not exist. ".format(file_name) +
refer_to_faq_msg(69))
try:
with open(file_name, 'r') as f:
data = json.load(f)
except Exception as exc:
raise Error("Failed to parse custom replacements configuration file '{}': {}. ".format(file_name, exc) +
refer_to_faq_msg(70)) from exc
data = load_and_validate_json_config(file_name)
result = list()
validation_errors = list()
for attrs in data:
@ -394,3 +391,22 @@ def generate_pattern_for_node(graph: Graph, sub_graph_pattern: str, node_name: s
raise RuntimeError('The pattern that uniquely identifies node "{}" using sub-graph pattern "{}" has not been found'.
format(node_name, sub_graph_pattern))
def load_and_validate_json_config(config_file_name: str):
"""
Reads and validate custom replacement configuration file config_file_name.
:param config_file_name: name of the file to read from.
:return: A dictionary serialized from json config file.
"""
try:
with open(config_file_name, 'r') as f:
json_config = json.load(f)
validator = json_validate.compile(schema_dict)
validator(json_config)
except Exception as e:
raise Error("Failed to parse custom replacements configuration file '{}': {}. ".format(config_file_name, e) +
refer_to_faq_msg(70)) from e
return json_config

129
model-optimizer/mo/utils/json_schema.py Normal file
View File

@ -0,0 +1,129 @@
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
schema_dict = {
"definitions": {},
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Root",
"type": "array",
"default": [],
"items": {
"$id": "#root/items",
"title": "Items",
"type": "object",
"required": [
"id",
"match_kind"
],
"properties": {
"custom_attributes": {
"$id": "#root/items/custom_attributes",
"title": "Custom_attributes",
"type": "object",
"properties": {
}
},
"id": {
"$id": "#root/items/id",
"title": "Id",
"type": "string",
"pattern": "^.*$",
"minLength": 1
},
"inputs": {
"$id": "#root/items/inputs",
"title": "Inputs",
"type": "array",
"default": [],
"items": {
"$id": "#root/items/inputs/items",
"title": "Items",
"type": "array",
"default": [],
"items": {
"$id": "#root/items/inputs/items/items",
"title": "Items",
"type": "object",
"properties": {
"node": {
"$id": "#root/items/inputs/items/items/node",
"title": "Node",
"type": "string",
"default": "",
"pattern": "^.*$"
},
"port": {
"$id": "#root/items/inputs/items/items/port",
"title": "Port",
"type": "integer",
"default": 0
}
},
"required": ["node", "port"]
}
}
},
"instances": {
"$id": "#root/items/instances",
"title": "Instances",
"type": ["array", "object"],
"items": {
"$id": "#root/items/instances/items",
"title": "Items",
"type": "string",
"default": "",
"pattern": "^.*$"
}
},
"match_kind": {
"$id": "#root/items/match_kind",
"title": "Match_kind",
"type": "string",
"enum": ["points", "scope", "general"],
"default": "points",
"pattern": "^.*$"
},
"outputs": {
"$id": "#root/items/outputs",
"title": "Outputs",
"type": "array",
"default": [],
"items": {
"$id": "#root/items/outputs/items",
"title": "Items",
"type": "object",
"properties": {
"node": {
"$id": "#root/items/outputs/items/node",
"title": "Node",
"type": "string",
"default": "",
"pattern": "^.*$"
},
"port": {
"$id": "#root/items/outputs/items/port",
"title": "Port",
"type": "integer",
"default": 0
}
},
"required": ["node", "port"]
}
},
"include_inputs_to_sub_graph": {
"$id": "#root/items/include_inputs_to_sub_graph",
"title": "Include_inputs_to_sub_graph",
"type": "boolean",
"default": False
},
"include_outputs_to_sub_graph": {
"$id": "#root/items/include_outputs_to_sub_graph",
"title": "Include_outputs_to_sub_graph",
"type": "boolean",
"default": False
}
}
}
}

1
model-optimizer/requirements.txt
View File

@ -8,3 +8,4 @@ onnx>=1.8.1
defusedxml>=0.7.1
urllib3>=1.26.4
requests>=2.25.1
fastjsonschema~=2.15.1

1
model-optimizer/requirements_caffe.txt
View File

@ -3,3 +3,4 @@ numpy>=1.16.6,<1.20
protobuf>=3.15.6
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema~=2.15.1

1
model-optimizer/requirements_dev.txt
View File

@ -6,3 +6,4 @@ test-generator==0.1.1
defusedxml>=0.5.0
requests>=2.20.0
pytest>=6.2.4
fastjsonschema~=2.15.1

1
model-optimizer/requirements_kaldi.txt
View File

@ -2,3 +2,4 @@ networkx~=2.5
numpy>=1.16.6,<1.20
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema~=2.15.1

1
model-optimizer/requirements_mxnet.txt
View File

@ -5,3 +5,4 @@ numpy>=1.16.6,<1.20
defusedxml>=0.7.1
urllib3>=1.26.4
requests>=2.25.1
fastjsonschema~=2.15.1

1
model-optimizer/requirements_onnx.txt
View File

@ -3,3 +3,4 @@ networkx~=2.5
numpy>=1.16.6,<1.20
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema~=2.15.1

1
model-optimizer/requirements_tf.txt
View File

@ -4,3 +4,4 @@ networkx~=2.5
numpy>=1.16.6,<1.19
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema~=2.15.1

1
model-optimizer/requirements_tf2.txt
View File

@ -3,3 +3,4 @@ networkx~=2.5
numpy>=1.16.6,<1.20
defusedxml>=0.7.1
requests>=2.25.1
fastjsonschema~=2.15.1

40
model-optimizer/unit_tests/mo/utils/custom_replacement_config_test.py Normal file
View File

@ -0,0 +1,40 @@
# Copyright (C) 2018-2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import os
import unittest
from fnmatch import fnmatch
from generator import generator, generate
from mo.utils.custom_replacement_config import load_and_validate_json_config
from mo.utils.error import Error
from mo.utils.utils import get_mo_root_dir
def get_json_configs(mo_root_dir):
config_path = os.path.join(mo_root_dir, 'extensions', 'front')
pattern = "*.json"
config_files_list = []
for path, subdirs, files in os.walk(config_path):
for name in files:
if fnmatch(name, pattern):
config_files_list.append((os.path.join(path, name),))
return config_files_list
@generator
class TestSchema(unittest.TestCase):
base_dir = get_mo_root_dir()
schema_file = os.path.join(base_dir, 'mo', 'utils', 'schema.json')
transformation_configs = get_json_configs(base_dir)
test_json1 = '[{"id": "", "match_kind": "general", "custom_attributes": {}}]'
test_json2 = '[{"id": "someid", "match_kind": "abc", "custom_attributes": {}}]'
@generate(*transformation_configs)
def test_schema_file(self, transformation_config):
self.assertTrue(load_and_validate_json_config(transformation_config))
def test_schema_id_empty(self):
self.assertRaises(Error, load_and_validate_json_config, self.test_json1)
def test_schema_match_kind_wrong(self):
self.assertRaises(Error, load_and_validate_json_config, self.test_json2)

1
tests/stress_tests/scripts/requirements.txt
View File

@ -1,5 +1,6 @@
pymongo
Jinja2
PyYAML
fastjsonschema~=2.15.1
h5py<3.0.0 # WA for OMZ Keras models. Details: https://github.com/openvinotoolkit/open_model_zoo/issues/1806