[IE Samples] New command line parameters format for speech sample (#11051)
* New command line parameters format for speech sample * fixed notes * changed format for scale factor * changed format for scale factor in tests * added more variants, when name is directy specified for i/o/r like it is done for sf * removed nthreads flag * fixed notes * changed output params * updated tests with new format Co-authored-by: Alexander Zhogov <alexander.zhogov@intel.com>
This commit is contained in:
Maxim Gordeev
GitHub
@@ -49,7 +49,10 @@ int main(int argc, char* argv[]) {
|
||||
BaseFile* fileOutput;
|
||||
ArkFile arkFile;
|
||||
NumpyFile numpyFile;
|
||||
auto extInputFile = fileExt(FLAGS_i);
|
||||
std::pair<std::string, std::vector<std::string>> input_data;
|
||||
if (!FLAGS_i.empty())
|
||||
input_data = parse_parameters(FLAGS_i);
|
||||
auto extInputFile = fileExt(input_data.first);
|
||||
if (extInputFile == "ark") {
|
||||
file = &arkFile;
|
||||
} else if (extInputFile == "npz") {
|
||||
@@ -60,9 +63,9 @@ int main(int argc, char* argv[]) {
|
||||
std::vector<std::string> inputFiles;
|
||||
std::vector<uint32_t> numBytesThisUtterance;
|
||||
uint32_t numUtterances(0);
|
||||
if (!FLAGS_i.empty()) {
|
||||
if (!input_data.first.empty()) {
|
||||
std::string outStr;
|
||||
std::istringstream stream(FLAGS_i);
|
||||
std::istringstream stream(input_data.first);
|
||||
uint32_t currentNumUtterances(0), currentNumBytesThisUtterance(0);
|
||||
while (getline(stream, outStr, ',')) {
|
||||
std::string filename(fileNameNoExt(outStr) + "." + extInputFile);
|
||||
@@ -89,19 +92,26 @@ int main(int argc, char* argv[]) {
|
||||
std::vector<std::string> output_names;
|
||||
std::vector<size_t> ports;
|
||||
// --------------------------- Processing custom outputs ---------------------------------------------
|
||||
if (!FLAGS_oname.empty()) {
|
||||
output_names = convert_str_to_vector(FLAGS_oname);
|
||||
for (const auto& output_name : output_names) {
|
||||
auto pos_layer = output_name.rfind(":");
|
||||
if (pos_layer == std::string::npos) {
|
||||
throw std::logic_error("Output " + output_name + " doesn't have a port");
|
||||
}
|
||||
outputs.push_back(output_name.substr(0, pos_layer));
|
||||
try {
|
||||
ports.push_back(std::stoi(output_name.substr(pos_layer + 1)));
|
||||
} catch (const std::exception&) {
|
||||
throw std::logic_error("Ports should have integer type");
|
||||
}
|
||||
std::pair<std::string, std::vector<std::string>> output_data;
|
||||
std::pair<std::string, std::vector<std::string>> reference_data;
|
||||
if (!FLAGS_o.empty())
|
||||
output_data = parse_parameters(FLAGS_o);
|
||||
if (!FLAGS_r.empty())
|
||||
reference_data = parse_parameters(FLAGS_r);
|
||||
if (!output_data.second.empty())
|
||||
output_names = output_data.second;
|
||||
else if (!reference_data.second.empty())
|
||||
output_names = reference_data.second;
|
||||
for (const auto& output_name : output_names) {
|
||||
auto pos_layer = output_name.rfind(":");
|
||||
if (pos_layer == std::string::npos) {
|
||||
throw std::logic_error("Output " + output_name + " doesn't have a port");
|
||||
}
|
||||
outputs.push_back(output_name.substr(0, pos_layer));
|
||||
try {
|
||||
ports.push_back(std::stoi(output_name.substr(pos_layer + 1)));
|
||||
} catch (const std::exception&) {
|
||||
throw std::logic_error("Ports should have integer type");
|
||||
}
|
||||
}
|
||||
// ------------------------------ Preprocessing ------------------------------------------------------
|
||||
@@ -304,8 +314,8 @@ int main(int argc, char* argv[]) {
|
||||
std::vector<ov::Tensor> ptrInputBlobs;
|
||||
auto cInputInfo = executableNet.inputs();
|
||||
check_number_of_inputs(cInputInfo.size(), numInputFiles);
|
||||
if (!FLAGS_iname.empty()) {
|
||||
std::vector<std::string> inputNameBlobs = convert_str_to_vector(FLAGS_iname);
|
||||
if (!input_data.second.empty()) {
|
||||
std::vector<std::string> inputNameBlobs = input_data.second;
|
||||
if (inputNameBlobs.size() != cInputInfo.size()) {
|
||||
std::string errMessage(std::string("Number of network inputs ( ") + std::to_string(cInputInfo.size()) +
|
||||
" ) is not equal to the number of inputs entered in the -iname argument ( " +
|
||||
@@ -328,15 +338,15 @@ int main(int argc, char* argv[]) {
|
||||
std::vector<std::string> output_name_files;
|
||||
std::vector<std::string> reference_name_files;
|
||||
size_t count_file = 1;
|
||||
if (!FLAGS_o.empty()) {
|
||||
output_name_files = convert_str_to_vector(FLAGS_o);
|
||||
if (!output_data.first.empty()) {
|
||||
output_name_files = convert_str_to_vector(output_data.first);
|
||||
if (output_name_files.size() != outputs.size() && !outputs.empty()) {
|
||||
throw std::logic_error("The number of output files is not equal to the number of network outputs.");
|
||||
}
|
||||
count_file = output_name_files.empty() ? 1 : output_name_files.size();
|
||||
}
|
||||
if (!FLAGS_r.empty()) {
|
||||
reference_name_files = convert_str_to_vector(FLAGS_r);
|
||||
if (!reference_data.first.empty()) {
|
||||
reference_name_files = convert_str_to_vector(reference_data.first);
|
||||
if (reference_name_files.size() != outputs.size() && !outputs.empty()) {
|
||||
throw std::logic_error("The number of reference files is not equal to the number of network outputs.");
|
||||
}
|
||||
@@ -429,9 +439,9 @@ int main(int argc, char* argv[]) {
|
||||
BaseFile* fileReferenceScores;
|
||||
std::string refUtteranceName;
|
||||
|
||||
if (!FLAGS_r.empty()) {
|
||||
if (!reference_data.first.empty()) {
|
||||
/** Read file with reference scores **/
|
||||
auto exReferenceScoresFile = fileExt(FLAGS_r);
|
||||
auto exReferenceScoresFile = fileExt(reference_data.first);
|
||||
if (exReferenceScoresFile == "ark") {
|
||||
fileReferenceScores = &arkFile;
|
||||
} else if (exReferenceScoresFile == "npz") {
|
||||
@@ -540,12 +550,12 @@ int main(int argc, char* argv[]) {
|
||||
continue;
|
||||
}
|
||||
ptrInputBlobs.clear();
|
||||
if (FLAGS_iname.empty()) {
|
||||
if (input_data.second.empty()) {
|
||||
for (auto& input : cInputInfo) {
|
||||
ptrInputBlobs.push_back(inferRequest.inferRequest.get_tensor(input));
|
||||
}
|
||||
} else {
|
||||
std::vector<std::string> inputNameBlobs = convert_str_to_vector(FLAGS_iname);
|
||||
std::vector<std::string> inputNameBlobs = input_data.second;
|
||||
for (const auto& input : inputNameBlobs) {
|
||||
ov::Tensor blob = inferRequests.begin()->inferRequest.get_tensor(input);
|
||||
if (!blob) {
|
||||
@@ -638,7 +648,7 @@ int main(int argc, char* argv[]) {
|
||||
|
||||
for (size_t next_output = 0; next_output < count_file; next_output++) {
|
||||
if (!FLAGS_o.empty()) {
|
||||
auto exOutputScoresFile = fileExt(FLAGS_o);
|
||||
auto exOutputScoresFile = fileExt(output_data.first);
|
||||
if (exOutputScoresFile == "ark") {
|
||||
fileOutput = &arkFile;
|
||||
} else if (exOutputScoresFile == "npz") {
|
||||
|
||||
@@ -96,10 +96,6 @@ static const char scale_factor_message[] =
|
||||
/// @brief message for batch size argument
|
||||
static const char batch_size_message[] = "Optional. Batch size 1-8 (default 1)";
|
||||
|
||||
/// @brief message for #threads for CPU inference
|
||||
static const char infer_num_threads_message[] = "Optional. Number of threads to use for concurrent async"
|
||||
" inference requests on the GNA.";
|
||||
|
||||
/// @brief message for left context window argument
|
||||
static const char context_window_message_l[] =
|
||||
"Optional. Number of frames for left context windows (default is 0). "
|
||||
@@ -184,9 +180,6 @@ DEFINE_string(sf, "", scale_factor_message);
|
||||
/// @brief Batch size (default 0)
|
||||
DEFINE_int32(bs, 0, batch_size_message);
|
||||
|
||||
/// @brief Number of threads to use for inference on the CPU (also affects Hetero cases)
|
||||
DEFINE_int32(nthreads, 1, infer_num_threads_message);
|
||||
|
||||
/// @brief Right context window size (default 0)
|
||||
DEFINE_int32(cw_r, 0, context_window_message_r);
|
||||
|
||||
|
||||
@@ -432,7 +432,7 @@ bool check_name(const ov::OutputVector& nodes, const std::string& node_name) {
|
||||
|
||||
/**
|
||||
* @brief Parse scale factors per input
|
||||
* Format : <input_name1>:<sf1>,<input2>:<sf2> or just <sf>
|
||||
* Format : <input_name1>=<sf1>,<input2>=<sf2> or just <sf>
|
||||
* @param inputs model inputs
|
||||
* @param values_string values_string input string
|
||||
* @return map of scale factors per input
|
||||
@@ -454,11 +454,11 @@ std::map<std::string, float> parse_scale_factors(const ov::OutputVector& inputs,
|
||||
std::map<std::string, float> result;
|
||||
auto scale_factor_strings = split(values_string, ',');
|
||||
for (auto& scale_factor_string : scale_factor_strings) {
|
||||
auto values = split(scale_factor_string, ':');
|
||||
auto values = split(scale_factor_string, '=');
|
||||
if (values.size() == 1) {
|
||||
if (scale_factor_strings.size() != 1) {
|
||||
throw std::logic_error("Unrecognized scale factor format! "
|
||||
"Please specify <input_name1>:<sf1>,<input_name2>:<sf2> or "
|
||||
"Please specify <input_name1>=<sf1>,<input_name2>=<sf2> or "
|
||||
"just <sf> to be applied to all inputs");
|
||||
}
|
||||
auto scale_factor = get_sf(values.at(0));
|
||||
@@ -468,8 +468,7 @@ std::map<std::string, float> parse_scale_factors(const ov::OutputVector& inputs,
|
||||
} else if (values.size() > 0) {
|
||||
auto sf_sting = values.back();
|
||||
values.pop_back();
|
||||
// input name can contain port, concat back
|
||||
auto input_name = concat(values, ':');
|
||||
auto input_name = values.back();
|
||||
check_name(inputs, input_name);
|
||||
result[input_name] = get_sf(sf_sting, input_name);
|
||||
}
|
||||
@@ -535,3 +534,37 @@ std::map<std::string, std::string> parse_input_layouts(const std::string& layout
|
||||
throw std::logic_error("Can't parse input parameter string: " + layout_string);
|
||||
return return_value;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Parse parameters for inputs/outputs like as "<name1>=<file1.ark/.npz>,<name2>=<file2.ark/.npz>" or
|
||||
* "<file.ark/.npz>" in case of one input/output
|
||||
* @param file_paths_string input/output path
|
||||
* @return pair of filename and vector of tensor_names
|
||||
*/
|
||||
std::pair<std::string, std::vector<std::string>> parse_parameters(const std::string file_paths_string) {
|
||||
auto search_string = file_paths_string;
|
||||
char comma_delim = ',';
|
||||
char equal_delim = '=';
|
||||
std::string filename = "";
|
||||
std::vector<std::string> tensor_names;
|
||||
std::vector<std::string> filenames;
|
||||
if (!std::count(search_string.begin(), search_string.end(), comma_delim) &&
|
||||
!std::count(search_string.begin(), search_string.end(), equal_delim)) {
|
||||
return {search_string, tensor_names};
|
||||
}
|
||||
search_string += comma_delim;
|
||||
std::vector<std::string> splitted = split(search_string, comma_delim);
|
||||
for (size_t j = 0; j < splitted.size(); j++) {
|
||||
auto semicolon_pos = splitted[j].find_first_of(equal_delim);
|
||||
if (semicolon_pos != std::string::npos) {
|
||||
tensor_names.push_back(splitted[j].substr(0, semicolon_pos));
|
||||
filenames.push_back(splitted[j].substr(semicolon_pos + 1, std::string::npos));
|
||||
}
|
||||
}
|
||||
for (std::vector<std::string>::const_iterator name = filenames.begin(); name != filenames.end(); ++name) {
|
||||
filename += *name;
|
||||
if (name != filenames.end() - 1)
|
||||
filename += comma_delim;
|
||||
}
|
||||
return {filename, tensor_names};
|
||||
}
|
||||
@@ -186,16 +186,16 @@ class SamplesCommonTestClass():
|
||||
return model
|
||||
|
||||
@staticmethod
|
||||
def join_env_path(param, executable_path):
|
||||
def join_env_path(param, executable_path, complete_path=True):
|
||||
gpu_lib_path = os.path.join(os.environ.get('IE_APP_PATH'), 'lib')
|
||||
if 'i' in param:
|
||||
# If batch > 1, then concatenate images
|
||||
if ' ' in param['i']:
|
||||
param['i'] = param['i'].split(' ')
|
||||
else:
|
||||
elif complete_path:
|
||||
param['i'] = list([param['i']])
|
||||
for k in param.keys():
|
||||
if ('i' == k):
|
||||
if ('i' == k) and complete_path:
|
||||
param['i'] = [os.path.join(Environment.env['test_data'], e) for e in param['i']]
|
||||
param['i'] = ' '.join(param['i'])
|
||||
elif ('ref_m' == k):
|
||||
@@ -236,10 +236,10 @@ class SamplesCommonTestClass():
|
||||
param['l'] = os.path.join(Environment.env['test_data'], param['l'])
|
||||
elif ('pp' == k):
|
||||
param['pp'] = gpu_lib_path
|
||||
elif ('r' == k):
|
||||
elif ('r' == k) and complete_path:
|
||||
if len(param['r']) > 0:
|
||||
param['r'] = os.path.join(Environment.env['test_data'], param['r'])
|
||||
elif ('o' == k):
|
||||
elif ('o' == k) and complete_path:
|
||||
param['o'] = os.path.join(Environment.env['out_directory'], param['o'])
|
||||
elif ('wg' == k):
|
||||
param['wg'] = os.path.join(Environment.env['out_directory'], param['wg'])
|
||||
@@ -344,7 +344,7 @@ class SamplesCommonTestClass():
|
||||
"Path for test data {} is not exist!".format(Environment.env['test_data'])
|
||||
cls.output_dir = Environment.env['out_directory']
|
||||
|
||||
def _test(self, param, use_preffix=True, get_cmd_func=None, get_shell_result=False, long_hyphen=None):
|
||||
def _test(self, param, use_preffix=True, get_cmd_func=None, get_shell_result=False, long_hyphen=None, complete_path=True):
|
||||
"""
|
||||
:param param:
|
||||
:param use_preffix: use it when sample doesn't require keys (i.e. hello_classification <path_to_model> <path_to_image>
|
||||
@@ -379,7 +379,7 @@ class SamplesCommonTestClass():
|
||||
if get_cmd_func is None:
|
||||
get_cmd_func = self.get_cmd_line
|
||||
|
||||
self.join_env_path(param_cp, executable_path=self.executable_path)
|
||||
self.join_env_path(param_cp, executable_path=self.executable_path, complete_path=complete_path)
|
||||
|
||||
# Updating all attributes in the original dictionary (param), because param_cp was changes (join_env_path)
|
||||
for key in param.keys():
|
||||
|
||||
@@ -15,6 +15,7 @@ import pytest
|
||||
import sys
|
||||
import logging as log
|
||||
from common.samples_common_test_clas import SamplesCommonTestClass
|
||||
from common.samples_common_test_clas import Environment
|
||||
from common.samples_common_test_clas import get_tests
|
||||
from common.common_utils import parse_avg_err
|
||||
|
||||
@@ -27,11 +28,24 @@ test_data = get_tests(cmd_params={'i': [os.path.join('ark', 'dev93_10.ark')],
|
||||
'o': ['res_output.ark'],
|
||||
'r': [os.path.join('ark', 'dev93_scores_10.ark')],
|
||||
'qb': [8, 16],
|
||||
'sf': ["Parameter:2175.43", "2175.43"],
|
||||
'sf': ["2175.43"],
|
||||
'q': ["static", "user"],
|
||||
'd': ['GNA_SW_EXACT']},
|
||||
use_device=False
|
||||
)
|
||||
|
||||
new_format_test_data = get_tests(cmd_params={'i': ['Parameter=' + os.path.join(Environment.env['test_data'], 'ark', 'dev93_10.ark')],
|
||||
'm': [os.path.join('wsj', 'FP32', 'wsj_dnn5b.xml')],
|
||||
'layout': ["[NC]"],
|
||||
'bs': [1],
|
||||
'o': ['affinetransform14/Fused_Add_:0=' + os.path.join(Environment.env['test_data'], 'res_output.ark')],
|
||||
'r': ['affinetransform14/Fused_Add_:0=' + os.path.join(Environment.env['test_data'], 'ark', 'dev93_scores_10.ark')],
|
||||
'qb': [8],
|
||||
'sf': ["Parameter=2175.43"],
|
||||
'q': ["static"],
|
||||
'd': ['GNA_SW_EXACT']},
|
||||
use_device=False
|
||||
)
|
||||
|
||||
class TestSpeechSample(SamplesCommonTestClass):
|
||||
@classmethod
|
||||
@@ -43,7 +57,14 @@ class TestSpeechSample(SamplesCommonTestClass):
|
||||
@pytest.mark.parametrize("param", test_data)
|
||||
def test_speech_sample_nthreads(self, param):
|
||||
stdout = self._test(param).split('\n')
|
||||
assert os.path.isfile(param['o']), "Ark file after infer was not found"
|
||||
|
||||
avg_error = parse_avg_err(stdout)
|
||||
log.info('Average scores diff: {}'.format(avg_error))
|
||||
assert avg_error <= self.threshold
|
||||
|
||||
@pytest.mark.parametrize("param", new_format_test_data)
|
||||
def test_speech_sample_new_format(self, param):
|
||||
stdout = self._test(param, complete_path=False).split('\n')
|
||||
|
||||
avg_error = parse_avg_err(stdout)
|
||||
log.info('Average scores diff: {}'.format(avg_error))
|
||||
|
||||
Reference in New Issue
Block a user