From fad66d8442a4d60f7fdcd2ae4fe78368ac252b7d Mon Sep 17 00:00:00 2001 From: Maxim Gordeev Date: Thu, 31 Mar 2022 10:09:31 +0300 Subject: [PATCH] [IE Samples] New command line parameters format for speech sample (#11051) * New command line parameters format for speech sample * fixed notes * changed format for scale factor * changed format for scale factor in tests * added more variants, when name is directy specified for i/o/r like it is done for sf * removed nthreads flag * fixed notes * changed output params * updated tests with new format Co-authored-by: Alexander Zhogov --- samples/cpp/speech_sample/main.cpp | 64 +++++++++++-------- samples/cpp/speech_sample/speech_sample.hpp | 7 -- samples/cpp/speech_sample/utils.hpp | 43 +++++++++++-- .../common/samples_common_test_clas.py | 14 ++-- .../smoke_tests/test_speech_sample.py | 25 +++++++- 5 files changed, 105 insertions(+), 48 deletions(-) diff --git a/samples/cpp/speech_sample/main.cpp b/samples/cpp/speech_sample/main.cpp index 861058bf94c..c034c64dad7 100644 --- a/samples/cpp/speech_sample/main.cpp +++ b/samples/cpp/speech_sample/main.cpp @@ -49,7 +49,10 @@ int main(int argc, char* argv[]) { BaseFile* fileOutput; ArkFile arkFile; NumpyFile numpyFile; - auto extInputFile = fileExt(FLAGS_i); + std::pair> input_data; + if (!FLAGS_i.empty()) + input_data = parse_parameters(FLAGS_i); + auto extInputFile = fileExt(input_data.first); if (extInputFile == "ark") { file = &arkFile; } else if (extInputFile == "npz") { @@ -60,9 +63,9 @@ int main(int argc, char* argv[]) { std::vector inputFiles; std::vector numBytesThisUtterance; uint32_t numUtterances(0); - if (!FLAGS_i.empty()) { + if (!input_data.first.empty()) { std::string outStr; - std::istringstream stream(FLAGS_i); + std::istringstream stream(input_data.first); uint32_t currentNumUtterances(0), currentNumBytesThisUtterance(0); while (getline(stream, outStr, ',')) { std::string filename(fileNameNoExt(outStr) + "." + extInputFile); @@ -89,19 +92,26 @@ int main(int argc, char* argv[]) { std::vector output_names; std::vector ports; // --------------------------- Processing custom outputs --------------------------------------------- - if (!FLAGS_oname.empty()) { - output_names = convert_str_to_vector(FLAGS_oname); - for (const auto& output_name : output_names) { - auto pos_layer = output_name.rfind(":"); - if (pos_layer == std::string::npos) { - throw std::logic_error("Output " + output_name + " doesn't have a port"); - } - outputs.push_back(output_name.substr(0, pos_layer)); - try { - ports.push_back(std::stoi(output_name.substr(pos_layer + 1))); - } catch (const std::exception&) { - throw std::logic_error("Ports should have integer type"); - } + std::pair> output_data; + std::pair> reference_data; + if (!FLAGS_o.empty()) + output_data = parse_parameters(FLAGS_o); + if (!FLAGS_r.empty()) + reference_data = parse_parameters(FLAGS_r); + if (!output_data.second.empty()) + output_names = output_data.second; + else if (!reference_data.second.empty()) + output_names = reference_data.second; + for (const auto& output_name : output_names) { + auto pos_layer = output_name.rfind(":"); + if (pos_layer == std::string::npos) { + throw std::logic_error("Output " + output_name + " doesn't have a port"); + } + outputs.push_back(output_name.substr(0, pos_layer)); + try { + ports.push_back(std::stoi(output_name.substr(pos_layer + 1))); + } catch (const std::exception&) { + throw std::logic_error("Ports should have integer type"); } } // ------------------------------ Preprocessing ------------------------------------------------------ @@ -304,8 +314,8 @@ int main(int argc, char* argv[]) { std::vector ptrInputBlobs; auto cInputInfo = executableNet.inputs(); check_number_of_inputs(cInputInfo.size(), numInputFiles); - if (!FLAGS_iname.empty()) { - std::vector inputNameBlobs = convert_str_to_vector(FLAGS_iname); + if (!input_data.second.empty()) { + std::vector inputNameBlobs = input_data.second; if (inputNameBlobs.size() != cInputInfo.size()) { std::string errMessage(std::string("Number of network inputs ( ") + std::to_string(cInputInfo.size()) + " ) is not equal to the number of inputs entered in the -iname argument ( " + @@ -328,15 +338,15 @@ int main(int argc, char* argv[]) { std::vector output_name_files; std::vector reference_name_files; size_t count_file = 1; - if (!FLAGS_o.empty()) { - output_name_files = convert_str_to_vector(FLAGS_o); + if (!output_data.first.empty()) { + output_name_files = convert_str_to_vector(output_data.first); if (output_name_files.size() != outputs.size() && !outputs.empty()) { throw std::logic_error("The number of output files is not equal to the number of network outputs."); } count_file = output_name_files.empty() ? 1 : output_name_files.size(); } - if (!FLAGS_r.empty()) { - reference_name_files = convert_str_to_vector(FLAGS_r); + if (!reference_data.first.empty()) { + reference_name_files = convert_str_to_vector(reference_data.first); if (reference_name_files.size() != outputs.size() && !outputs.empty()) { throw std::logic_error("The number of reference files is not equal to the number of network outputs."); } @@ -429,9 +439,9 @@ int main(int argc, char* argv[]) { BaseFile* fileReferenceScores; std::string refUtteranceName; - if (!FLAGS_r.empty()) { + if (!reference_data.first.empty()) { /** Read file with reference scores **/ - auto exReferenceScoresFile = fileExt(FLAGS_r); + auto exReferenceScoresFile = fileExt(reference_data.first); if (exReferenceScoresFile == "ark") { fileReferenceScores = &arkFile; } else if (exReferenceScoresFile == "npz") { @@ -540,12 +550,12 @@ int main(int argc, char* argv[]) { continue; } ptrInputBlobs.clear(); - if (FLAGS_iname.empty()) { + if (input_data.second.empty()) { for (auto& input : cInputInfo) { ptrInputBlobs.push_back(inferRequest.inferRequest.get_tensor(input)); } } else { - std::vector inputNameBlobs = convert_str_to_vector(FLAGS_iname); + std::vector inputNameBlobs = input_data.second; for (const auto& input : inputNameBlobs) { ov::Tensor blob = inferRequests.begin()->inferRequest.get_tensor(input); if (!blob) { @@ -638,7 +648,7 @@ int main(int argc, char* argv[]) { for (size_t next_output = 0; next_output < count_file; next_output++) { if (!FLAGS_o.empty()) { - auto exOutputScoresFile = fileExt(FLAGS_o); + auto exOutputScoresFile = fileExt(output_data.first); if (exOutputScoresFile == "ark") { fileOutput = &arkFile; } else if (exOutputScoresFile == "npz") { diff --git a/samples/cpp/speech_sample/speech_sample.hpp b/samples/cpp/speech_sample/speech_sample.hpp index f398d709cb3..c0961b768ac 100644 --- a/samples/cpp/speech_sample/speech_sample.hpp +++ b/samples/cpp/speech_sample/speech_sample.hpp @@ -96,10 +96,6 @@ static const char scale_factor_message[] = /// @brief message for batch size argument static const char batch_size_message[] = "Optional. Batch size 1-8 (default 1)"; -/// @brief message for #threads for CPU inference -static const char infer_num_threads_message[] = "Optional. Number of threads to use for concurrent async" - " inference requests on the GNA."; - /// @brief message for left context window argument static const char context_window_message_l[] = "Optional. Number of frames for left context windows (default is 0). " @@ -184,9 +180,6 @@ DEFINE_string(sf, "", scale_factor_message); /// @brief Batch size (default 0) DEFINE_int32(bs, 0, batch_size_message); -/// @brief Number of threads to use for inference on the CPU (also affects Hetero cases) -DEFINE_int32(nthreads, 1, infer_num_threads_message); - /// @brief Right context window size (default 0) DEFINE_int32(cw_r, 0, context_window_message_r); diff --git a/samples/cpp/speech_sample/utils.hpp b/samples/cpp/speech_sample/utils.hpp index 4626cf55207..f194c4e66ab 100644 --- a/samples/cpp/speech_sample/utils.hpp +++ b/samples/cpp/speech_sample/utils.hpp @@ -432,7 +432,7 @@ bool check_name(const ov::OutputVector& nodes, const std::string& node_name) { /** * @brief Parse scale factors per input - * Format : :,: or just + * Format : =,= or just * @param inputs model inputs * @param values_string values_string input string * @return map of scale factors per input @@ -454,11 +454,11 @@ std::map parse_scale_factors(const ov::OutputVector& inputs, std::map result; auto scale_factor_strings = split(values_string, ','); for (auto& scale_factor_string : scale_factor_strings) { - auto values = split(scale_factor_string, ':'); + auto values = split(scale_factor_string, '='); if (values.size() == 1) { if (scale_factor_strings.size() != 1) { throw std::logic_error("Unrecognized scale factor format! " - "Please specify :,: or " + "Please specify =,= or " "just to be applied to all inputs"); } auto scale_factor = get_sf(values.at(0)); @@ -468,8 +468,7 @@ std::map parse_scale_factors(const ov::OutputVector& inputs, } else if (values.size() > 0) { auto sf_sting = values.back(); values.pop_back(); - // input name can contain port, concat back - auto input_name = concat(values, ':'); + auto input_name = values.back(); check_name(inputs, input_name); result[input_name] = get_sf(sf_sting, input_name); } @@ -535,3 +534,37 @@ std::map parse_input_layouts(const std::string& layout throw std::logic_error("Can't parse input parameter string: " + layout_string); return return_value; } + +/** + * @brief Parse parameters for inputs/outputs like as "=,=" or + * "" in case of one input/output + * @param file_paths_string input/output path + * @return pair of filename and vector of tensor_names + */ +std::pair> parse_parameters(const std::string file_paths_string) { + auto search_string = file_paths_string; + char comma_delim = ','; + char equal_delim = '='; + std::string filename = ""; + std::vector tensor_names; + std::vector filenames; + if (!std::count(search_string.begin(), search_string.end(), comma_delim) && + !std::count(search_string.begin(), search_string.end(), equal_delim)) { + return {search_string, tensor_names}; + } + search_string += comma_delim; + std::vector splitted = split(search_string, comma_delim); + for (size_t j = 0; j < splitted.size(); j++) { + auto semicolon_pos = splitted[j].find_first_of(equal_delim); + if (semicolon_pos != std::string::npos) { + tensor_names.push_back(splitted[j].substr(0, semicolon_pos)); + filenames.push_back(splitted[j].substr(semicolon_pos + 1, std::string::npos)); + } + } + for (std::vector::const_iterator name = filenames.begin(); name != filenames.end(); ++name) { + filename += *name; + if (name != filenames.end() - 1) + filename += comma_delim; + } + return {filename, tensor_names}; +} \ No newline at end of file diff --git a/tests/samples_tests/smoke_tests/common/samples_common_test_clas.py b/tests/samples_tests/smoke_tests/common/samples_common_test_clas.py index 9e789976a54..b62c2fdb6f8 100644 --- a/tests/samples_tests/smoke_tests/common/samples_common_test_clas.py +++ b/tests/samples_tests/smoke_tests/common/samples_common_test_clas.py @@ -186,16 +186,16 @@ class SamplesCommonTestClass(): return model @staticmethod - def join_env_path(param, executable_path): + def join_env_path(param, executable_path, complete_path=True): gpu_lib_path = os.path.join(os.environ.get('IE_APP_PATH'), 'lib') if 'i' in param: # If batch > 1, then concatenate images if ' ' in param['i']: param['i'] = param['i'].split(' ') - else: + elif complete_path: param['i'] = list([param['i']]) for k in param.keys(): - if ('i' == k): + if ('i' == k) and complete_path: param['i'] = [os.path.join(Environment.env['test_data'], e) for e in param['i']] param['i'] = ' '.join(param['i']) elif ('ref_m' == k): @@ -236,10 +236,10 @@ class SamplesCommonTestClass(): param['l'] = os.path.join(Environment.env['test_data'], param['l']) elif ('pp' == k): param['pp'] = gpu_lib_path - elif ('r' == k): + elif ('r' == k) and complete_path: if len(param['r']) > 0: param['r'] = os.path.join(Environment.env['test_data'], param['r']) - elif ('o' == k): + elif ('o' == k) and complete_path: param['o'] = os.path.join(Environment.env['out_directory'], param['o']) elif ('wg' == k): param['wg'] = os.path.join(Environment.env['out_directory'], param['wg']) @@ -344,7 +344,7 @@ class SamplesCommonTestClass(): "Path for test data {} is not exist!".format(Environment.env['test_data']) cls.output_dir = Environment.env['out_directory'] - def _test(self, param, use_preffix=True, get_cmd_func=None, get_shell_result=False, long_hyphen=None): + def _test(self, param, use_preffix=True, get_cmd_func=None, get_shell_result=False, long_hyphen=None, complete_path=True): """ :param param: :param use_preffix: use it when sample doesn't require keys (i.e. hello_classification @@ -379,7 +379,7 @@ class SamplesCommonTestClass(): if get_cmd_func is None: get_cmd_func = self.get_cmd_line - self.join_env_path(param_cp, executable_path=self.executable_path) + self.join_env_path(param_cp, executable_path=self.executable_path, complete_path=complete_path) # Updating all attributes in the original dictionary (param), because param_cp was changes (join_env_path) for key in param.keys(): diff --git a/tests/samples_tests/smoke_tests/test_speech_sample.py b/tests/samples_tests/smoke_tests/test_speech_sample.py index 0ee5095d719..686a1e803a8 100644 --- a/tests/samples_tests/smoke_tests/test_speech_sample.py +++ b/tests/samples_tests/smoke_tests/test_speech_sample.py @@ -15,6 +15,7 @@ import pytest import sys import logging as log from common.samples_common_test_clas import SamplesCommonTestClass +from common.samples_common_test_clas import Environment from common.samples_common_test_clas import get_tests from common.common_utils import parse_avg_err @@ -27,11 +28,24 @@ test_data = get_tests(cmd_params={'i': [os.path.join('ark', 'dev93_10.ark')], 'o': ['res_output.ark'], 'r': [os.path.join('ark', 'dev93_scores_10.ark')], 'qb': [8, 16], - 'sf': ["Parameter:2175.43", "2175.43"], + 'sf': ["2175.43"], 'q': ["static", "user"], 'd': ['GNA_SW_EXACT']}, use_device=False ) + +new_format_test_data = get_tests(cmd_params={'i': ['Parameter=' + os.path.join(Environment.env['test_data'], 'ark', 'dev93_10.ark')], + 'm': [os.path.join('wsj', 'FP32', 'wsj_dnn5b.xml')], + 'layout': ["[NC]"], + 'bs': [1], + 'o': ['affinetransform14/Fused_Add_:0=' + os.path.join(Environment.env['test_data'], 'res_output.ark')], + 'r': ['affinetransform14/Fused_Add_:0=' + os.path.join(Environment.env['test_data'], 'ark', 'dev93_scores_10.ark')], + 'qb': [8], + 'sf': ["Parameter=2175.43"], + 'q': ["static"], + 'd': ['GNA_SW_EXACT']}, + use_device=False + ) class TestSpeechSample(SamplesCommonTestClass): @classmethod @@ -43,7 +57,14 @@ class TestSpeechSample(SamplesCommonTestClass): @pytest.mark.parametrize("param", test_data) def test_speech_sample_nthreads(self, param): stdout = self._test(param).split('\n') - assert os.path.isfile(param['o']), "Ark file after infer was not found" + + avg_error = parse_avg_err(stdout) + log.info('Average scores diff: {}'.format(avg_error)) + assert avg_error <= self.threshold + + @pytest.mark.parametrize("param", new_format_test_data) + def test_speech_sample_new_format(self, param): + stdout = self._test(param, complete_path=False).split('\n') avg_error = parse_avg_err(stdout) log.info('Average scores diff: {}'.format(avg_error))