Added processing of layout for speech sample (#10254)

* Added processing of layout for speech sample

* fixed notes

* some improvements

* Code style format

* changed NCC value for NullStatement

* improved batch processing

* added loading batch for imported model

* fixed notes

* fixed notes

* added layout parameter to azure tests

samples/cpp/speech_sample/main.cpp

@@ -83,7 +83,7 @@ int main(int argc, char* argv[]) {
         // -------------------------------------
         ov::Core core;
         slog::info << "Loading model files:" << slog::endl << FLAGS_m << slog::endl;
-        uint32_t batchSize = (FLAGS_cw_r > 0 || FLAGS_cw_l > 0) ? 1 : (uint32_t)FLAGS_bs;
+        uint32_t batchSize = (FLAGS_cw_r > 0 || FLAGS_cw_l > 0 || !FLAGS_bs) ? 1 : (uint32_t)FLAGS_bs;
         std::shared_ptr<ov::Model> model;
         std::vector<std::string> outputs;
         std::vector<size_t> ports;
@@ -115,16 +115,38 @@ int main(int argc, char* argv[]) {
                 }
             }
             check_number_of_inputs(model->inputs().size(), numInputFiles);
-            const ov::Layout tensor_layout{"NC"};
             ov::preprocess::PrePostProcessor proc(model);
-            for (int i = 0; i < model->inputs().size(); i++) {
-                proc.input(i).tensor().set_element_type(ov::element::f32).set_layout(tensor_layout);
+            const auto& inputs = model->inputs();
+            std::map<std::string, std::string> custom_layouts;
+            if (!FLAGS_layout.empty()) {
+                custom_layouts = parse_input_layouts(FLAGS_layout, inputs);
+            }
+            for (const auto& input : inputs) {
+                const auto& item_name = input.get_any_name();
+                auto& in = proc.input(item_name);
+                in.tensor().set_element_type(ov::element::f32);
+                // Explicitly set inputs layout
+                if (custom_layouts.count(item_name) > 0) {
+                    in.model().set_layout(ov::Layout(custom_layouts.at(item_name)));
+                }
             }
             for (int i = 0; i < model->outputs().size(); i++) {
                 proc.output(i).tensor().set_element_type(ov::element::f32);
             }
             model = proc.build();
-            ov::set_batch(model, batchSize);
+            if (FLAGS_bs) {
+                if (FLAGS_layout.empty() &&
+                    std::any_of(inputs.begin(), inputs.end(), [](const ov::Output<ov::Node>& i) {
+                        return ov::layout::get_layout(i).empty();
+                    })) {
+                    throw std::logic_error(
+                        "-bs option is set to " + std::to_string(FLAGS_bs) +
+                        " but model does not contain layout information for any input. Please "
+                        "specify it explicitly using -layout option. For example, input1[NCHW], input2[NC] or [NC]");
+                } else {
+                    ov::set_batch(model, batchSize);
+                }
+            }
         }
         // ------------------------------ Get Available Devices ------------------------------------------------------
         auto isFeature = [&](const std::string xFeature) {
@@ -235,6 +257,22 @@ int main(int argc, char* argv[]) {
                 throw std::runtime_error("Cannot open model file " + FLAGS_rg);
             }
             executableNet = core.import_model(streamrq, deviceStr, genericPluginConfig);
+            // loading batch from exported model
+            const auto& imported_inputs = executableNet.inputs();
+            if (std::any_of(imported_inputs.begin(), imported_inputs.end(), [](const ov::Output<const ov::Node>& i) {
+                    return ov::layout::get_layout(i).empty();
+                })) {
+                slog::warn << "No batch dimension was found at any input, assuming batch to be 1." << slog::endl;
+                batchSize = 1;
+            } else {
+                for (auto& info : imported_inputs) {
+                    auto imported_layout = ov::layout::get_layout(info);
+                    if (ov::layout::has_batch(imported_layout)) {
+                        batchSize = (uint32_t)info.get_shape()[ov::layout::batch_idx(imported_layout)];
+                        break;
+                    }
+                }
+            }
         }
         // --------------------------- Exporting gna model using InferenceEngine AOT API---------------------
         if (!FLAGS_wg.empty()) {
@@ -251,7 +289,8 @@ int main(int argc, char* argv[]) {
             return 0;
         }
         // ---------------------------------------------------------------------------------------------------------
-        // --------------------------- Step 3. Create infer request --------------------------------------------------
+        // --------------------------- Step 3. Create infer request
+        // --------------------------------------------------
         std::vector<InferRequestStruct> inferRequests(1);
 
         for (auto& inferRequest : inferRequests) {
@@ -433,7 +472,8 @@ int main(int argc, char* argv[]) {
                                         outputBlob =
                                             inferRequest.inferRequest.get_tensor(executableNet.output(FLAGS_oname));
                                     }
-                                    // locked memory holder should be alive all time while access to its buffer happens
+                                    // locked memory holder should be alive all time while access to its buffer
+                                    // happens
                                     auto byteSize = numScoresPerFrame * sizeof(float);
                                     std::memcpy(outputFrame, outputBlob.data<float>(), byteSize);
                                 }