From d20900e235189ba53c38c8ace4a3389a69c84ce2 Mon Sep 17 00:00:00 2001 From: Mikhail Nosov Date: Mon, 17 May 2021 13:41:15 +0300 Subject: [PATCH] [Caching] Add caching options to benchmark app (#4909) * Python API for LoadNetwork by model file name * BenchmarkApp: Add caching and LoadNetworkFromFile support 2 new options are introduced - cache_dir - enables models caching - load_from_file - use new perform "LoadNetwork" by model file name Using both parameters will achieve maximum performance of read/load network on startup Tests: 1) Run "benchmark_app -h". Help will display 2 new options. After available devices there will be list of devices with cache support 2) ./benchmark_app -d CPU -i -load_from_file Verify that some test steps are skipped (related to ReadNetwork, re-shaping etc) 3) Pre-requisite: support of caching shall be enabled for Template plugin ./benchmark_app -d TEMPLATE -i -load_from_file -cache_dir someDir Verify that "someDir" is created and generated blob is available Run again, verify that loading works as well (should be faster as it will not load onnx model) 4) Run same test as (3), but without -load_from_file option. Verify that cache is properly created For some devices loadNetwork time shall be improved when cache is available * Removed additional timing prints * Correction from old code * Revert "Removed additional timing prints" Additional change - when .blob is chosen instead of .xml, it takes priority over caching flags * Removed new time printings As discussed, these time measurements like 'total first inference time' will be available in 'timeTests' scripts * Fix clang-format issues --- .../samples/benchmark_app/README.md | 2 ++ .../samples/benchmark_app/benchmark_app.hpp | 16 +++++++++ .../samples/benchmark_app/main.cpp | 26 ++++++++++++-- tools/benchmark/README.md | 5 +++ tools/benchmark/benchmark.py | 13 +++++++ tools/benchmark/main.py | 35 +++++++++++++++++-- tools/benchmark/parameters.py | 4 +++ 7 files changed, 96 insertions(+), 5 deletions(-) diff --git a/inference-engine/samples/benchmark_app/README.md b/inference-engine/samples/benchmark_app/README.md index 432e8b28ec0..7c61bc570d5 100644 --- a/inference-engine/samples/benchmark_app/README.md +++ b/inference-engine/samples/benchmark_app/README.md @@ -93,6 +93,8 @@ Options: -progress Optional. Show progress bar (can affect performance measurement). Default values is "false". -shape Optional. Set shape for input. For example, "input1[1,3,224,224],input2[1,4]" or "[1,3,224,224]" in case of one input size. -layout Optional. Prompts how network layouts should be treated by application. For example, "input1[NCHW],input2[NC]" or "[NCHW]" in case of one input size. + -cache_dir "" Optional. Enables caching of loaded models to specified directory. + -load_from_file Optional. Loads model from file directly without ReadNetwork. CPU-specific performance options: -nstreams "" Optional. Number of streams to use for inference on the CPU, GPU or MYRIAD devices diff --git a/inference-engine/samples/benchmark_app/benchmark_app.hpp b/inference-engine/samples/benchmark_app/benchmark_app.hpp index 8cc5d19c781..af18c908e31 100644 --- a/inference-engine/samples/benchmark_app/benchmark_app.hpp +++ b/inference-engine/samples/benchmark_app/benchmark_app.hpp @@ -122,6 +122,14 @@ static const char shape_message[] = "Optional. Set shape for input. For example, static const char layout_message[] = "Optional. Prompts how network layouts should be treated by application. " "For example, \"input1[NCHW],input2[NC]\" or \"[NCHW]\" in case of one input size."; +// @brief message for enabling caching +static const char cache_dir_message[] = "Optional. Enables caching of loaded models to specified directory. " + "List of devices which support caching is shown at the end of this message."; + +// @brief message for single load network +static const char load_from_file_message[] = "Optional. Loads model from file directly without ReadNetwork." + "All CNNNetwork options (like re-shape) will be ignored"; + // @brief message for quantization bits static const char gna_qb_message[] = "Optional. Weight bits for quantization: 8 or 16 (default)"; @@ -238,6 +246,12 @@ DEFINE_string(op, "", outputs_precision_message); /// Overwrites layout from ip and op options for specified layers."; DEFINE_string(iop, "", iop_message); +/// @brief Define parameter for cache model dir
+DEFINE_string(cache_dir, "", cache_dir_message); + +/// @brief Define flag for load network from model file by name without ReadNetwork
+DEFINE_bool(load_from_file, false, load_from_file_message); + /** * @brief This function show a help message */ @@ -262,6 +276,8 @@ static void showUsage() { std::cout << " -progress " << progress_message << std::endl; std::cout << " -shape " << shape_message << std::endl; std::cout << " -layout " << layout_message << std::endl; + std::cout << " -cache_dir \"\" " << cache_dir_message << std::endl; + std::cout << " -load_from_file " << load_from_file_message << std::endl; std::cout << std::endl << " device-specific performance options:" << std::endl; std::cout << " -nstreams \"\" " << infer_num_streams_message << std::endl; std::cout << " -nthreads \"\" " << infer_num_threads_message << std::endl; diff --git a/inference-engine/samples/benchmark_app/main.cpp b/inference-engine/samples/benchmark_app/main.cpp index 5a34e558cd3..849dc05ad33 100644 --- a/inference-engine/samples/benchmark_app/main.cpp +++ b/inference-engine/samples/benchmark_app/main.cpp @@ -330,7 +330,29 @@ int main(int argc, char* argv[]) { std::string topology_name = ""; benchmark_app::InputsInfo app_inputs_info; std::string output_name; - if (!isNetworkCompiled) { + + // Takes priority over config from file + if (!FLAGS_cache_dir.empty()) { + ie.SetConfig({{CONFIG_KEY(CACHE_DIR), FLAGS_cache_dir}}); + } + + if (FLAGS_load_from_file && !isNetworkCompiled) { + next_step(); + slog::info << "Skipping the step for loading network from file" << slog::endl; + next_step(); + slog::info << "Skipping the step for loading network from file" << slog::endl; + next_step(); + slog::info << "Skipping the step for loading network from file" << slog::endl; + auto startTime = Time::now(); + exeNetwork = ie.LoadNetwork(FLAGS_m, device_name); + auto duration_ms = double_to_string(get_total_ms_time(startTime)); + slog::info << "Load network took " << duration_ms << " ms" << slog::endl; + if (statistics) + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"load network time (ms)", duration_ms}}); + if (batchSize == 0) { + batchSize = 1; + } + } else if (!isNetworkCompiled) { // ----------------- 4. Reading the Intermediate Representation network // ---------------------------------------- next_step(); @@ -363,7 +385,7 @@ int main(int argc, char* argv[]) { slog::info << "Reshaping network: " << getShapesString(shapes) << slog::endl; startTime = Time::now(); cnnNetwork.reshape(shapes); - auto duration_ms = double_to_string(get_total_ms_time(startTime)); + duration_ms = double_to_string(get_total_ms_time(startTime)); slog::info << "Reshape network took " << duration_ms << " ms" << slog::endl; if (statistics) statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, {{"reshape network time (ms)", duration_ms}}); diff --git a/tools/benchmark/README.md b/tools/benchmark/README.md index 280b7a0ef53..50caa430fd4 100644 --- a/tools/benchmark/README.md +++ b/tools/benchmark/README.md @@ -142,6 +142,11 @@ Options: parameters. Please note, command line parameters have higher priority then parameters from configuration file. + -cdir CACHE_DIR, --cache_dir CACHE_DIR + Optional. Enable model caching to specified directory + -lfile [LOAD_FROM_FILE], --load_from_file [LOAD_FROM_FILE] + Optional. Loads model from file directly without + read_network. ``` Running the application with the empty list of options yields the usage message given above and an error message. diff --git a/tools/benchmark/benchmark.py b/tools/benchmark/benchmark.py index 754bdd194e7..c73477acfc2 100644 --- a/tools/benchmark/benchmark.py +++ b/tools/benchmark/benchmark.py @@ -46,6 +46,9 @@ class Benchmark: for device in config.keys(): self.ie.set_config(config[device], device) + def set_cache_dir(self, cache_dir: str): + self.ie.set_config({'CACHE_DIR': cache_dir}, '') + def read_network(self, path_to_model: str): model_filename = os.path.abspath(path_to_model) head, ext = os.path.splitext(model_filename) @@ -63,6 +66,16 @@ class Benchmark: return exe_network + def load_network_from_file(self, path_to_model: str, config = {}): + exe_network = self.ie.load_network(path_to_model, + self.device, + config=config, + num_requests=1 if self.api_type == 'sync' else self.nireq or 0) + # Number of requests + self.nireq = len(exe_network.requests) + + return exe_network + def import_network(self, path_to_file : str, config = {}): exe_network = self.ie.import_network(model_file=path_to_file, device_name=self.device, diff --git a/tools/benchmark/main.py b/tools/benchmark/main.py index 998e95638fb..29aff45742e 100644 --- a/tools/benchmark/main.py +++ b/tools/benchmark/main.py @@ -176,12 +176,41 @@ def run(args): benchmark.set_config(config) batch_size = args.batch_size - if not is_network_compiled: + if args.cache_dir: + benchmark.set_cache_dir(args.cache_dir) + + topology_name = "" + load_from_file_enabled = is_flag_set_in_command_line('load_from_file') or is_flag_set_in_command_line('lfile') + if load_from_file_enabled and not is_network_compiled: + next_step() + print("Skipping the step for loading network from file") + next_step() + print("Skipping the step for loading network from file") + next_step() + print("Skipping the step for loading network from file") + + # --------------------- 7. Loading the model to the device ------------------------------------------------- + next_step() + + start_time = datetime.utcnow() + exe_network = benchmark.load_network(args.path_to_model) + duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}" + logger.info(f"Load network took {duration_ms} ms") + if statistics: + statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, + [ + ('load network time (ms)', duration_ms) + ]) + app_inputs_info, _ = get_inputs_info(args.shape, args.layout, args.batch_size, exe_network.input_info) + if batch_size == 0: + batch_size = 1 + elif not is_network_compiled: # --------------------- 4. Read the Intermediate Representation of the network ----------------------------- next_step() start_time = datetime.utcnow() ie_network = benchmark.read_network(args.path_to_model) + topology_name = ie_network.name duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}" logger.info(f"Read network took {duration_ms} ms") if statistics: @@ -281,7 +310,7 @@ def run(args): if statistics: statistics.add_parameters(StatisticsReport.Category.RUNTIME_CONFIG, [ - ('topology', ie_network.name), + ('topology', topology_name), ('target device', device_name), ('API', args.api_type), ('precision', "UNSPECIFIED"), @@ -308,7 +337,7 @@ def run(args): progress_bar = ProgressBar(progress_bar_total_count, args.stream_output, args.progress) if args.progress else None - duration_ms = f"{benchmark.first_infer(exe_network):.2f}" + duration_ms = f"{benchmark.first_infer(exe_network):.2f}" logger.info(f"First inference took {duration_ms} ms") if statistics: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, diff --git a/tools/benchmark/parameters.py b/tools/benchmark/parameters.py index ca29fb5158c..73e1a931928 100644 --- a/tools/benchmark/parameters.py +++ b/tools/benchmark/parameters.py @@ -124,6 +124,10 @@ def parse_args(): help='Optional. Specifies precision for all output layers of the network.') args.add_argument('-iop', '--input_output_precision', type=str, required=False, help='Optional. Specifies precision for input and output layers by name. Example: -iop "input:FP16, output:FP16". Notice that quotes are required. Overwrites precision from ip and op options for specified layers.') + args.add_argument('-cdir', '--cache_dir', type=str, required=False, default='', + help="Optional. Enable model caching to specified directory") + args.add_argument('-lfile', '--load_from_file', required=False, nargs='?', default=argparse.SUPPRESS, + help="Optional. Loads model from file directly without read_network.") parsed_args = parser.parse_args() return parsed_args