pdpd Place: remaining methods, refactoring, unit tests (#6637)

* pdpd Place: remaining methods, refactoring, unit tests

* resolve merge issues

* use references instead of shared_ptr for Var and Op descs

* Add import pdpd fuzz test; To enable collection of pdpd models, cmake NGRAPH_PDPD_FRONTEND_ENABLE key should be set, the models will be generated to pdpd_test_models folder.

* fix ngraph codestyle

* fix review comments

* Add new methods for Place class

* fix implementation, add tests

* Place: Return nullptr instead of throwing an exception

* ngraph codestyle

* revert return nullptr

* fix build

* fix tests

* fix fuzzing tests

* fix fuzzing tests

* fix readme file

* Revert "fix readme file"

This reverts commit d061e69a0d.

* Resolve review comments

* ngraph codestyle

Co-authored-by: Somsikov, Andrey <andrey.somsikov@intel.com>

tests/fuzz/fuzz-testhelper/tokenizer.h

@@ -0,0 +1,40 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <cstdint>
+#include <cstring>
+
+class Tokenizer {
+public:
+  /// Initialize tokenizer with an input and token separator buffers.
+  Tokenizer(const void *str, size_t str_size, const void *separator,
+            size_t separator_size)
+      : str((const uint8_t *)str), str_size(str_size), separator(separator),
+        separator_size(separator_size) {}
+
+  /// Get next token.
+  const void *next(size_t *token_size) {
+    const void *token = this->str;
+    if (this->str_size >= this->separator_size) {
+      for (size_t i = 0; i < this->str_size - this->separator_size; i++)
+        if (0 == memcmp((const uint8_t *)this->str + i, this->separator,
+                        this->separator_size)) {
+          *token_size = this->str_size - this->separator_size;
+          this->str += i + this->separator_size;
+          this->str_size -= i + this->separator_size;
+          return token;
+        }
+    }
+    *token_size = this->str_size;
+    this->str = nullptr;
+    this->str_size = 0;
+    return token;
+  }
+
+private:
+  const uint8_t *str;
+  size_t str_size;
+  const void *separator;
+  size_t separator_size;
+};

tests/fuzz/scripts/init_corpus.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# Copyright (C) 2018-2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Sample usage:
+#   ./scripts/init_corpus.py ./pdpd_layer_models/**/*.pdmodel --join pdiparams
+#   mkdir -p corpus && find ./pdpd_layer_models/ -name "*.fuzz" -exec cp \{\} .//import_pdpd-corpus \;
+
+import argparse
+import glob
+import os
+from pathlib import Path
+import shutil
+import sys
+
+
+def globber(paths):
+    """Generator extending paths with wildcards"""
+    for path in paths:
+        # XXX: use non-public `has_magic` here as we'd like to differentiate between glob and normal paths
+        #  i.e. in the case when user specifies "normal" but non-existing path - we'd like to handle it by ourselves
+        if glob.has_magic(path):
+            for resolved in glob.iglob(path, recursive=True):
+                yield resolved
+        else:
+            yield path
+
+
+def main():
+    """ Main entrypoint """
+    parser = argparse.ArgumentParser(
+        description="Join multiple files of the same name to a single *.fuzz file"
+    )
+    parser.add_argument("input", nargs="+", help="A file to add to the corpus")
+    parser.add_argument(
+        "--join",
+        help="Colon separated list of file extensions to concatenate to corpus entry",
+    )
+    args = parser.parse_args()
+
+    for input in globber(args.input):
+        base = os.path.splitext(input)[0]
+        output = f"{base}.fuzz"
+        shutil.copyfile(input, output)
+        if args.join:
+            with open(output, "ab") as output_file:
+                for join in args.join.split(":"):
+                    join = f"{base}.{join}"
+                    if os.path.isfile(join):
+                        with open(join, "rb") as join_file:
+                            output_file.write(bytes("FUZZ_NEXT_FIELD", "utf-8"))
+                            output_file.write(join_file.read())
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/fuzz/src/CMakeLists.txt

@@ -16,7 +16,8 @@ foreach(test_source ${tests})
     get_filename_component(test_name ${test_source} NAME_WE)
     add_fuzzer(${test_name} ${test_source})
 
-    target_link_libraries(${test_name} PRIVATE IE::inference_engine cnpy zlib)
+    target_link_libraries(${test_name} PRIVATE IE::inference_engine cnpy zlib ${NGRAPH_LIBRARIES}
+            ngraph::frontend_manager)
 
     add_dependencies(fuzz ${test_name})
 

tests/fuzz/src/import_pdpd-fuzzer.cc

@@ -0,0 +1,45 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+#include "frontend_manager/frontend_manager.hpp"
+#include "ngraph/ngraph.hpp"
+#include "tokenizer.h"
+#include <string>
+
+#define COUNT_OF(A) (sizeof(A) / sizeof(A[0]))
+const char split_sequence[] = {'F', 'U', 'Z', 'Z', '_', 'N', 'E', 'X',
+                               'T', '_', 'F', 'I', 'E', 'L', 'D'};
+const char *PDPD = "pdpd";
+
+using namespace ngraph;
+using namespace ngraph::frontend;
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  /// split input buffer to model and params
+  Tokenizer tokenizer(data, size, split_sequence, sizeof(split_sequence));
+  size_t model_size = 0;
+  const void *model_buf = tokenizer.next(&model_size);
+  size_t params_size = 0;
+  const void *params_buf = tokenizer.next(&params_size);
+
+  try {
+    ngraph::frontend::FrontEndManager frontend_manager = FrontEndManager();
+    ngraph::frontend::FrontEnd::Ptr frontend =
+        frontend_manager.load_by_framework(PDPD);
+    ngraph::frontend::InputModel::Ptr input_model;
+    std::stringstream model;
+    model << std::string((const char *)model_buf, model_size);
+    std::shared_ptr<std::istream> in_model(&model);
+    if (params_buf) {
+      std::stringstream params;
+      params << std::string((const char *)params_buf, params_size);
+      std::shared_ptr<std::istream> in_params(&params);
+      input_model = frontend->load(in_model, in_params);
+    } else
+      input_model = frontend->load(in_model);
+    std::shared_ptr<ngraph::Function> function = frontend->convert(input_model);
+  } catch (const std::exception&) {
+    return 0;  // fail gracefully on expected exceptions
+  }
+  return 0;
+}