refactor the virtdata docs facility

nb/gendocs.sh

@@ -37,6 +37,7 @@ then
 fi
 
 $JAVA -jar target/nb.jar docserver generate ${GUIDEBOOK}/
+$JAVA -jar target/nb.jar virtdata gendocs basedir ${GUIDEBOOK}/services/docs/markdown/binding_functions
 
 #JAVA_HOME=${JAVA_HOME:-JAVA_HOME must be specified if java isn not in the path}
 #

virtdata-annotations/src/main/java/io/nosqlbench/virtdata/processors/ExampleDocData.java

@@ -6,6 +6,18 @@ import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 
+/**
+ * Example class doc.
+ *
+ * Example unordered list:
+ * <UL>
+ *     <LI>An item1</LI>
+ *     <LI>An item2</LI>
+ * </UL>
+ *
+ * An example class doc paragraph.
+ *
+ */
 public class ExampleDocData implements DocFuncData {
 
     @Override
@@ -46,7 +58,7 @@ public class ExampleDocData implements DocFuncData {
         LinkedHashMap<String, String> args = new LinkedHashMap<>();
         args.put("arg1", "val1");
         List<List<String>> examples = new ArrayList<>();
-        examples.add(new ArrayList<String>() {{ add("example"); add("one"); }});
+        examples.add(new ArrayList<>() {{ add("example"); add("one"); }});
         DocForFuncCtor ctordoc = new DocForFuncCtor("name", "ctordoc", args, examples);
         ctors.add(ctordoc);
 
@@ -54,24 +66,24 @@ public class ExampleDocData implements DocFuncData {
     }
 
     public List<DocForFuncCtor> getCtorsAlternate() {
-        return new ArrayList<DocForFuncCtor>() {{
+        return new ArrayList<>() {{
             add(new DocForFuncCtor("name", "ctordoc",
-                    new LinkedHashMap<String, String>() {{
+                    new LinkedHashMap<>() {{
                         put("aname", "atype");
                     }},
-                    new ArrayList<List<String>>() {{
-                        add(new ArrayList<String>() {{
+                    new ArrayList<>() {{
+                        add(new ArrayList<>() {{
                             add("example");
                             add("description");
                         }});
                     }}
             ));
             add(new DocForFuncCtor("name", "ctordoc",
-                    new LinkedHashMap<String, String>() {{
+                    new LinkedHashMap<>() {{
                         put("aname", "atype");
                     }},
-                    new ArrayList<List<String>>() {{
-                        add(new ArrayList<String>() {{
+                    new ArrayList<>() {{
+                        add(new ArrayList<>() {{
                             add("example");
                         }});
                     }}

virtdata-api/src/main/java/io/nosqlbench/virtdata/api/VirtDataResources.java

@@ -35,7 +35,9 @@ import java.util.stream.Collectors;
 
 public class VirtDataResources {
     public final static String DATA_DIR = "data";
-    private final static Logger logger  = LogManager.getLogger(VirtDataResources.class);public static CharBuffer readDataFileToCharBuffer(String basename) {
+    private final static Logger logger = LogManager.getLogger(VirtDataResources.class);
+
+    public static CharBuffer readDataFileToCharBuffer(String basename) {
         return loadFileToCharBuffer(basename, DATA_DIR);
     }
 
@@ -50,23 +52,23 @@ public class VirtDataResources {
     public static InputStream findRequiredStreamOrFile(String basename, String extension, String... searchPaths) {
         Optional<InputStream> optionalStreamOrFile = findOptionalStreamOrFile(basename, extension, searchPaths);
         return optionalStreamOrFile.orElseThrow(() -> new RuntimeException(
-                "Unable to find " + basename + " with extension " + extension + " in file system or in classpath, with"
-                        + " search paths: " + Arrays.stream(searchPaths).collect(Collectors.joining(","))
+            "Unable to find " + basename + " with extension " + extension + " in file system or in classpath, with"
+                + " search paths: " + Arrays.stream(searchPaths).collect(Collectors.joining(","))
         ));
     }
 
     public static Reader findRequiredReader(String basename, String extension, String... searchPaths) {
         Optional<Reader> optionalReader = findOptionalReader(basename, extension, searchPaths);
         return optionalReader.orElseThrow(() -> new RuntimeException(
-                "Unable to find " + basename + " with extension " + extension + " in file system or in classpath, with"
-                        + " search paths: " + Arrays.stream(searchPaths).collect(Collectors.joining(","))
+            "Unable to find " + basename + " with extension " + extension + " in file system or in classpath, with"
+                + " search paths: " + Arrays.stream(searchPaths).collect(Collectors.joining(","))
         ));
     }
 
     public static Optional<Reader> findOptionalReader(String basename, String extenion, String... searchPaths) {
         return findOptionalStreamOrFile(basename, extenion, searchPaths)
-                .map(InputStreamReader::new)
-                .map(BufferedReader::new);
+            .map(InputStreamReader::new)
+            .map(BufferedReader::new);
     }
 
     public static Optional<Path> findOptionalDirPath(String pathName) {
@@ -92,7 +94,7 @@ public class VirtDataResources {
             add(filename);
             if (!isRemote(basename)) {
                 addAll(Arrays.stream(searchPaths).map(s -> s + File.separator + filename)
-                        .collect(Collectors.toCollection(ArrayList::new)));
+                    .collect(Collectors.toCollection(ArrayList::new)));
             }
         }};
 
@@ -108,7 +110,7 @@ public class VirtDataResources {
 
     private static boolean isRemote(String path) {
         return (path.toLowerCase().startsWith("http:")
-                || path.toLowerCase().startsWith("https:"));
+            || path.toLowerCase().startsWith("https:"));
     }
 
     public static Optional<InputStream> getInputStream(String path) {
@@ -224,9 +226,14 @@ public class VirtDataResources {
      * @throws RuntimeException if none of the specified paths is found in any of the locations
      */
     public static Path findPathIn(String... pathspecs) {
+        Optional<Path> found = FindOptionalPathIn(pathspecs);
+        return found.orElseThrow();
+    }
 
+    public static Optional<Path> FindOptionalPathIn(String... pathspecs) {
+
+        Path foundPath = null;
         for (String pathspec : pathspecs) {
-            Path foundPath = null;
 
             if (isRemote(pathspec)) {
                 try {
@@ -248,9 +255,9 @@ public class VirtDataResources {
                     boolean foundADirectory = attrs.isDirectory();
                     if (wantsADirectory != foundADirectory) {
                         throw new RuntimeException("for path " + pathspec + ", user wanted a " +
-                                (wantsADirectory ? "directory" : "file") + ", but found a " +
-                                (foundADirectory ? "directory" : "file") + " while searching paths " +
-                                Arrays.toString(pathspecs));
+                            (wantsADirectory ? "directory" : "file") + ", but found a " +
+                            (foundADirectory ? "directory" : "file") + " while searching paths " +
+                            Arrays.toString(pathspecs));
                     }
                     foundPath = candidate;
                 } catch (Exception ignored) {
@@ -269,14 +276,11 @@ public class VirtDataResources {
 
                 }
             }
-
-            if (foundPath != null) {
-                return foundPath;
-            }
         }
-        throw new RuntimeException("Unable to find path in " + Arrays.toString(pathspecs));
+        return Optional.ofNullable(foundPath);
     }
 
+
     private synchronized static Path getPathInFilesystem(URI uri) {
         FileSystem fileSystem = null;
         try {

virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/AutoDocsApp.java

@@ -2,66 +2,93 @@ package io.nosqlbench.virtdata.userlibs.apps.docsapp;
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
-import com.vladsch.flexmark.convert.html.FlexmarkHtmlParser;
 import io.nosqlbench.virtdata.annotations.Category;
 import io.nosqlbench.virtdata.api.VirtDataDocs;
-import io.nosqlbench.virtdata.processors.DocCtorData;
+import io.nosqlbench.virtdata.api.VirtDataResources;
 import io.nosqlbench.virtdata.processors.DocFuncData;
+import io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs.FDoc;
+import io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs.FDocCat;
+import io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs.FDocFunc;
+import io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs.FDocFuncs;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 
 import java.io.*;
-import java.security.InvalidParameterException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.*;
-import java.util.stream.Collectors;
 
-public class AutoDocsApp {
-    private final static Logger logger  = LogManager.getLogger(AutoDocsApp.class);private final static String SPLIT = "split";
-    private final static String COMBINED = "combined";
-    private final static String ALL = "all";
-    private final static String DEFAULT_FILE = "funcref";
-    private static final String MARKDOWN = "markdown";
-    private static final String JSON = "json";
+public class AutoDocsApp implements Runnable {
 
-    // category -> funcname -> [docfuncdata, ...]
-    private Map<Category, Map<String, List<DocFuncData>>> groupedModels = new HashMap<>();
+    private final static Logger logger = LogManager.getLogger(AutoDocsApp.class);
 
-    private String baseFileName = DEFAULT_FILE;
-    private String print = ALL;
-    private String categories = SPLIT;
-    private String format = MARKDOWN;
+    private final static String CATEGORIES = "categories";
+    private final static String CATEGORIES_SPLIT = "split";
+    private final static String CATEGORIES_COMBINED = "combined";
+
+    private final static String FORMAT = "format";
+    private static final String FORMAT_MARKDOWN = "markdown";
+    private static final String FORMAT_JSON = "json";
+
+    private final static String BLURBS_DIRS = "blurbsdirs";
+
+    private final static String BASE_FILENAME = "funcref";
+
+    private final String[] args;
+    private final Map<String, Writer> writers = new HashMap<>();
+
+    private String baseFileName = BASE_FILENAME;
+    private String categories = CATEGORIES_SPLIT;
+    private String format = FORMAT_MARKDOWN;
+
+    private String blurbsDirs = "docs/category_blurbs:src/main/resources/docs/category_blurbs:virtdata-userlibs/src/main/resources/docs/category_blurbs";
+    private String basedir = "";
 
     public static void main(String[] args) {
-        new AutoDocsApp().invoke(args);
+        new AutoDocsApp(args).run();
     }
 
-    private void invoke(String[] args) {
+    public AutoDocsApp(String[] args) {
+        this.args = args;
+    }
+
+    public void run() {
         LinkedList<String> largs = new LinkedList<>(Arrays.asList(args));
+        if (args.length>0 && args[0].contains("help")) {
+            System.out.println(
+                "usage:\n" +
+                    "[basefile <name>] [basedir <dir>] [categories combined|split] [format json|markdown] " +
+                    "[blurbsdirs <dir>[:...]]\n\n"
+            );
+            System.exit(0);
+        }
         while (largs.peekFirst() != null) {
             String argtype = largs.removeFirst();
             if (largs.peekFirst() == null) {
                 throw new RuntimeException(AutoDocsApp.class.toString() + " expects args in param value couplets.");
             }
+
             String argval = largs.removeFirst().toLowerCase();
             switch (argtype) {
-                case "output":
+                case "basefile":
                     this.baseFileName = argval;
                     break;
-                case "print":
-                    if (argval.equals("all") || argval.equals("logs")) {
-                        this.print = argval;
-                    } else {
-                        throw new InvalidParameterException("valid args for print: print all, print logs");
-                    }
-                case "categories":
-                    if (!argval.equals(SPLIT) && !argval.equals(COMBINED)) {
-                        throw new RuntimeException("categories must either be " + SPLIT + ", or " + COMBINED + ".");
+                case "basedir":
+                    this.basedir = argval;
+                    break;
+                case BLURBS_DIRS:
+                    this.blurbsDirs = argval;
+                    break;
+                case CATEGORIES:
+                    if (!argval.equals(CATEGORIES_SPLIT) && !argval.equals(CATEGORIES_COMBINED)) {
+                        throw new RuntimeException("categories must either be " + CATEGORIES_SPLIT + ", or " + CATEGORIES_COMBINED + ".");
                     }
                     this.categories = argval;
                     break;
-                case "format":
-                    if (!argval.equals(MARKDOWN) && !argval.equals(JSON)) {
-                        throw new RuntimeException("format must either be " + MARKDOWN + ", or " + JSON + ".");
+                case FORMAT:
+                    if (!argval.equals(FORMAT_MARKDOWN) && !argval.equals(FORMAT_JSON)) {
+                        throw new RuntimeException("format must either be " + FORMAT_MARKDOWN + ", or " + FORMAT_JSON + ".");
                     }
                     this.format = argval;
                     break;
@@ -69,226 +96,82 @@ public class AutoDocsApp {
             }
         }
 
-        List<DocFuncData> docModels = VirtDataDocs.getAllDocs();
-        for (DocFuncData docModel : docModels) {
-            for (Category category : docModel.getCategories()) {
-                Map<String, List<DocFuncData>> category_funcname_list = this.groupedModels.get(category);
-                if (category_funcname_list == null) {
-                    category_funcname_list = new HashMap<>();
-                    this.groupedModels.put(category, category_funcname_list);
-                }
-                List<DocFuncData> group = category_funcname_list.getOrDefault(docModel.getClassName(), new ArrayList<>());
-                group.add(docModel);
-                category_funcname_list.put(docModel.getClassName(), group);
-            }
-        }
+        FDoc docsinfo = loadAllDocs();
 
-
-//        StringBuilder sb = new StringBuilder();
-//        Map<Category,String> docsByCategory=new HashMap<>();
-
-        Map<String, Set<Category>> assignments = new HashMap<>();
-        // Map single category annotation to global Name -> Category assignment
-        for (DocFuncData docModel : docModels) {
-
-            Set<Category> listForFuncName = assignments.getOrDefault(docModel.getClassName(), new HashSet<>());
-            assignments.put(docModel.getClassName(), listForFuncName);
-            if (listForFuncName.size() > 0) {
-                logger.warn("Func name " + docModel.getClassName() + " has " + listForFuncName.size() + " multiple category annotations:");
-            }
-            listForFuncName.addAll(Arrays.asList(docModel.getCategories()));
-            logger.info("Assigning " + docModel.getClassName() + " to categories " + listForFuncName.toString());
-        }
-
-        Set<Category> generalSet = new HashSet<>() {{
-            add(Category.general);
-        }};
-        // regroup docs under categories
-        // category -> funcname -> [docfuncdata, ...]
-        Map<Category, Map<String, List<DocFuncData>>> regrouped = new HashMap<>();
-        for (DocFuncData docModel : docModels) {
-            Set<Category> assignment = assignments.getOrDefault(docModel.getClassName(), generalSet);
-            if (assignment.size() == 0) {
-                assignment = generalSet;
-            }
-            logger.info("looking up assignment for " + docModel.getClassName() + ":" + assignment.toString());
-            for (Category category : assignment) {
-                Map<String, List<DocFuncData>> assignToCategory = regrouped.getOrDefault(category, new HashMap<>());
-                regrouped.put(category, assignToCategory);
-
-                List<DocFuncData> assignToClass = assignToCategory.getOrDefault(docModel.getClassName(), new ArrayList<>());
-                assignToCategory.put(docModel.getClassName(), assignToClass);
-
-                assignToClass.add(docModel);
-            }
-        }
-        groupedModels = regrouped;
-
-
-        Map<String, Writer> writers = new HashMap<>();
-        Writer writer = new OutputStreamWriter(System.out);
         try {
+            String extension = (this.format.equals(FORMAT_MARKDOWN)) ? ".md" : ".json";
 
-            String extension = (this.format.equals(MARKDOWN)) ? ".md" : ".json";
+            for (FDocCat docsForCatName : docsinfo) {
+                String categoryName = docsForCatName.getCategoryName();
+                categoryName = categoryName.isEmpty() ? "EMPTY" : categoryName;
 
-            for (Category category : Category.values()) {
-                if (groupedModels.keySet().contains(category)) {
+                String filename = this.baseFileName
+                    + (this.categories.equals(CATEGORIES_SPLIT) ? "_" + categoryName : "")
+                    + extension;
 
-                    if (!this.baseFileName.isEmpty() && this.categories.equals(SPLIT)) {
-                        writer = writers.getOrDefault(category.toString(), new FileWriter(baseFileName + "_" + category.toString() + extension));
-                    } else if (!this.baseFileName.isEmpty() && this.categories.equals(COMBINED)) {
-                        writer = writers.getOrDefault(baseFileName + extension, new FileWriter(baseFileName));
+                Writer writer = getWriterFor(filename);
+
+                for (FDocFuncs docsForFuncName : docsForCatName) {
+                    if (format.equals(FORMAT_JSON)) {
+                        Gson gson = new GsonBuilder().setPrettyPrinting().create();
+                        writer.append(gson.toJson(docsForFuncName));
+                    } else if (format.equals(FORMAT_MARKDOWN)) {
+                        writer.append(docsForFuncName.asMarkdown());
                     }
-
-                    String docs = writeCategoryDocs(category, groupedModels.get(category));
-                    //docs = replacePatterns(docs);
-                    writer.write(docs);
-                    writer.flush();
                 }
             }
-            for (Writer writer1 : writers.values()) {
-                writer1.close();
+            for (Writer writer : writers.values()) {
+                writer.flush();
+                writer.close();
             }
-
-        } catch (IOException e) {
+        } catch (Exception e) {
             throw new RuntimeException(e);
         }
     }
 
-    private String writeCategoryDocs(Category category, Map<String, List<DocFuncData>> groupedDocs) {
-
-        StringBuilder sb = new StringBuilder();
-        ArrayList<String> funcNames = new ArrayList<>(groupedDocs.keySet());
-        Collections.sort(funcNames);
-
-        List<FunctionDoc> fdocs = new ArrayList<>(groupedDocs.size());
-
-        for (String name : funcNames) {
-            FunctionDoc fdoc = new FunctionDoc(name);
-
-            List<DocFuncData> docs = groupedDocs.get(name);
-
-            List<DocFuncData> classdocs = docs.stream()
-                    .filter(d -> d.getClassJavadoc() != null && !d.getClassJavadoc().isEmpty())
-                    .collect(Collectors.toList());
-
-            List<String> distinctClassDocs = classdocs.stream()
-                    .map(DocFuncData::getClassJavadoc)
-                    .map(String::trim)
-                    .distinct()
-                    .collect(Collectors.toList());
-
-
-            if (distinctClassDocs.size() == 0) {
-                logger.warn("There were no class docs found for types named " + name);
-            }
-            if (distinctClassDocs.size() > 1) {
-                logger.warn("There were multiple class docs found for types named " + name);
-            }
-
-            if (distinctClassDocs.size() == 1) {
-                String classdoc = distinctClassDocs.get(0);
-                classdoc = parseHtmlToMarkdown(classdoc);
-                sb.append(classdoc);
-                if (!classdoc.endsWith("\n")) {
-                    classdoc=classdoc + "\n";
+    private Writer getWriterFor(String outputname) {
+        FileWriter fileWriter = null;
+        if (!writers.containsKey(outputname)) {
+            try {
+                outputname = basedir.isEmpty() ? outputname : basedir + "/" + outputname;
+                Path parent = Path.of(outputname).getParent();
+                if (parent!=null) {
+                    Files.createDirectories(parent);
                 }
-                if (!classdoc.endsWith("\n\n")) {
-                    classdoc=classdoc + "\n";
-                }
-                fdoc.setClassDocs(classdoc);
-            }
+                fileWriter = new FileWriter(outputname, false);
+                writers.put(outputname, fileWriter);
 
-            for (DocFuncData doc : docs) {
-                fdoc.addCategories(doc.getCategories());
+                String[] blurbsdirs = blurbsDirs.split(":");
+                for (String blurbsdir : blurbsdirs) {
+                    Optional<Path> bdir = VirtDataResources.FindOptionalPathIn(blurbsdir+"/");
+                    if (bdir.isPresent()) {
+                        Path blurbsFile = bdir.get().resolve(Path.of(outputname).getFileName().toString());
+                        if (Files.exists(blurbsFile)) {
+                            String blurb = Files.readString(blurbsFile, StandardCharsets.UTF_8);
 
-                for (DocCtorData ctor : doc.getCtors()) {
-                    fdoc.addCtor(ctor);
-                }
-            }
-            fdocs.add(fdoc);
-        }
-
-        if (format.equals(MARKDOWN)) {
-
-            sb.append("# CATEGORY ").append(category).append("\n");
-
-            for (String name : funcNames) {
-
-                List<DocFuncData> docs = groupedDocs.get(name);
-
-                sb.append("## ").append(name).append("\n\n");
-
-                List<DocFuncData> classdocs = docs.stream()
-                        .filter(d -> d.getClassJavadoc() != null && !d.getClassJavadoc().isEmpty())
-                        .collect(Collectors.toList());
-
-                List<String> distinctClassDocs = classdocs.stream()
-                        .map(DocFuncData::getClassJavadoc)
-                        .map(String::trim)
-                        .distinct()
-                        .collect(Collectors.toList());
-
-                if (distinctClassDocs.size() == 0) {
-                    logger.warn("There were no class docs found for types named " + name);
-                }
-                if (distinctClassDocs.size() > 1) {
-                    logger.warn("There were multiple class docs found for types named " + name);
-                }
-
-                if (distinctClassDocs.size() == 1) {
-                    String classdoc = distinctClassDocs.get(0);
-                    classdoc = parseHtmlToMarkdown(classdoc);
-                    sb.append(classdoc);
-                    if (!classdoc.endsWith("\n\n")) {
-                        sb.append("\n");
-                    }
-                    if (!classdoc.endsWith("\n")) {
-                        sb.append("\n");
-                    }
-                }
-
-                for (DocFuncData doc : docs) {
-                    List<DocCtorData> ctors = doc.getCtors();
-                    for (DocCtorData ctor : ctors) {
-                        sb.append("- ").append(doc.getInType()).append(" -> ");
-                        sb.append(doc.getClassName());
-                        sb.append("(");
-                        sb.append(
-                                ctor.getArgs().entrySet().stream().map(
-                                        e -> e.getValue() + ": " + e.getKey()
-                                ).collect(Collectors.joining(", "))
-                        );
-                        sb.append(")");
-                        sb.append(" -> ").append(doc.getOutType()).append("\n");
-
-                        String ctorDoc = ctor.getCtorJavaDoc();
-                        if (!ctorDoc.isEmpty()) {
-                            sb.append("  - *notes:* ").append(ctorDoc);
-                        }
-                        for (List<String> example : ctor.getExamples()) {
-                            sb.append("  - *ex:* `").append(example.get(0)).append("`");
-                            if (example.size() > 1) {
-                                sb.append(" - *").append(example.get(1)).append("*");
-                            }
-                            sb.append("\n");
+                            logger.debug("writing blurb to " + outputname);
+                            fileWriter.append(blurb);
                         }
                     }
                 }
-                sb.append("\n");
-                sb.append("\n");
+            } catch (IOException e) {
+                throw new RuntimeException(e);
             }
-        } else if (format.equals(JSON)) {
-            Gson gson = new GsonBuilder().setPrettyPrinting().create();
-            sb.append(gson.toJson(fdocs));
         }
-        return sb.toString();
+        return writers.get(outputname);
     }
 
-    private String parseHtmlToMarkdown(String classdoc) {
-        String markdown = FlexmarkHtmlParser.parse(classdoc);
-        return markdown;
+    private FDoc loadAllDocs() {
+        FDoc docsinfo = new FDoc();
+        List<DocFuncData> allDocs = VirtDataDocs.getAllDocs();
+        for (DocFuncData docFuncData : allDocs) {
+            FDocFunc FDocFunc = new FDocFunc(docFuncData);
+            for (Category categoryName : FDocFunc.getCategories()) {
+                FDocCat fDocCat = docsinfo.addCategory(categoryName.toString());
+                fDocCat.addFunctionDoc(FDocFunc);
+            }
+        }
+        return docsinfo;
     }
 
-
 }

virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/fdocs/FDoc.java

@@ -0,0 +1,22 @@
+package io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+public class FDoc implements Iterable<FDocCat> {
+
+    private final Map<String, FDocCat> docs = new HashMap<>();
+
+    public FDoc() {
+    }
+
+    public FDocCat addCategory(String categoryName) {
+        return docs.computeIfAbsent(categoryName, FDocCat::new);
+    }
+
+    @Override
+    public Iterator<FDocCat> iterator() {
+        return docs.values().iterator();
+    }
+}

virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/fdocs/FDocCat.java

@@ -0,0 +1,35 @@
+package io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs;
+
+import java.util.*;
+
+public class FDocCat implements Iterable<FDocFuncs> {
+    private final Map<String, FDocFuncs> docsByFuncName= new HashMap<>();
+    private final String categoryName;
+
+    public FDocCat(String categoryName) {
+        this.categoryName = categoryName;
+    }
+
+    public String getCategoryName() {
+        return categoryName;
+    }
+
+    public void addFunctionDoc(FDocFunc FDocFunc) {
+        String name = FDocFunc.getPackageName() + "." + FDocFunc.getClassName();
+        FDocFuncs fDocFuncs = docsByFuncName.computeIfAbsent(FDocFunc.getClassName(),
+            FDocFuncs::new);
+        fDocFuncs.addFunctionDoc(FDocFunc);
+    }
+
+    public List<FDocFuncs> getFunctionDocsList() {
+        return new ArrayList<>(docsByFuncName.values());
+
+    }
+
+    @Override
+    public Iterator<FDocFuncs> iterator() {
+        ArrayList<FDocFuncs> fdocs = new ArrayList<>(docsByFuncName.values());
+        fdocs.sort(Comparator.comparing(FDocFuncs::getFunctionName));
+        return docsByFuncName.values().iterator();
+    }
+}

virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/fdocs/FDocCtor.java

@@ -0,0 +1,48 @@
+package io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs;
+
+import io.nosqlbench.virtdata.processors.DocCtorData;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class FDocCtor {
+
+    private final List<List<String>> examples;
+    private final String ctorJavaDoc;
+    private final Map<String, String> args;
+    private final String in;
+    private final String out;
+    private final String className;
+
+    public FDocCtor(DocCtorData docCtorData, String in, String out) {
+        this.examples =docCtorData.getExamples();
+        this.ctorJavaDoc = docCtorData.getCtorJavaDoc();
+        this.args = docCtorData.getArgs();
+        this.className=docCtorData.getClassName();
+        this.in = in;
+        this.out = out;
+    }
+
+    public String asMarkdown() {
+        StringBuilder sb = new StringBuilder();
+        // - in->Name(arg1: type1, ...) ->out
+        sb.append("- ").append(in).append(" -> ");
+        sb.append(className).append("(");
+        String args = this.args.entrySet().stream().map(e -> e.getValue() + ": " + e.getKey()).collect(Collectors.joining(", "));
+        sb.append(args);
+        sb.append(") -> ").append(out).append("\n");
+
+        if (!ctorJavaDoc.isEmpty()) {
+            sb.append("  - *notes:* ").append(ctorJavaDoc).append("\n");
+        }
+        for (List<String> example : examples) {
+            sb.append("  - *ex:* `").append(example.get(0)).append("`\n");
+            if (example.size()>1) {
+                sb.append("  - *").append(example.get(1)).append("*\n");
+            }
+        }
+        sb.append("\n");
+        return sb.toString();
+    }
+}

virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/fdocs/FDocFunc.java

@@ -0,0 +1,77 @@
+package io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs;
+
+import io.nosqlbench.virtdata.annotations.Category;
+import io.nosqlbench.virtdata.processors.DocCtorData;
+import io.nosqlbench.virtdata.processors.DocFuncData;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class FDocFunc implements Comparable<FDocFunc> {
+
+    private final String funcName;
+    private final Set<Category> categories;
+    private String className;
+    private String classJavaDoc;
+    private String packageName;
+    private List<FDocCtor> ctors;
+    private String inType;
+    private String outType;
+
+    public FDocFunc(DocFuncData docFuncData) {
+        this.funcName = docFuncData.getClassName();
+        this.categories = new HashSet<>(Arrays.asList(docFuncData.getCategories()));
+        this.className = docFuncData.getClassName();
+        this.classJavaDoc= docFuncData.getClassJavadoc();
+        this.packageName=docFuncData.getPackageName();
+        this.inType=docFuncData.getInType();
+        this.outType=docFuncData.getOutType();
+        this.ctors=docFuncData.getCtors().stream().map(f -> new FDocCtor(f,inType,outType)).collect(Collectors.toList());
+    }
+
+    public String getClassName() {
+        return className;
+    }
+
+    public String getClassJavaDoc() {
+        return classJavaDoc;
+    }
+
+    public String getPackageName() {
+        return packageName;
+    }
+
+    public String getInType() {
+        return inType;
+    }
+
+    public String getOutType() {
+        return outType;
+    }
+
+    public String getFuncName() {
+        return funcName;
+    }
+
+    public Set<Category> getCategories() {
+        return categories;
+    }
+
+    @Override
+    public int compareTo(FDocFunc o) {
+        int result = this.className.compareTo(o.className);
+        if (result!=0) return result;
+        result = this.getPackageName().compareTo(o.getPackageName());
+        return result;
+    }
+
+    public List<FDocCtor> getCtors() {
+        return ctors;
+    }
+
+    public CharSequence asMarkdown() {
+        StringBuilder sb = new StringBuilder();
+
+        return sb.toString();
+    }
+}

virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/fdocs/FDocFuncs.java

@@ -0,0 +1,71 @@
+package io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs;
+
+import com.vladsch.flexmark.convert.html.FlexmarkHtmlParser;
+import io.nosqlbench.virtdata.processors.DocCtorData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * holds all FunctionDoc instances for the same basic function name
+ */
+public class FDocFuncs implements Iterable<FDocFunc> {
+    private final static Logger logger  = LogManager.getLogger(FDocFuncs.class);
+
+    private final Map<String, FDocFunc> functionsByPackage = new HashMap<>();
+    private final String functionName;
+
+    public FDocFuncs(String functionName) {
+        this.functionName=functionName;
+    }
+    public String getFunctionName() {
+        return this.functionName;
+    }
+
+    public void addFunctionDoc(FDocFunc FDocFunc) {
+        String name = FDocFunc.getPackageName() + "." + FDocFunc.getClassName();
+        if (functionsByPackage.containsKey(name)) {
+            throw new RuntimeException("Name '" + name + " is already present.");
+        }
+        functionsByPackage.put(name, FDocFunc);
+    }
+
+    @Override
+    public Iterator<FDocFunc> iterator() {
+        List<FDocFunc> fdocs = new ArrayList<>(functionsByPackage.values());
+        Collections.sort(fdocs);
+        return fdocs.iterator();
+    }
+
+    public String getCombinedClassDocs() {
+        List<String> cdocs = functionsByPackage.values().stream()
+            .sorted()
+            .map(f -> f.getClassJavaDoc().trim())
+            .filter(s -> s.length() > 0)
+            .collect(Collectors.toList());
+
+        if (cdocs.size()!=1) {
+            logger.warn("There were " + cdocs.size() + " class docs found for types named " + getFunctionName());
+        }
+
+        return String.join("\n\n",cdocs);
+    }
+
+    public String asMarkdown() {
+        StringBuilder sb = new StringBuilder();
+
+        sb.append("## ").append(getFunctionName()).append("\n\n");
+
+        String classDocMarkdown = FlexmarkHtmlParser.parse(getCombinedClassDocs());
+        sb.append(classDocMarkdown).append("\n");
+
+        for (FDocFunc fdf : functionsByPackage.values()) {
+            for (FDocCtor ctor : fdf.getCtors()) {
+                sb.append(ctor.asMarkdown());
+            }
+        }
+        return sb.toString();
+    }
+}

virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/fdocs/package-info.java

@@ -0,0 +1,6 @@
+/**
+ * FIles in this package are useful for sharing a canonical data structure
+ * for function documetnation purposes. This is the stable API that should
+ * be used for any downstream docs consumers.
+ */
+package io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs;

virtdata-userlibs/src/test/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/fdocs/ExampleDocFunc1.java

@@ -0,0 +1,98 @@
+package io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs;
+
+import io.nosqlbench.virtdata.annotations.Category;
+import io.nosqlbench.virtdata.processors.DocCtorData;
+import io.nosqlbench.virtdata.processors.DocForFuncCtor;
+import io.nosqlbench.virtdata.processors.DocFuncData;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+/**
+ * Example class doc part 1.
+ *
+ * Example unordered list part 1:
+ * <UL>
+ *     <LI>An item1 part 1</LI>
+ *     <LI>An item2 part 1</LI>
+ * </UL>
+ *
+ * An example class doc paragraph.
+ *
+ */
+public class ExampleDocFunc1 implements DocFuncData {
+
+    @Override
+    public String getPackageName() {
+        return "package.name.one";
+    }
+
+    @Override
+    public Category[] getCategories() {
+        return new Category[] { Category.general };
+    }
+
+    @Override
+    public String getClassName() {
+        return "ClassName1";
+    }
+
+    @Override
+    public String getClassJavadoc() {
+        return "javadoc";
+    }
+
+    @Override
+    public String getInType() {
+        return "intype";
+    }
+
+    @Override
+    public String getOutType() {
+        return "outtype";
+    }
+
+    @Override
+    public List<DocCtorData> getCtors() {
+        ArrayList<DocCtorData> ctors = new ArrayList<>();
+
+        // for each ctor
+        LinkedHashMap<String, String> args = new LinkedHashMap<>();
+        args.put("arg1", "val1");
+        args.put("arg2", "val2");
+        List<List<String>> examples = new ArrayList<>();
+        examples.add(new ArrayList<>() {{ add("example"); add("one"); }});
+        DocForFuncCtor ctordoc = new DocForFuncCtor("ClassName1", "ctordoc", args, examples);
+        ctors.add(ctordoc);
+
+        return ctors;
+    }
+
+    public List<DocForFuncCtor> getCtorsAlternate() {
+        return new ArrayList<>() {{
+            add(new DocForFuncCtor("ClassName1", "ctordoc",
+                    new LinkedHashMap<>() {{
+                        put("aname", "atype");
+                    }},
+                    new ArrayList<>() {{
+                        add(new ArrayList<>() {{
+                            add("example");
+                            add("description");
+                        }});
+                    }}
+            ));
+            add(new DocForFuncCtor("ClassName1", "ctordoc",
+                    new LinkedHashMap<>() {{
+                        put("aname", "atype");
+                    }},
+                    new ArrayList<>() {{
+                        add(new ArrayList<>() {{
+                            add("example");
+                        }});
+                    }}
+            ));
+
+        }};
+    }
+}

virtdata-userlibs/src/test/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/fdocs/ExampleDocFunc2.java

@@ -0,0 +1,97 @@
+package io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs;
+
+import io.nosqlbench.virtdata.annotations.Category;
+import io.nosqlbench.virtdata.processors.DocCtorData;
+import io.nosqlbench.virtdata.processors.DocForFuncCtor;
+import io.nosqlbench.virtdata.processors.DocFuncData;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+/**
+ * Example class doc part 2.
+ *
+ * Example unordered list part 2:
+ * <UL>
+ *     <LI>An item1 part 2</LI>
+ *     <LI>An item2 part 2</LI>
+ * </UL>
+ *
+ * An example class doc paragraph.
+ *
+ */
+public class ExampleDocFunc2 implements DocFuncData {
+
+    @Override
+    public String getPackageName() {
+        return "package.name.two";
+    }
+
+    @Override
+    public Category[] getCategories() {
+        return new Category[] { Category.general };
+    }
+
+    @Override
+    public String getClassName() {
+        return "ClassName2";
+    }
+
+    @Override
+    public String getClassJavadoc() {
+        return "javadoc";
+    }
+
+    @Override
+    public String getInType() {
+        return "intype";
+    }
+
+    @Override
+    public String getOutType() {
+        return "outtype";
+    }
+
+    @Override
+    public List<DocCtorData> getCtors() {
+        ArrayList<DocCtorData> ctors = new ArrayList<>();
+
+        // for each ctor
+        LinkedHashMap<String, String> args = new LinkedHashMap<>();
+        args.put("arg1", "val1");
+        List<List<String>> examples = new ArrayList<>();
+        examples.add(new ArrayList<>() {{ add("example"); add("one"); }});
+        DocForFuncCtor ctordoc = new DocForFuncCtor("ClassName2", "ctordoc", args, examples);
+        ctors.add(ctordoc);
+
+        return ctors;
+    }
+
+    public List<DocForFuncCtor> getCtorsAlternate() {
+        return new ArrayList<>() {{
+            add(new DocForFuncCtor("ClassName2", "ctordoc",
+                    new LinkedHashMap<>() {{
+                        put("aname", "atype");
+                    }},
+                    new ArrayList<>() {{
+                        add(new ArrayList<>() {{
+                            add("example");
+                            add("description");
+                        }});
+                    }}
+            ));
+            add(new DocForFuncCtor("ClassName2", "ctordoc",
+                    new LinkedHashMap<>() {{
+                        put("aname", "atype");
+                    }},
+                    new ArrayList<>() {{
+                        add(new ArrayList<>() {{
+                            add("example");
+                        }});
+                    }}
+            ));
+
+        }};
+    }
+}