temlpate parameters and unit test fixes

engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlStatementLoader.java

@@ -27,6 +27,7 @@ import org.yaml.snakeyaml.Yaml;
 import java.io.BufferedReader;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;

engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/StmtsDocList.java

@@ -92,4 +92,5 @@ public class StmtsDocList implements Iterable<StmtsDoc> {
     public Scenarios getDocScenarios() {
         return this.getStmtDocs().get(0).getScenarios();
     }
+
 }

engine-api/src/main/java/io/nosqlbench/engine/api/util/NosqlBenchFiles.java

@@ -26,19 +26,20 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
-import java.net.URI;
-import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 public class NosqlBenchFiles {
 
     private final static Logger logger = LoggerFactory.getLogger(NosqlBenchFiles.class);
+    private static Pattern templatePattern = Pattern.compile("TEMPLATE\\((.+)\\)");
+    private static Pattern templatePattern2 = Pattern.compile("<<(.+)>>");
+
 
     public static InputStream findRequiredStreamOrFile(String basename, String extension, String... searchPaths) {
         Optional<InputStream> optionalStreamOrFile = findOptionalStreamOrFile(basename, extension, searchPaths);
@@ -145,28 +146,76 @@ public class NosqlBenchFiles {
         }
     }
 
-    public static Map<String, List<String>> getWorkloadsWithScenarioScripts() {
+    public static List<WorkloadDesc> getWorkloadsWithScenarioScripts() {
 
-        String dir = "activities/";
+        String dir = "./";
 
         Path basePath = VirtDataResources.findPathIn(dir);
-        List<Path> yamlPathList = PathWalker.findAll(basePath).stream().filter(f -> f.toString().endsWith(".yaml")).collect(Collectors.toList());
+        List<Path> yamlPathList = PathWalker.findAll(basePath)
+            .stream()
+            .filter(f -> f.toString().endsWith(".yaml"))
+            .filter(f -> f.toString().contains("activities"))
+            .collect(Collectors.toList());
 
-        HashMap workloadMap = new HashMap();
+        List<WorkloadDesc> workloadDescriptions = new ArrayList<>();
         for (Path yamlPath : yamlPathList) {
-            String substring = yamlPath.toString().substring(1);
+            String substring = yamlPath.toString().substring(2);
             StmtsDocList stmts = StatementsLoader.load(logger, substring);
 
+            Set<String> templates = new HashSet<>();
+            try {
+                List<String> lines = Files.readAllLines(yamlPath);
+                for (String line : lines) {
+                    Matcher matcher = templatePattern.matcher(line);
+
+                    while (matcher.find()) {
+                        templates.add(matcher.group(1));
+                    }
+                    matcher = templatePattern2.matcher(line);
+
+                    while (matcher.find()) {
+                        templates.add(matcher.group(1));
+                    }
+
+                }
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+
+
             Scenarios scenarios = stmts.getDocScenarios();
 
             List<String> scenarioNames = scenarios.getScenarioNames();
 
             if (scenarioNames != null && scenarioNames.size() >0){
-                workloadMap.put(yamlPath.getFileName().toString(), scenarioNames);
+                workloadDescriptions.add(new WorkloadDesc(yamlPath.getFileName().toString(), scenarioNames, templates));
             }
         }
 
-        return workloadMap;
+        return workloadDescriptions;
     }
 
+    public static class WorkloadDesc {
+        private final String yamlPath;
+        private final List<String> scenarioNames;
+        private final Set<String> temlpates;
+
+        public WorkloadDesc(String yamlPath, List<String> scenarioNames, Set<String> templates) {
+            this.yamlPath = yamlPath;
+            this.scenarioNames = scenarioNames;
+            this.temlpates = templates;
+        }
+
+        public String getYamlPath() {
+            return yamlPath;
+        }
+
+        public List<String> getScenarioNames() {
+            return scenarioNames;
+        }
+
+        public Set<String> getTemlpates() {
+            return temlpates;
+        }
+    }
 }

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java

@@ -24,10 +24,7 @@ import org.slf4j.LoggerFactory;
 import java.io.BufferedReader;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.util.Arrays;
+import java.util.*;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
 import java.util.stream.Collectors;
 
 public class NBCLI {
@@ -85,11 +82,12 @@ public class NBCLI {
 
         if (options.wantsWorkloads()) {
             //ActivityType.FINDER.getAll().stream().map(ActivityType::getName).forEach(System.out::println);
-            Map<String, List<String>> workloads = NosqlBenchFiles.getWorkloadsWithScenarioScripts();
+            List<NosqlBenchFiles.WorkloadDesc> workloads = NosqlBenchFiles.getWorkloadsWithScenarioScripts();
-            for (Map.Entry<String, List<String>> entry : workloads.entrySet()) {
+            for (NosqlBenchFiles.WorkloadDesc workload : workloads) {
-                System.out.println("# from: "+ entry.getKey());
+                System.out.println("# from: "+ workload.getYamlPath());
-                List<String> scenarioList = entry.getValue();
+                List<String> scenarioList = workload.getScenarioNames();
-                String workloadName = entry.getKey().replaceAll("\\.yaml", "") ;
+                String workloadName = workload.getYamlPath().replaceAll("\\.yaml", "") ;
+                Set<String> templates = workload.getTemlpates();
 
                 for (String scenario : scenarioList) {
                     if (scenario.equals("default")) {
@@ -97,6 +95,13 @@ public class NBCLI {
                     }
                     System.out.println("  ./nb " + workloadName + " " + scenario);
                 }
+                if (templates.size()>0){
+                    System.out.println("# with the following optional parameters and defaults: ");
+                    templates.stream()
+                        .map(x -> x.replaceAll(",","="))
+                        .map(x -> "# "+x)
+                        .forEach(System.out::println);
+                }
             }
             System.exit(0);
         }

engine-cli/src/test/java/io/nosqlbench/engine/cli/TestNBCLIOptions.java

@@ -231,7 +231,7 @@ public class TestNBCLIOptions {
 
     @Test
     public void clTest() {
-        String dir= "activities/";
+        String dir= "./";
         URL resource = getClass().getClassLoader().getResource(dir);
         assertThat(resource);
         Path basePath = VirtDataResources.findPathIn(dir);