Handle nested parantheses in TEMPLATE processing (#2072)

* Handle nested parantheses in TEMPLATE processing

* Add more complex template to scenario

* Debugging and other advisors

* Tab cleanup

* Add double nesting fix and test

nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java

@@ -20,6 +20,7 @@ import com.amazonaws.util.StringInputStream;
 import com.google.gson.GsonBuilder;
 import io.nosqlbench.nb.api.nbio.Content;
 import io.nosqlbench.nb.api.nbio.NBIO;
+import io.nosqlbench.nb.api.advisor.NBAdvisorOutput;
 import io.nosqlbench.nb.api.advisor.NBAdvisorException;
 import io.nosqlbench.nb.api.errors.BasicError;
 import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsDocList;
@@ -29,6 +30,7 @@ import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList;
 import io.nosqlbench.adapters.api.templating.StrInterpolator;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.Level;
 import org.snakeyaml.engine.v2.api.Load;
 import org.snakeyaml.engine.v2.api.LoadSettings;
 import scala.Option;
@@ -69,6 +71,9 @@ public class OpsLoader {
         logger.trace(() -> "Applying string transformer to data:" + sourceData);
         StrInterpolator transformer = new StrInterpolator(params);
         String data = transformer.apply(sourceData);
+        NBAdvisorOutput.render(Level.INFO,"Transform:");
+        NBAdvisorOutput.render(Level.INFO,"From: "+sourceData);
+        NBAdvisorOutput.render(Level.INFO,"  To: "+data);
         if (srcuri!=null) {
             logger.info("workload URI: '" + srcuri + "'");
         }

nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java

@@ -17,10 +17,12 @@
 package io.nosqlbench.adapters.api.templating;
 
 import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef;
+import io.nosqlbench.nb.api.advisor.NBAdvisorOutput;
 import org.apache.commons.text.StrLookup;
 import org.apache.commons.text.StringSubstitutor;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.Level;
 
 import java.util.*;
 import java.util.function.Function;
@@ -35,12 +37,6 @@ public class StrInterpolator implements Function<String, String> {
             .setEnableUndefinedVariableException(true)
             .setDisableSubstitutionInValues(true);
 
-    private final StringSubstitutor substitutor2 =
-        new StringSubstitutor(multimap, "TEMPLATE(", ")", '\\')
-            .setEnableSubstitutionInVariables(true)
-            .setEnableUndefinedVariableException(true)
-            .setDisableSubstitutionInValues(true);
-
     public StrInterpolator(ActivityDef... activityDefs) {
         Arrays.stream(activityDefs)
             .map(ad -> ad.getParams().getStringStringMap())
@@ -58,10 +54,23 @@ public class StrInterpolator implements Function<String, String> {
 
     @Override
     public String apply(String raw) {
-        String after = substitutor.replace(substitutor2.replace(raw));
+        String after = matchTemplates(raw);
         while (!after.equals(raw)) {
             raw = after;
-            after = substitutor.replace(substitutor2.replace(raw));
+            after = matchTemplates(raw);
+        }
+        raw = after;
+        String original = raw;
+        after = substitutor.replace(raw);
+        while (!after.equals(raw)) {
+            raw = after;
+            after = substitutor.replace(raw);
+        }
+        if ( !original.equals(after)) {
+            NBAdvisorOutput.output(Level.WARN,"Transform <<key:value>>");
+            NBAdvisorOutput.output(Level.WARN,"From: "+original);
+            NBAdvisorOutput.output(Level.WARN,"  To: "+after);
+            NBAdvisorOutput.test("Using the deprecated template for of <<key:value>> please use TEMPLATE(key,value)");
         }
         return after;
     }
@@ -76,6 +85,50 @@ public class StrInterpolator implements Function<String, String> {
         return details;
     }
 
+    public String matchTemplates(String original) {
+        String line = original;
+        int length = line.length();
+        int i = 0;
+        while (i < length) {
+            // Detect an instance of "TEMPLATE("
+            if (line.startsWith("TEMPLATE(", i)) {
+                int start = i + "TEMPLATE(".length();
+                int openParensCount = 1; // We found one '(' with "TEMPLATE("
+                // Find the corresponding closing ')' for this TEMPLATE instance
+                int j = start;
+                int k = start;
+                while (j < length && openParensCount > 0) {
+                    if (line.charAt(j) == '(') {
+                        openParensCount++;
+                    } else if (line.charAt(j) == ')') {
+                        k = j;
+                        openParensCount--;
+                    }
+                    j++;
+                }
+                // check for case of not enough ')'
+                if (openParensCount > 0 ) {
+                    if ( k != start ) {
+                        j = k + 1;
+                    }
+                    openParensCount = 0;
+                }
+                // `j` now points just after the closing ')' of this TEMPLATE
+                if (openParensCount == 0) {
+                    String templateContent = line.substring(start, j - 1);
+                    // Resolve the template content
+                    String resolvedContent = multimap.lookup(templateContent);
+                    line = line.substring(0, i) + resolvedContent + line.substring(j);
+                    // Update `length` and `i` based on the modified `line`
+                    // i += resolvedContent.length() - 1;
+                    length = line.length();
+                }
+            }
+            i++;
+        }
+        return line;
+    }
+
     public static class MultiMap extends StrLookup<String> {
 
         private final List<Map<String, ?>> maps = new ArrayList<>();
@@ -132,5 +185,4 @@ public class StrInterpolator implements Function<String, String> {
         }
     }
 
-
 }

nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/templating/StrInterpolatorTest.java

@@ -19,6 +19,7 @@ package io.nosqlbench.adapters.api.templating;
 import org.junit.jupiter.api.Test;
 
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -77,6 +78,13 @@ public class StrInterpolatorTest {
         assertThat(b).isEqualTo("value2");
     }
 
+    @Test
+    public void shouldMatchNestedParens() {
+        StrInterpolator interp = new StrInterpolator(abcd);
+        String a = interp.apply("TEMPLATE(keydist,Uniform(0,1000000000)->int);");
+        assertThat(a).isEqualTo("Uniform(0,1000000000)->int;");
+    }
+
     @Test
     public void shouldReturnWarningWhenUnmatched() {
         StrInterpolator interp = new StrInterpolator(abcd);
@@ -129,20 +137,22 @@ public class StrInterpolatorTest {
         assertThat(a).isEqualTo("'Key': 'Value'.'Stuff'");
     }
 
-//    @Test
+    @Test
-//    public void shouldExpandNestedTemplates() {
+    public void shouldExpandNestedTemplates() {
-//        String a = interp.apply("-TEMPLATE(akey,TEMPLATE(dkey,whee)-");
+        StrInterpolator interp = new StrInterpolator(abcd);
-//        assertThat(a).isEqualTo("-aval1-");
+        String a = interp.apply("-TEMPLATE(akey,TEMPLATE(dkey,whee)-");
-//        String b = interp.apply("-TEMPLATE(unknown,TEMPLATE(bkey,whee))-");
+        assertThat(a).isEqualTo("-aval1-");
-//        assertThat(b).isEqualTo("-bval1-");
+        String b = interp.apply("-TEMPLATE(unknown,TEMPLATE(bkey,whee))-");
-//    }
+        assertThat(b).isEqualTo("-bval1-");
-//
+    }
+
 //    @Test
 //    public void shouldGetBasicDetails() {
+//        StrInterpolator interp = new StrInterpolator(abcd);
 //        LinkedHashMap<String, String> details = interp.getTemplateDetails("-TEMPLATE(akey,TEMPLATE(dkey,whee)-");
 //        assertThat(details).containsOnlyKeys("akey","dkey");
 //        assertThat(details).containsValues("test1");
 //
 //    }
-//
+
 }

nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/advisor/NBAdvisorOutput.java

@@ -0,0 +1,50 @@
+package io.nosqlbench.nb.api.advisor;
+
+/*
+ * Copyright (c) nosqlbench
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.Level;
+
+public class NBAdvisorOutput {
+    private final static Logger logger = LogManager.getLogger("ADVISOR");
+
+    public static void render(Level level,String message) {
+        if (NBAdvisorLevel.get() == NBAdvisorLevel.validate) {
+            output(level, message);
+        }
+    }
+    
+    public static void test(String message) {
+	output(Level.WARN, message);
+        if (NBAdvisorLevel.get() == NBAdvisorLevel.enforce) {
+	    throw new NBAdvisorException(message, 2);
+        }
+    }
+    
+    public static void output(Level level,String message) {
+	if (level ==  Level.INFO) {
+	    logger.info(message);
+	} else if (level == Level.WARN) {
+	    logger.warn(message);
+	} else if (level == Level.ERROR) {
+	    logger.error(message);
+}
+    }
+
+}

nb-engine/nb-engine-cli/src/test/resources/activities/example_scenarios_templatevars.yaml

@@ -6,5 +6,6 @@ scenarios:
 bindings:
   cycle: Identity()
   name: NumberNameToCycle()
+  rw_key: TEMPLATE(keydist,Uniform(0,1000000000)->int); ToString() -> String
 ops:
   cycle: "cycle {cycle} TEMPLATE(tvar1,def1) TEMPLATE(tvar1)\n"

nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioPreprocessor.java

@@ -312,10 +312,6 @@ public class NBCLIScenarioPreprocessor {
         return parsedStep;
     }
 
-    private static final Pattern templatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)");
-    private static final Pattern innerTemplatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)");
-    private static final Pattern templatePattern2 = Pattern.compile("<<(.+?)>>");
-
     public static List<WorkloadDesc> filterForScenarios(List<Content<?>> candidates) {
 
         List<Path> yamlPathList = candidates.stream().map(Content::asPath).toList();
@@ -437,28 +433,43 @@ public class NBCLIScenarioPreprocessor {
 
     }
 
+    //    private static final Pattern templatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)");
+    //    private static final Pattern innerTemplatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)");
+    private static final Pattern templatePattern2 = Pattern.compile("<<(.+?)>>");
+
     public static Map<String, String> matchTemplates(String line, Map<String, String> templates) {
-        Matcher matcher = templatePattern.matcher(line);
+        int length = line.length();
-
+        int i = 0;
-        while (matcher.find()) {
+        while (i < length) {
-            String match = matcher.group(1);
+            // Detect an instance of "TEMPLATE("
-
+            if (line.startsWith("TEMPLATE(", i)) {
-            Matcher innerMatcher = innerTemplatePattern.matcher(match);
+                int start = i + "TEMPLATE(".length();
-            if (innerMatcher.find()) {
+                int openParensCount = 1; // We found one '(' with "TEMPLATE("
-                String innerMatch = innerMatcher.group(1);
+                // Find the corresponding closing ')' for this TEMPLATE instance
-                templates = matchTemplates("TEMPLATE(" + innerMatch + ")", templates);
+                int j = start;
-                String resolvedInner = templates.getOrDefault(innerMatch.split("[,:]")[0], "");
+                while (j < length && openParensCount > 0) {
-                match = match.replace("TEMPLATE(" + innerMatch + ")", resolvedInner);
+                    if (line.charAt(j) == '(') {
+                        openParensCount++;
+                    } else if (line.charAt(j) == ')') {
+                        openParensCount--;
+                    }
+                    j++;
+                }
+                // `j` now points just after the closing ')' of this TEMPLATE
+                if (openParensCount == 0) {
+                    String templateContent = line.substring(start, j - 1);
+                    // Resolve the template content
+                    String resolvedContent = resolveTemplate(templateContent, templates);
+                    line = line.substring(0, i) + resolvedContent + line.substring(j);
+                    // Update `length` and `i` based on the modified `line`
+                    i += resolvedContent.length() - 1;
+                    length = line.length();
+                }
+            }
+            i++;
         }
 
-            String[] matchArray = match.split("[,:]");
+        Matcher matcher = templatePattern2.matcher(line);
-            if (matchArray.length == 1) {
-                matchArray = new String[]{matchArray[0], ""};
-            }
-            templates.put(matchArray[0], matchArray[1]);
-        }
-        matcher = templatePattern2.matcher(line);
-
         while (matcher.find()) {
             String match = matcher.group(1);
             String[] matchArray = match.split(":");
@@ -471,5 +482,17 @@ public class NBCLIScenarioPreprocessor {
         return templates;
     }
 
+    private static String resolveTemplate(String content, Map<String, String> templates) {
+        String[] parts = content.split("[,:]", 2);
+        String key = parts[0];
+        String value = parts.length > 1 ? parts[1] : "";
+
+        // Store or retrieve the resolved value for the template key
+        if (!templates.containsKey(key)) {
+            templates.put(key, value);
+        }
+        
+        return templates.getOrDefault(key, "");
+    }
 
 }