From 64917a2879e28fdbfcc209bf1238ed97b55d9f7d Mon Sep 17 00:00:00 2001 From: Dave Fisher Date: Tue, 12 Nov 2024 11:42:07 -0800 Subject: [PATCH] Handle nested parantheses in TEMPLATE processing (#2072) * Handle nested parantheses in TEMPLATE processing * Add more complex template to scenario * Debugging and other advisors * Tab cleanup * Add double nesting fix and test --- .../api/activityconfig/OpsLoader.java | 5 ++ .../api/templating/StrInterpolator.java | 70 ++++++++++++++++--- .../api/templating/StrInterpolatorTest.java | 28 +++++--- .../nb/api/advisor/NBAdvisorOutput.java | 50 +++++++++++++ .../example_scenarios_templatevars.yaml | 1 + .../scenarios/NBCLIScenarioPreprocessor.java | 67 ++++++++++++------ 6 files changed, 181 insertions(+), 40 deletions(-) create mode 100644 nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/advisor/NBAdvisorOutput.java diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java index 797b682d5..122489114 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java @@ -20,6 +20,7 @@ import com.amazonaws.util.StringInputStream; import com.google.gson.GsonBuilder; import io.nosqlbench.nb.api.nbio.Content; import io.nosqlbench.nb.api.nbio.NBIO; +import io.nosqlbench.nb.api.advisor.NBAdvisorOutput; import io.nosqlbench.nb.api.advisor.NBAdvisorException; import io.nosqlbench.nb.api.errors.BasicError; import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsDocList; @@ -29,6 +30,7 @@ import io.nosqlbench.adapters.api.activityconfig.yaml.OpsDocList; import io.nosqlbench.adapters.api.templating.StrInterpolator; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Level; import org.snakeyaml.engine.v2.api.Load; import org.snakeyaml.engine.v2.api.LoadSettings; import scala.Option; @@ -69,6 +71,9 @@ public class OpsLoader { logger.trace(() -> "Applying string transformer to data:" + sourceData); StrInterpolator transformer = new StrInterpolator(params); String data = transformer.apply(sourceData); + NBAdvisorOutput.render(Level.INFO,"Transform:"); + NBAdvisorOutput.render(Level.INFO,"From: "+sourceData); + NBAdvisorOutput.render(Level.INFO," To: "+data); if (srcuri!=null) { logger.info("workload URI: '" + srcuri + "'"); } diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java index 1c6fe9100..d6f643700 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java @@ -17,10 +17,12 @@ package io.nosqlbench.adapters.api.templating; import io.nosqlbench.nb.api.engine.activityimpl.ActivityDef; +import io.nosqlbench.nb.api.advisor.NBAdvisorOutput; import org.apache.commons.text.StrLookup; import org.apache.commons.text.StringSubstitutor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Level; import java.util.*; import java.util.function.Function; @@ -35,12 +37,6 @@ public class StrInterpolator implements Function { .setEnableUndefinedVariableException(true) .setDisableSubstitutionInValues(true); - private final StringSubstitutor substitutor2 = - new StringSubstitutor(multimap, "TEMPLATE(", ")", '\\') - .setEnableSubstitutionInVariables(true) - .setEnableUndefinedVariableException(true) - .setDisableSubstitutionInValues(true); - public StrInterpolator(ActivityDef... activityDefs) { Arrays.stream(activityDefs) .map(ad -> ad.getParams().getStringStringMap()) @@ -58,10 +54,23 @@ public class StrInterpolator implements Function { @Override public String apply(String raw) { - String after = substitutor.replace(substitutor2.replace(raw)); + String after = matchTemplates(raw); while (!after.equals(raw)) { raw = after; - after = substitutor.replace(substitutor2.replace(raw)); + after = matchTemplates(raw); + } + raw = after; + String original = raw; + after = substitutor.replace(raw); + while (!after.equals(raw)) { + raw = after; + after = substitutor.replace(raw); + } + if ( !original.equals(after)) { + NBAdvisorOutput.output(Level.WARN,"Transform <>"); + NBAdvisorOutput.output(Level.WARN,"From: "+original); + NBAdvisorOutput.output(Level.WARN," To: "+after); + NBAdvisorOutput.test("Using the deprecated template for of <> please use TEMPLATE(key,value)"); } return after; } @@ -76,6 +85,50 @@ public class StrInterpolator implements Function { return details; } + public String matchTemplates(String original) { + String line = original; + int length = line.length(); + int i = 0; + while (i < length) { + // Detect an instance of "TEMPLATE(" + if (line.startsWith("TEMPLATE(", i)) { + int start = i + "TEMPLATE(".length(); + int openParensCount = 1; // We found one '(' with "TEMPLATE(" + // Find the corresponding closing ')' for this TEMPLATE instance + int j = start; + int k = start; + while (j < length && openParensCount > 0) { + if (line.charAt(j) == '(') { + openParensCount++; + } else if (line.charAt(j) == ')') { + k = j; + openParensCount--; + } + j++; + } + // check for case of not enough ')' + if (openParensCount > 0 ) { + if ( k != start ) { + j = k + 1; + } + openParensCount = 0; + } + // `j` now points just after the closing ')' of this TEMPLATE + if (openParensCount == 0) { + String templateContent = line.substring(start, j - 1); + // Resolve the template content + String resolvedContent = multimap.lookup(templateContent); + line = line.substring(0, i) + resolvedContent + line.substring(j); + // Update `length` and `i` based on the modified `line` + // i += resolvedContent.length() - 1; + length = line.length(); + } + } + i++; + } + return line; + } + public static class MultiMap extends StrLookup { private final List> maps = new ArrayList<>(); @@ -132,5 +185,4 @@ public class StrInterpolator implements Function { } } - } diff --git a/nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/templating/StrInterpolatorTest.java b/nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/templating/StrInterpolatorTest.java index c2273fb58..ec7bd282a 100644 --- a/nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/templating/StrInterpolatorTest.java +++ b/nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/templating/StrInterpolatorTest.java @@ -19,6 +19,7 @@ package io.nosqlbench.adapters.api.templating; import org.junit.jupiter.api.Test; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -77,6 +78,13 @@ public class StrInterpolatorTest { assertThat(b).isEqualTo("value2"); } + @Test + public void shouldMatchNestedParens() { + StrInterpolator interp = new StrInterpolator(abcd); + String a = interp.apply("TEMPLATE(keydist,Uniform(0,1000000000)->int);"); + assertThat(a).isEqualTo("Uniform(0,1000000000)->int;"); + } + @Test public void shouldReturnWarningWhenUnmatched() { StrInterpolator interp = new StrInterpolator(abcd); @@ -129,20 +137,22 @@ public class StrInterpolatorTest { assertThat(a).isEqualTo("'Key': 'Value'.'Stuff'"); } -// @Test -// public void shouldExpandNestedTemplates() { -// String a = interp.apply("-TEMPLATE(akey,TEMPLATE(dkey,whee)-"); -// assertThat(a).isEqualTo("-aval1-"); -// String b = interp.apply("-TEMPLATE(unknown,TEMPLATE(bkey,whee))-"); -// assertThat(b).isEqualTo("-bval1-"); -// } -// + @Test + public void shouldExpandNestedTemplates() { + StrInterpolator interp = new StrInterpolator(abcd); + String a = interp.apply("-TEMPLATE(akey,TEMPLATE(dkey,whee)-"); + assertThat(a).isEqualTo("-aval1-"); + String b = interp.apply("-TEMPLATE(unknown,TEMPLATE(bkey,whee))-"); + assertThat(b).isEqualTo("-bval1-"); + } + // @Test // public void shouldGetBasicDetails() { +// StrInterpolator interp = new StrInterpolator(abcd); // LinkedHashMap details = interp.getTemplateDetails("-TEMPLATE(akey,TEMPLATE(dkey,whee)-"); // assertThat(details).containsOnlyKeys("akey","dkey"); // assertThat(details).containsValues("test1"); // // } -// + } diff --git a/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/advisor/NBAdvisorOutput.java b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/advisor/NBAdvisorOutput.java new file mode 100644 index 000000000..c0f42f525 --- /dev/null +++ b/nb-apis/nb-api/src/main/java/io/nosqlbench/nb/api/advisor/NBAdvisorOutput.java @@ -0,0 +1,50 @@ +package io.nosqlbench.nb.api.advisor; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Level; + +public class NBAdvisorOutput { + private final static Logger logger = LogManager.getLogger("ADVISOR"); + + public static void render(Level level,String message) { + if (NBAdvisorLevel.get() == NBAdvisorLevel.validate) { + output(level, message); + } + } + + public static void test(String message) { + output(Level.WARN, message); + if (NBAdvisorLevel.get() == NBAdvisorLevel.enforce) { + throw new NBAdvisorException(message, 2); + } + } + + public static void output(Level level,String message) { + if (level == Level.INFO) { + logger.info(message); + } else if (level == Level.WARN) { + logger.warn(message); + } else if (level == Level.ERROR) { + logger.error(message); +} + } + +} diff --git a/nb-engine/nb-engine-cli/src/test/resources/activities/example_scenarios_templatevars.yaml b/nb-engine/nb-engine-cli/src/test/resources/activities/example_scenarios_templatevars.yaml index 026304bbf..6a532ee17 100644 --- a/nb-engine/nb-engine-cli/src/test/resources/activities/example_scenarios_templatevars.yaml +++ b/nb-engine/nb-engine-cli/src/test/resources/activities/example_scenarios_templatevars.yaml @@ -6,5 +6,6 @@ scenarios: bindings: cycle: Identity() name: NumberNameToCycle() + rw_key: TEMPLATE(keydist,Uniform(0,1000000000)->int); ToString() -> String ops: cycle: "cycle {cycle} TEMPLATE(tvar1,def1) TEMPLATE(tvar1)\n" diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioPreprocessor.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioPreprocessor.java index 6168bbe84..3ea85cff5 100644 --- a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioPreprocessor.java +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioPreprocessor.java @@ -312,10 +312,6 @@ public class NBCLIScenarioPreprocessor { return parsedStep; } - private static final Pattern templatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)"); - private static final Pattern innerTemplatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)"); - private static final Pattern templatePattern2 = Pattern.compile("<<(.+?)>>"); - public static List filterForScenarios(List> candidates) { List yamlPathList = candidates.stream().map(Content::asPath).toList(); @@ -437,28 +433,43 @@ public class NBCLIScenarioPreprocessor { } + // private static final Pattern templatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)"); + // private static final Pattern innerTemplatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)"); + private static final Pattern templatePattern2 = Pattern.compile("<<(.+?)>>"); + public static Map matchTemplates(String line, Map templates) { - Matcher matcher = templatePattern.matcher(line); - - while (matcher.find()) { - String match = matcher.group(1); - - Matcher innerMatcher = innerTemplatePattern.matcher(match); - if (innerMatcher.find()) { - String innerMatch = innerMatcher.group(1); - templates = matchTemplates("TEMPLATE(" + innerMatch + ")", templates); - String resolvedInner = templates.getOrDefault(innerMatch.split("[,:]")[0], ""); - match = match.replace("TEMPLATE(" + innerMatch + ")", resolvedInner); + int length = line.length(); + int i = 0; + while (i < length) { + // Detect an instance of "TEMPLATE(" + if (line.startsWith("TEMPLATE(", i)) { + int start = i + "TEMPLATE(".length(); + int openParensCount = 1; // We found one '(' with "TEMPLATE(" + // Find the corresponding closing ')' for this TEMPLATE instance + int j = start; + while (j < length && openParensCount > 0) { + if (line.charAt(j) == '(') { + openParensCount++; + } else if (line.charAt(j) == ')') { + openParensCount--; + } + j++; + } + // `j` now points just after the closing ')' of this TEMPLATE + if (openParensCount == 0) { + String templateContent = line.substring(start, j - 1); + // Resolve the template content + String resolvedContent = resolveTemplate(templateContent, templates); + line = line.substring(0, i) + resolvedContent + line.substring(j); + // Update `length` and `i` based on the modified `line` + i += resolvedContent.length() - 1; + length = line.length(); + } } - - String[] matchArray = match.split("[,:]"); - if (matchArray.length == 1) { - matchArray = new String[]{matchArray[0], ""}; - } - templates.put(matchArray[0], matchArray[1]); + i++; } - matcher = templatePattern2.matcher(line); + Matcher matcher = templatePattern2.matcher(line); while (matcher.find()) { String match = matcher.group(1); String[] matchArray = match.split(":"); @@ -471,5 +482,17 @@ public class NBCLIScenarioPreprocessor { return templates; } + private static String resolveTemplate(String content, Map templates) { + String[] parts = content.split("[,:]", 2); + String key = parts[0]; + String value = parts.length > 1 ? parts[1] : ""; + + // Store or retrieve the resolved value for the template key + if (!templates.containsKey(key)) { + templates.put(key, value); + } + + return templates.getOrDefault(key, ""); + } }