From da609d76dad41db01ca9eb708004ff3000392f24 Mon Sep 17 00:00:00 2001 From: Dave Fisher Date: Thu, 21 Nov 2024 09:59:23 -0800 Subject: [PATCH] Refactor StrInterpolator including Skipping Comments and Adding Support for Bash Variable Substitution (#2092) * Commented TEMPLATEs in yamls are ignored * Templates only processed in StrInterpolator * Improve ends with new line test * Adjust deprecated template advisor message * Recursively process template key-values * Adds bash style simple templates --------- Co-authored-by: Jonathan Shook --- .../api/activityconfig/OpsLoader.java | 8 +- .../api/templating/StrInterpolator.java | 83 ++++++++++++++----- .../api/templating/StrInterpolatorTest.java | 30 +++++++ .../scenarios/NBCLIScenarioPreprocessor.java | 69 +-------------- 4 files changed, 95 insertions(+), 95 deletions(-) diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java index 122489114..770a6cccb 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/OpsLoader.java @@ -20,7 +20,6 @@ import com.amazonaws.util.StringInputStream; import com.google.gson.GsonBuilder; import io.nosqlbench.nb.api.nbio.Content; import io.nosqlbench.nb.api.nbio.NBIO; -import io.nosqlbench.nb.api.advisor.NBAdvisorOutput; import io.nosqlbench.nb.api.advisor.NBAdvisorException; import io.nosqlbench.nb.api.errors.BasicError; import io.nosqlbench.adapters.api.activityconfig.rawyaml.RawOpsDocList; @@ -69,14 +68,11 @@ public class OpsLoader { public static OpsDocList loadString(final String sourceData, OpTemplateFormat fmt, Map params, URI srcuri) { logger.trace(() -> "Applying string transformer to data:" + sourceData); - StrInterpolator transformer = new StrInterpolator(params); - String data = transformer.apply(sourceData); - NBAdvisorOutput.render(Level.INFO,"Transform:"); - NBAdvisorOutput.render(Level.INFO,"From: "+sourceData); - NBAdvisorOutput.render(Level.INFO," To: "+data); if (srcuri!=null) { logger.info("workload URI: '" + srcuri + "'"); } + StrInterpolator transformer = new StrInterpolator(params); + String data = transformer.apply(sourceData); RawOpsLoader loader = new RawOpsLoader(transformer); RawOpsDocList rawOpsDocList = switch (fmt) { diff --git a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java index 680660ab0..08271ada2 100644 --- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java +++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/templating/StrInterpolator.java @@ -54,25 +54,23 @@ public class StrInterpolator implements Function { @Override public String apply(String raw) { - String after = matchTemplates(raw); - while (!after.equals(raw)) { - raw = after; - after = matchTemplates(raw); + String[] lines = raw.split("\\R"); + boolean endsWithNewline = raw.endsWith("\n"); + int i = 0; + for (String line : lines) { + if (!line.startsWith("#")) { + String result = matchTemplates(line); + if (!result.equals(line)) { + lines[i] = result; + } + } + i++; } - raw = after; - String original = raw; - after = substitutor.replace(raw); - while (!after.equals(raw)) { - raw = after; - after = substitutor.replace(raw); + String results = String.join(System.lineSeparator(), lines); + if (endsWithNewline) { + results += System.lineSeparator(); } - if ( !original.equals(after)) { - NBAdvisorOutput.render(Level.WARN,"Transform <>"); - NBAdvisorOutput.render(Level.WARN,"From: "+original); - NBAdvisorOutput.render(Level.WARN," To: "+after); - NBAdvisorOutput.render(Level.WARN,"The deprecated template <> in use. Please use TEMPLATE(key,value)"); - } - return after; + return results; } public Map checkpointAccesses() { @@ -86,12 +84,44 @@ public class StrInterpolator implements Function { } public String matchTemplates(String original) { + // process TEMPLATE(...) String line = original; int length = line.length(); int i = 0; while (i < length) { - // Detect an instance of "TEMPLATE(" - if (line.startsWith("TEMPLATE(", i)) { + if (line.startsWith("${", i)) { + int start = i + "${".length(); + int openParensCount = 1; // We found one '{' with "${" + // Find the corresponding closing ')' for this TEMPLATE instance + int j = start; + int k = start; + while (j < length && openParensCount > 0) { + if (line.charAt(j) == '{') { + openParensCount++; + } else if (line.charAt(j) == '}') { + k = j; + openParensCount--; + } + j++; + } + // check for case of not enough '}' + if (openParensCount > 0 ) { + if ( k != start ) { + j = k + 1; + } + openParensCount = 0; + } + // `j` now points just after the closing '}' of this ${ + if (openParensCount == 0) { + String templateContent = line.substring(start, j - 1); + // Recursively process + String templateValue = matchTemplates(templateContent); + String resolvedContent = multimap.lookup(templateValue); + line = line.substring(0, i) + resolvedContent + line.substring(j); + length = line.length(); + i--; + } + } else if (line.startsWith("TEMPLATE(", i)) { int start = i + "TEMPLATE(".length(); int openParensCount = 1; // We found one '(' with "TEMPLATE(" // Find the corresponding closing ')' for this TEMPLATE instance @@ -116,16 +146,23 @@ public class StrInterpolator implements Function { // `j` now points just after the closing ')' of this TEMPLATE if (openParensCount == 0) { String templateContent = line.substring(start, j - 1); - // Resolve the template content - String resolvedContent = multimap.lookup(templateContent); + // Recursively process + String templateValue = matchTemplates(templateContent); + String resolvedContent = multimap.lookup(templateValue); line = line.substring(0, i) + resolvedContent + line.substring(j); - // Update `length` and `i` based on the modified `line` - // i += resolvedContent.length() - 1; length = line.length(); + i--; } } i++; } + // Process << ... >> + String after = substitutor.replace(line); + while (!after.equals(line)) { + NBAdvisorOutput.test("<> deprecated in "+line); + line = after; + after = substitutor.replace(line); + } return line; } diff --git a/nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/templating/StrInterpolatorTest.java b/nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/templating/StrInterpolatorTest.java index ec7bd282a..0e59edbc6 100644 --- a/nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/templating/StrInterpolatorTest.java +++ b/nb-apis/adapters-api/src/test/java/io/nosqlbench/adapters/api/templating/StrInterpolatorTest.java @@ -78,6 +78,15 @@ public class StrInterpolatorTest { assertThat(b).isEqualTo("value2"); } + @Test + public void shouldMatchNewAlternateSubst() { + StrInterpolator interp = new StrInterpolator(abcd); + String a = interp.apply("${akey}"); + assertThat(a).isEqualTo("aval1"); + String b = interp.apply("${nokeymatches:value2}"); + assertThat(b).isEqualTo("value2"); + } + @Test public void shouldMatchNestedParens() { StrInterpolator interp = new StrInterpolator(abcd); @@ -85,6 +94,27 @@ public class StrInterpolatorTest { assertThat(a).isEqualTo("Uniform(0,1000000000)->int;"); } + @Test + public void shouldMatchNestedParens2() { + StrInterpolator interp = new StrInterpolator(abcd); + String a = interp.apply("${keydist:Uniform(0,1000000000)->int};"); + assertThat(a).isEqualTo("Uniform(0,1000000000)->int;"); + } + + @Test + public void shouldMatchNestedParens3() { + StrInterpolator interp = new StrInterpolator(abcd); + String a = interp.apply("${keydist:${nokeymatchesthis2:Uniform(0,1000000000)->int}};"); + assertThat(a).isEqualTo("Uniform(0,1000000000)->int;"); + } + + @Test + public void shouldMatchWithComments() { + StrInterpolator interp = new StrInterpolator(abcd); + String a = interp.apply("TEMPLATE(start,START)\n# TEMPLATE(blahblah,blah)\nTEMPLATE(keydist,Uniform(0,1000000000)->int);"); + assertThat(a).isEqualTo("START\n# TEMPLATE(blahblah,blah)\nUniform(0,1000000000)->int;"); + } + @Test public void shouldReturnWarningWhenUnmatched() { StrInterpolator interp = new StrInterpolator(abcd); diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioPreprocessor.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioPreprocessor.java index 3ea85cff5..e966e0ce2 100644 --- a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioPreprocessor.java +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioPreprocessor.java @@ -350,11 +350,10 @@ public class NBCLIScenarioPreprocessor { } Map templates = new LinkedHashMap<>(); + StrInterpolator templateparser = new StrInterpolator(templates); try { - List lines = Files.readAllLines(yamlPath); - for (String line : lines) { - templates = matchTemplates(line, templates); - } + templateparser.apply( Files.readString(yamlPath) ); + templates = templateparser.checkpointAccesses(); } catch (IOException e) { throw new RuntimeException(e); } @@ -433,66 +432,4 @@ public class NBCLIScenarioPreprocessor { } - // private static final Pattern templatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)"); - // private static final Pattern innerTemplatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)"); - private static final Pattern templatePattern2 = Pattern.compile("<<(.+?)>>"); - - public static Map matchTemplates(String line, Map templates) { - int length = line.length(); - int i = 0; - while (i < length) { - // Detect an instance of "TEMPLATE(" - if (line.startsWith("TEMPLATE(", i)) { - int start = i + "TEMPLATE(".length(); - int openParensCount = 1; // We found one '(' with "TEMPLATE(" - // Find the corresponding closing ')' for this TEMPLATE instance - int j = start; - while (j < length && openParensCount > 0) { - if (line.charAt(j) == '(') { - openParensCount++; - } else if (line.charAt(j) == ')') { - openParensCount--; - } - j++; - } - // `j` now points just after the closing ')' of this TEMPLATE - if (openParensCount == 0) { - String templateContent = line.substring(start, j - 1); - // Resolve the template content - String resolvedContent = resolveTemplate(templateContent, templates); - line = line.substring(0, i) + resolvedContent + line.substring(j); - // Update `length` and `i` based on the modified `line` - i += resolvedContent.length() - 1; - length = line.length(); - } - } - i++; - } - - Matcher matcher = templatePattern2.matcher(line); - while (matcher.find()) { - String match = matcher.group(1); - String[] matchArray = match.split(":"); - if (matchArray.length == 1) { - templates.put(matchArray[0], "-none-"); - } else { - templates.put(matchArray[0], matchArray[1]); - } - } - return templates; - } - - private static String resolveTemplate(String content, Map templates) { - String[] parts = content.split("[,:]", 2); - String key = parts[0]; - String value = parts.length > 1 ? parts[1] : ""; - - // Store or retrieve the resolved value for the template key - if (!templates.containsKey(key)) { - templates.put(key, value); - } - - return templates.getOrDefault(key, ""); - } - }