IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-01-09 15:33:12 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1129 from nosqlbench/nosqlbench-1092-parsedop-captures

Browse Source 
nosqlbench-1092-parsedop-captures
This commit is contained in:
Jonathan Shook 2023-02-22 16:26:26 -06:00 committed by GitHub
commit 62d5832d1a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 208 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4PreparedStmtDispenser.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -26,7 +26,7 @@ import io.nosqlbench.adapter.cqld4.optypes.Cqld4CqlPreparedStatement;
import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.engine.api.templating.ParsedOp;
import io.nosqlbench.api.errors.OpConfigError;
import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -37,13 +37,13 @@ public class Cqld4PreparedStmtDispenser extends Cqld4BaseOpDispenser {
private final RSProcessors processors;
private final LongFunction<Statement> stmtFunc;
private final ParsedStringTemplate stmtTpl;
private final ParsedTemplateString stmtTpl;
private final LongFunction<Object[]> fieldsF;
private PreparedStatement preparedStmt;
private CqlSession boundSession;
public Cqld4PreparedStmtDispenser(
DriverAdapter adapter, LongFunction<CqlSession> sessionFunc, ParsedOp op, ParsedStringTemplate stmtTpl, RSProcessors processors) {
DriverAdapter adapter, LongFunction<CqlSession> sessionFunc, ParsedOp op, ParsedTemplateString stmtTpl, RSProcessors processors) {
super(adapter, sessionFunc, op);
if (op.isDynamic("space")) {
throw new RuntimeException("Prepared statements and dynamic space values are not supported." +

6
adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/CqlD4PreparedStmtMapper.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -30,7 +30,7 @@ import io.nosqlbench.engine.api.templating.ParsedOp;
import io.nosqlbench.engine.api.templating.TypeAndTarget;
import io.nosqlbench.api.config.params.ParamsParser;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
import java.util.List;
import java.util.Map;
@ -51,7 +51,7 @@ public class CqlD4PreparedStmtMapper implements OpMapper<Cqld4CqlOp> {
public OpDispenser<Cqld4CqlOp> apply(ParsedOp op) {
ParsedStringTemplate stmtTpl = op.getAsTemplate(target.field).orElseThrow(() -> new BasicError(
ParsedTemplateString stmtTpl = op.getAsTemplate(target.field).orElseThrow(() -> new BasicError(
"No statement was found in the op template:" + op
));

6
adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opmappers/Cqld4FluentGraphOpMapper.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -31,7 +31,7 @@ import io.nosqlbench.engine.api.templating.TypeAndTarget;
import io.nosqlbench.api.errors.OpConfigError;
import io.nosqlbench.virtdata.core.bindings.Bindings;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
@ -63,7 +63,7 @@ public class Cqld4FluentGraphOpMapper implements OpMapper<Op> {
public OpDispenser<? extends Op> apply(ParsedOp op) {
GraphTraversalSource g = DseGraph.g;
ParsedStringTemplate fluent = op.getAsTemplate(target.field).orElseThrow();
ParsedTemplateString fluent = op.getAsTemplate(target.field).orElseThrow();
String scriptBodyWithRawVarRefs = fluent.getPositionalStatement();
CompilerConfiguration compilerConfiguration = new CompilerConfiguration();

129
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/ParsedOpTemplate.java
View File

@ -1,129 +0,0 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityconfig;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.api.config.params.Element;
import io.nosqlbench.api.config.params.NBParams;
import io.nosqlbench.virtdata.core.templates.BindPoint;
import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
public class ParsedOpTemplate {
private final OpTemplate optpl;
private final ParsedStringTemplate parsed;
/**
* Construct a new ParsedStatement from the provided stmtDef and anchor token.
*
* @param optpl An existing statement def as read from the YAML API.
*/
public ParsedOpTemplate(OpTemplate optpl) {
this.optpl = optpl;
String transformed = getStmt();
parsed = new ParsedStringTemplate(transformed, optpl.getBindings());
}
public ParsedOpTemplate orError() {
if (hasError()) {
throw new RuntimeException("Unable to parse statement: " + this);
}
return this;
}
public String toString() {
return parsed.toString();
}
/**
* @return true if the parsed statement is not usable.
*/
public boolean hasError() {
return parsed.hasError();
}
/**
* Returns a list of binding names which were referenced
* in either <pre>{anchor}</pre> or <pre>?anchor</pre> form,
* but which were not present in the provided bindings map.
* If any binding names are present in the returned set, then
* this binding will not be usable.
*
* @return A list of binding names which were referenced but not defined*
*/
public Set<String> getMissingBindings() {
return parsed.getMissing();
}
/**
* Return the statement that can be used as-is by any driver specific version.
* This uses the anchor token as provided to yield a version of the statement
* which contains positional anchors, but no named bindings.
* @param tokenMapper A function which maps the anchor name to the needed form
* in the callers driver context
* @return A driver or usage-specific format of the statement, with anchors
*/
public String getPositionalStatement(Function<String,String> tokenMapper) {
return parsed.getPositionalStatement(tokenMapper);
}
/**
* @return the statement name from the enclosed {@link OpTemplate}
*/
public String getName() {
return optpl.getName();
}
/**
* @return the raw statement from the enclosed {@link OpTemplate}
*/
public String getStmt() {
return optpl.getStmt().orElseThrow();
}
/**
* @return the tags from the enclosed {@link OpTemplate}
*/
public Map<String, String> getTags() {
return optpl.getTags();
}
/**
* @return the bindings from the enclosed {@link OpTemplate}
*/
public Map<String, String> getBindings() {
return optpl.getBindings();
}
/**
* @return a params reader from the enclosed {@link OpTemplate} params map
*/
public Element getParamReader() {
return NBParams.one(getName(),optpl.getParams());
}
public List<BindPoint> getBindPoints() {
return parsed.getBindPoints();
}
}

12
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/yaml/OpTemplate.java
View File

@ -23,7 +23,7 @@ import io.nosqlbench.api.config.params.Element;
import io.nosqlbench.api.config.params.NBParams;
import io.nosqlbench.api.config.standard.NBTypeConverter;
import io.nosqlbench.api.errors.OpConfigError;
import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
import java.util.LinkedHashMap;
import java.util.Map;
@ -170,9 +170,9 @@ public abstract class OpTemplate implements Tagged {
* Parse the statement for anchors and return a richer view of the StmtDef which
* is simpler to use for most statement configuration needs.
*
* @return an optional {@link ParsedStringTemplate}
* @return an optional {@link ParsedTemplateString}
*/
public Optional<ParsedStringTemplate> getParsed(Function<String, String>... rewriters) {
public Optional<ParsedTemplateString> getParsed(Function<String, String>... rewriters) {
Optional<String> os = getStmt();
return os.map(s -> {
String result = s;
@ -180,11 +180,11 @@ public abstract class OpTemplate implements Tagged {
result = rewriter.apply(result);
}
return result;
}).map(s -> new ParsedStringTemplate(s, getBindings()));
}).map(s -> new ParsedTemplateString(s, getBindings()));
}
public Optional<ParsedStringTemplate> getParsed() {
return getStmt().map(s -> new ParsedStringTemplate(s, getBindings()));
public Optional<ParsedTemplateString> getParsed() {
return getStmt().map(s -> new ParsedTemplateString(s, getBindings()));
}
public abstract Optional<Map<String, Object>> getOp();

6
adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/flowtypes/VariableCapture.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,7 +16,7 @@
package io.nosqlbench.engine.api.activityimpl.uniform.flowtypes;
import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
import java.util.Map;
@ -24,7 +24,7 @@ import java.util.Map;
* If an op implements VariableCapture, then it is known to be able to
* extract variables from its result. Generally speaking, this should
* be implemented within an Op according to the standard format
* of {@link ParsedStringTemplate#getCaptures()}. Any op implementing
* of {@link ParsedTemplateString#getCaptures()}. Any op implementing
* this should use the interface below to support interop between adapters
* and to allow for auto documentation tha the feature is supported for
* a given adapter.

6
adapters-api/src/main/java/io/nosqlbench/engine/api/templating/CommandTemplate.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,7 +20,7 @@ import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.api.config.params.ParamsParser;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.virtdata.core.bindings.BindingsTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
import io.nosqlbench.virtdata.core.templates.StringBindings;
import io.nosqlbench.virtdata.core.templates.StringBindingsTemplate;
import org.apache.logging.log4j.Logger;
@ -160,7 +160,7 @@ public class CommandTemplate {
cmd.putAll(params);
cmd.forEach((param, value) -> {
ParsedStringTemplate paramTemplate = new ParsedStringTemplate(value, bindings);
ParsedTemplateString paramTemplate = new ParsedTemplateString(value, bindings);
if (paramTemplate.getBindPoints().size() > 0) {
BindingsTemplate paramBindings = new BindingsTemplate(paramTemplate.getBindPoints());
StringBindings paramStringBindings = new StringBindingsTemplate(value, paramBindings).resolve();

18
adapters-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedOp.java
View File

@ -16,18 +16,18 @@
package io.nosqlbench.engine.api.templating;
import io.nosqlbench.api.config.fieldreaders.DynamicFieldReader;
import io.nosqlbench.api.config.fieldreaders.StaticFieldReader;
import io.nosqlbench.api.config.standard.NBConfigError;
import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.api.errors.OpConfigError;
import io.nosqlbench.engine.api.activityconfig.yaml.OpTemplate;
import io.nosqlbench.engine.api.templating.binders.ArrayBinder;
import io.nosqlbench.engine.api.templating.binders.ListBinder;
import io.nosqlbench.engine.api.templating.binders.OrderedMapBinder;
import io.nosqlbench.api.config.fieldreaders.DynamicFieldReader;
import io.nosqlbench.api.config.fieldreaders.StaticFieldReader;
import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.api.errors.OpConfigError;
import io.nosqlbench.virtdata.core.templates.BindPoint;
import io.nosqlbench.virtdata.core.templates.CapturePoint;
import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -302,7 +302,7 @@ public class ParsedOp implements LongFunction<Map<String, ?>>, StaticFieldReader
* representation of a result. If the values are defined, then each one represents the name
* that the found value should be saved as instead of the original name.
*/
private final List<List<CapturePoint>> captures = new ArrayList<>();
private final List<CapturePoint> captures = new ArrayList<>();
private final OpTemplate _opTemplate;
private final NBConfiguration activityCfg;
@ -426,7 +426,7 @@ public class ParsedOp implements LongFunction<Map<String, ?>>, StaticFieldReader
return tmap.getStaticValue(field);
}
public Optional<ParsedStringTemplate> getAsTemplate(String fieldname) {
public Optional<ParsedTemplateString> getAsTemplate(String fieldname) {
return this.tmap.getAsStringTemplate(fieldname);
}
@ -916,4 +916,8 @@ public class ParsedOp implements LongFunction<Map<String, ?>>, StaticFieldReader
public String toString() {
return this.tmap.toString();
}
public List<CapturePoint> getCaptures() {
return tmap.getCaptures();
}
}

14
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedOpTemplateTest.java → adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/yaml/ParsedWorkloadTemplateTest.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.engine.api.activityconfig.yaml;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.BeforeAll;
@ -27,8 +27,8 @@ import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class ParsedOpTemplateTest {
private static final Logger logger = LogManager.getLogger(ParsedOpTemplateTest.class);
public class ParsedWorkloadTemplateTest {
private static final Logger logger = LogManager.getLogger(ParsedWorkloadTemplateTest.class);
private static OpsDocList doclist;
@BeforeAll
@ -40,13 +40,13 @@ public class ParsedOpTemplateTest {
public void testBasicParser() {
OpsBlock block0 = doclist.getStmtDocs().get(0).getBlocks().get(0);
OpTemplate stmtDef0 = block0.getOps().get(0);
ParsedStringTemplate parsed0 = stmtDef0.getParsed().orElseThrow();
ParsedTemplateString parsed0 = stmtDef0.getParsed().orElseThrow();
assertThat(parsed0.getMissing()).containsExactly("delta");
assertThat(parsed0.hasError()).isTrue();
OpsBlock block1 = doclist.getStmtDocs().get(0).getBlocks().get(1);
OpTemplate stmtDef1 = block1.getOps().get(0);
ParsedStringTemplate parsed1 = stmtDef1.getParsed().orElseThrow();
ParsedTemplateString parsed1 = stmtDef1.getParsed().orElseThrow();
assertThat(parsed1.getMissing()).containsExactly();
assertThat(parsed1.hasError()).isFalse();
}
@ -56,12 +56,12 @@ public class ParsedOpTemplateTest {
OpsBlock block2 = doclist.getStmtDocs().get(0).getBlocks().get(2);
OpTemplate stmtDef0 = block2.getOps().get(0);
ParsedStringTemplate parsed0 = stmtDef0.getParsed().orElseThrow();
ParsedTemplateString parsed0 = stmtDef0.getParsed().orElseThrow();
assertThat(parsed0.getMissing()).isEmpty();
assertThat(parsed0.hasError()).isFalse();
OpTemplate stmtDef1 = block2.getOps().get(1);
ParsedStringTemplate parsed1 = stmtDef1.getParsed().orElseThrow();
ParsedTemplateString parsed1 = stmtDef1.getParsed().orElseThrow();
assertThat(parsed1.getMissing()).isEmpty();
assertThat(parsed1.hasError()).isFalse();
}

5
virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/BindType.java → virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedSpanType.java
View File

@ -19,7 +19,7 @@ package io.nosqlbench.engine.api.templating;
/**
* The type of a parsed template depends on the structure of the bindings provided.
*/
public enum BindType {
public enum ParsedSpanType {
/**
* A literal template is one which has no bindings that need to be provided to render a specific statement.
@ -32,7 +32,7 @@ public enum BindType {
* A bindref template is one which has only a single bind point and no leading or trailing text.
* It represents a single value which is to be injected, with no clear indication as to whether the
* value should be in string form or not. These are used when referencing objects by bind point name.
* Callers which use rawbind templates where Strings are needed should convert them with {@link Object#toString()}}
* Callers which use rawbind templates where Strings are needed should convert them with {@link Object#toString()}
* Example: <em>{@code {myvalue}}</em>
*/
bindref,
@ -40,6 +40,7 @@ public enum BindType {
/**
* A string template is one which is neither a literal template nor a bindref template. This includes
* any template which has any amount of literal text and any template with more than one bind point.
* Example: <em>{@code Four {timeunit1} and several {timeunit2} ago}</em>
*/
concat

10
virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedTemplateList.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,6 +16,8 @@
package io.nosqlbench.engine.api.templating;
import io.nosqlbench.virtdata.core.templates.CapturePoint;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -25,6 +27,7 @@ public class ParsedTemplateList implements LongFunction<List<?>> {
private final List<Object> protolist = new ArrayList<>();
private final int[] dynamic_idx;
private final LongFunction<?>[] functions;
private final List<CapturePoint> captures = new ArrayList<>();
public ParsedTemplateList(List<Object> sublist, Map<String, String> bindings, List<Map<String, Object>> cfgsources) {
@ -34,6 +37,7 @@ public class ParsedTemplateList implements LongFunction<List<?>> {
for (int i = 0; i < sublist.size(); i++) {
Object item = sublist.get(i);
Templatizer.Result result = Templatizer.make(bindings, item, null, cfgsources);
this.captures.addAll(result.getCaptures());
switch (result.getType()) {
case literal:
protolist.add(result.getValue());
@ -64,4 +68,8 @@ public class ParsedTemplateList implements LongFunction<List<?>> {
public boolean isStatic() {
return dynamic_idx.length==0;
}
public List<CapturePoint> getCaptures() {
return this.captures;
}
}

35
virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedTemplateMap.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -30,7 +30,7 @@ import io.nosqlbench.virtdata.core.bindings.DataMapper;
import io.nosqlbench.virtdata.core.bindings.VirtData;
import io.nosqlbench.virtdata.core.templates.BindPoint;
import io.nosqlbench.virtdata.core.templates.CapturePoint;
import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
import io.nosqlbench.virtdata.core.templates.StringBindings;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -76,7 +76,7 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
* representation of a result. If the values are defined, then each one represents the name
* that the found value should be saved as instead of the original name.
*/
private final List<List<CapturePoint>> captures = new ArrayList<>();
private final List<CapturePoint> captures = new ArrayList<>();
private final int mapsize;
/**
@ -103,13 +103,13 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
this.specmap = map;
this.bindings = bindings;
map.forEach((k, v) -> {
if (v instanceof CharSequence) {
ParsedStringTemplate pt = ParsedStringTemplate.of(((CharSequence) v).toString(), bindings);
this.captures.add(pt.getCaptures());
if (v instanceof CharSequence charvalue) {
ParsedTemplateString pt = ParsedTemplateString.of(charvalue.toString(), bindings);
this.captures.addAll(pt.getCaptures());
switch (pt.getType()) {
case literal:
statics.put(k, ((CharSequence) v).toString());
protomap.put(k, ((CharSequence) v).toString());
statics.put(k, charvalue.toString());
protomap.put(k, charvalue.toString());
break;
case bindref:
String spec = pt.asBinding().orElseThrow().getBindspec();
@ -126,14 +126,15 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
protomap.put(k, null);
break;
}
} else if (v instanceof Map) {
((Map) v).keySet().forEach(smk -> {
} else if (v instanceof Map mapvalue) {
mapvalue.keySet().forEach(smk -> {
if (!CharSequence.class.isAssignableFrom(smk.getClass())) {
throw new OpConfigError("Only string keys are allowed in submaps.");
}
});
Map<String, Object> submap = (Map<String, Object>) v;
ParsedTemplateMap subtpl = new ParsedTemplateMap(getName(),submap, bindings, cfgsources);
this.captures.addAll(subtpl.getCaptures());
if (subtpl.isStatic()) {
statics.put(k, submap);
protomap.put(k, submap);
@ -141,9 +142,10 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
dynamics.put(k, subtpl);
protomap.put(k, null);
}
} else if (v instanceof List) {
List<Object> sublist = (List<Object>) v;
} else if (v instanceof List listvalue) {
List<Object> sublist = listvalue;
ParsedTemplateList subtpl = new ParsedTemplateList(sublist, bindings, cfgsources);
this.captures.addAll(subtpl.getCaptures());
if (subtpl.isStatic()) {
statics.put(k, sublist);
protomap.put(k, sublist);
@ -162,6 +164,10 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
}
public List<CapturePoint> getCaptures() {
return this.captures;
}
/**
* @return true if any field of this template map is dynamic
*/
@ -695,11 +701,11 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
return false;
}
public Optional<ParsedStringTemplate> getAsStringTemplate(String fieldname) {
public Optional<ParsedTemplateString> getAsStringTemplate(String fieldname) {
if (specmap.containsKey(fieldname)) {
Object fval = specmap.get(fieldname);
if (fval instanceof CharSequence) {
return Optional.of(new ParsedStringTemplate(fval.toString(), this.bindings));
return Optional.of(new ParsedTemplateString(fval.toString(), this.bindings));
} else {
throw new RuntimeException("Can not make a parsed text template from op template field '" + fieldname + "' of type '" + fval.getClass().getSimpleName() + "'");
}
@ -982,4 +988,5 @@ public class ParsedTemplateMap implements LongFunction<Map<String, ?>>, StaticFi
}
return sb.toString();
}
}

12
virtdata-api/src/main/java/io/nosqlbench/engine/api/templating/Templatizer.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,7 +20,7 @@ import io.nosqlbench.api.errors.OpConfigError;
import io.nosqlbench.virtdata.core.bindings.DataMapper;
import io.nosqlbench.virtdata.core.bindings.VirtData;
import io.nosqlbench.virtdata.core.templates.CapturePoint;
import io.nosqlbench.virtdata.core.templates.ParsedStringTemplate;
import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
import io.nosqlbench.virtdata.core.templates.StringBindings;
import java.util.ArrayList;
@ -36,7 +36,7 @@ public class Templatizer {
result.setName(name);
if (v instanceof CharSequence) {
ParsedStringTemplate pt = ParsedStringTemplate.of(((CharSequence) v).toString(), bindings);
ParsedTemplateString pt = ParsedTemplateString.of(((CharSequence) v).toString(), bindings);
result.addCaptures(pt.getCaptures());
result.setType(pt.getType());
switch (pt.getType()) {
@ -87,7 +87,7 @@ public class Templatizer {
}
public static class Result {
private BindType type;
private ParsedSpanType type;
private final List<CapturePoint> captures = new ArrayList<>();
private String name;
private Object value;
@ -97,7 +97,7 @@ public class Templatizer {
this.type = null;
}
public BindType getType() {
public ParsedSpanType getType() {
return this.type;
}
@ -109,7 +109,7 @@ public class Templatizer {
this.captures.addAll(captures);
}
public void setType(BindType type) {
public void setType(ParsedSpanType type) {
this.type = type;
}

14
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/BindPoint.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -53,7 +53,15 @@ public class BindPoint {
* a definition bindpoint is expressed as anything between double curly braces like <pre>{@code {{Identity()}}</pre>
* or <pre>{@code {(Identity())}}
*/
definition
definition,
/**
* a reference to a captured variable, local to the current thread
*/
localref,
/**
* a reference to a global captured variable, across the local JVM instance
*/
globalref
}
public BindPoint(String anchor, String bindspec, Type type) {
@ -84,7 +92,7 @@ public class BindPoint {
}
/**
* The name of the anchor for a binding as it appears in a user-specified template, parsed by {@link ParsedStringTemplate}.
* The name of the anchor for a binding as it appears in a user-specified template, parsed by {@link ParsedTemplateString}.
* @return A string name for the bind point anchor, or null for {@link BindPoint.Type#definition} types.
*/
public String getAnchor() {

32
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/ParsedStringTemplate.java → virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/ParsedTemplateString.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,7 +17,7 @@
package io.nosqlbench.virtdata.core.templates;
import io.nosqlbench.engine.api.templating.BindType;
import io.nosqlbench.engine.api.templating.ParsedSpanType;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -74,7 +74,7 @@ import java.util.stream.StreamSupport;
* <LI>provide a text template for re-assembly with injected data</LI>
* </UL>
*
* Once the parsed template is constructed, the method {@link ParsedStringTemplate#orError()}
* Once the parsed template is constructed, the method {@link ParsedTemplateString#orError()}
* should <em>always</em> called before it is used.
*
* <H2>Validity Checks</H2>
@ -106,16 +106,16 @@ import java.util.stream.StreamSupport;
* This is a list of binding names which were provided by the user, but which were not used in the raw template by name.
* </p>
*/
public class ParsedStringTemplate {
public class ParsedTemplateString {
private final static Logger logger = LogManager.getLogger(ParsedStringTemplate.class);
private final static Logger logger = LogManager.getLogger(ParsedTemplateString.class);
private final String rawtemplate;
private final List<CapturePoint> captures;
private final List<CapturePoint> captures = new ArrayList<>();
private final List<BindPoint> bindpoints;
public static ParsedStringTemplate of(String rawtemplate, Map<String, String> bindings) {
return new ParsedStringTemplate(rawtemplate, bindings);
public static ParsedTemplateString of(String rawtemplate, Map<String, String> bindings) {
return new ParsedTemplateString(rawtemplate, bindings);
}
/**
@ -130,19 +130,19 @@ public class ParsedStringTemplate {
private final Map<String, String> bindings = new LinkedHashMap<>();
/**
* Parse the given raw template, check the bind points against the provide bindings, and
* Parse the given raw template, check the bind points against the provided bindings, and
* provide detailed template checks for validity.
*
* @param rawtemplate A string template which contains optionally embedded named anchors
* @param availableBindings The bindings which are provided by the user to fulfill the named anchors in this raw template
*/
public ParsedStringTemplate(String rawtemplate, Map<String, String> availableBindings) {
public ParsedTemplateString(String rawtemplate, Map<String, String> availableBindings) {
this.bindings.putAll(availableBindings);
this.rawtemplate = rawtemplate;
CapturePointParser capturePointParser = new CapturePointParser();
CapturePointParser.Result captureData = capturePointParser.apply(rawtemplate);
this.captures = captureData.getCaptures();
this.captures.addAll(captureData.getCaptures());
BindPointParser bindPointParser = new BindPointParser();
BindPointParser.Result bindPointsResult = bindPointParser.apply(captureData.getRawTemplate(), availableBindings);
@ -150,17 +150,17 @@ public class ParsedStringTemplate {
this.bindpoints = bindPointsResult.getBindpoints();
}
public BindType getType() {
public ParsedSpanType getType() {
if (this.spans.length == 1) {
return BindType.literal;
return ParsedSpanType.literal;
} else if (this.spans[0].isEmpty() && this.spans[2].isEmpty()) {
return BindType.bindref;
return ParsedSpanType.bindref;
} else {
return BindType.concat;
return ParsedSpanType.concat;
}
}
public ParsedStringTemplate orError() {
public ParsedTemplateString orError() {
if (hasError()) {
throw new RuntimeException("Unable to parse statement: " + this);
}

8
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindings.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -37,15 +37,15 @@ public class StringBindings implements Binder<String> {
}
public StringBindings(String template, Map<String,String> bindings, Map<String,Object> fconfig) {
ParsedStringTemplate parsed = new ParsedStringTemplate(template,bindings);
ParsedTemplateString parsed = new ParsedTemplateString(template,bindings);
this.compositor = new StringCompositor(parsed, fconfig);
}
public StringBindings(ParsedStringTemplate parsedStringTemplate) {
public StringBindings(ParsedTemplateString parsedStringTemplate) {
this(parsedStringTemplate, Map.of());
}
public StringBindings(ParsedStringTemplate pt, Map<String,Object> fconfig) {
public StringBindings(ParsedTemplateString pt, Map<String,Object> fconfig) {
this.compositor = new StringCompositor(pt,fconfig);
}

6
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringCompositor.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -40,7 +40,7 @@ public class StringCompositor implements LongFunction<String> {
private final Function<Object, String> stringfunc;
public StringCompositor(ParsedStringTemplate template, Map<String,Object> fconfig, Function<Object,String> stringfunc) {
public StringCompositor(ParsedTemplateString template, Map<String,Object> fconfig, Function<Object,String> stringfunc) {
Map<String,Integer> specs = new HashMap<>();
List<BindPoint> bindpoints = template.getBindPoints();
for (BindPoint bindPoint : bindpoints) {
@ -69,7 +69,7 @@ public class StringCompositor implements LongFunction<String> {
bufsize = minsize*2;
}
public StringCompositor(ParsedStringTemplate template, Map<String,Object> fconfig) {
public StringCompositor(ParsedTemplateString template, Map<String,Object> fconfig) {
this(template,fconfig,Object::toString);
}

35
virtdata-api/src/test/java/io/nosqlbench/engine/api/templating/ParsedTemplateMapTest.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.templating;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.assertj.core.api.Assertions.assertThat;
public class ParsedTemplateMapTest {
@Test
public void testParsedTemplateMap() {
ParsedTemplateMap ptm = new ParsedTemplateMap("name1", Map.of("string1", "string2"), Map.of(), List.of());
assertThat(ptm.getOpFieldNames()).isEqualTo(Set.of("string1"));
}
}

39
virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/CapturePointParserTest.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.core.templates;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
public class CapturePointParserTest {
@Test
public void testCapturePoint1() {
CapturePointParser cpp = new CapturePointParser();
CapturePointParser.Result result = cpp.apply("string with [capture1]");
assertThat(result).isEqualTo(
new CapturePointParser.Result(
"string with capture1",
List.of(CapturePoint.of("capture1"))
)
);
}
}

4
virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/FastStringCompositorTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -28,7 +28,7 @@ public class FastStringCompositorTest {
public void testFastStringCompositor() {
String rawTpl = "template {b1}, {{TestValue(5)}}";
Map<String, String> bindings = Map.of("b1", "TestIdentity()");
ParsedStringTemplate ptpl = new ParsedStringTemplate(rawTpl, bindings);
ParsedTemplateString ptpl = new ParsedTemplateString(rawTpl, bindings);
StringCompositor fsc = new StringCompositor(ptpl,Map.of());
System.out.println(fsc);
}

22
virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/ParsedStringTemplateTest.java → virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/ParsedTemplateStringTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,14 +16,14 @@
package io.nosqlbench.virtdata.core.templates;
import io.nosqlbench.engine.api.templating.BindType;
import io.nosqlbench.engine.api.templating.ParsedSpanType;
import org.junit.jupiter.api.Test;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class ParsedStringTemplateTest {
public class ParsedTemplateStringTest {
private final Map<String, String> bindings = Map.of(
"bindname1", "bindspec1",
@ -32,7 +32,7 @@ public class ParsedStringTemplateTest {
@Test
public void testShouldMatchRawLiteral() {
String rawNothing = "This has no anchors";
ParsedStringTemplate pt = new ParsedStringTemplate(rawNothing, bindings);
ParsedTemplateString pt = new ParsedTemplateString(rawNothing, bindings);
assertThat(pt.getSpans()).containsExactly("This has no anchors");
assertThat(pt.getBindPoints()).isEmpty();
assertThat(pt.getMissing()).isEmpty();
@ -41,7 +41,7 @@ public class ParsedStringTemplateTest {
@Test
public void testShouldIgnoreExtraneousAnchors() {
String oneExtraneous = "An {this is an extraneous form} invalid anchor.";
ParsedStringTemplate pt = new ParsedStringTemplate(oneExtraneous, bindings);
ParsedTemplateString pt = new ParsedTemplateString(oneExtraneous, bindings);
assertThat(pt.getSpans()).containsExactly("An {this is an extraneous form} invalid anchor.");
assertThat(pt.getBindPoints()).isEmpty();
assertThat(pt.getMissing()).isEmpty();
@ -50,7 +50,7 @@ public class ParsedStringTemplateTest {
@Test
public void testShouldAllowArbitraryNonGreedyInExtendedBindPoint() {
String oneExtendedBindPoint = "An {{this is an extended form}} {{and another}} invalid anchor.";
ParsedStringTemplate pt = new ParsedStringTemplate(oneExtendedBindPoint, bindings);
ParsedTemplateString pt = new ParsedTemplateString(oneExtendedBindPoint, bindings);
assertThat(pt.getSpans()).containsExactly("An ","this is an extended form"," ","and another"," invalid anchor.");
assertThat(pt.getAnchors()).containsExactly("this is an extended form","and another");
}
@ -58,7 +58,7 @@ public class ParsedStringTemplateTest {
@Test
public void testShouldMatchLiteralVariableOnly() {
String literalVariableOnly = "literal {bindname1}";
ParsedStringTemplate pt = new ParsedStringTemplate(literalVariableOnly, bindings);
ParsedTemplateString pt = new ParsedTemplateString(literalVariableOnly, bindings);
assertThat(pt.getSpans()).containsExactly("literal ", "bindname1", "");
assertThat(pt.getAnchors()).containsOnly("bindname1");
assertThat(pt.getMissing()).isEmpty();
@ -67,7 +67,7 @@ public class ParsedStringTemplateTest {
@Test
public void testShouldMatchVariableLiteralOnly() {
String variableLiteralOnly = "{bindname2} literal";
ParsedStringTemplate pt = new ParsedStringTemplate(variableLiteralOnly, bindings);
ParsedTemplateString pt = new ParsedTemplateString(variableLiteralOnly, bindings);
assertThat(pt.getSpans()).containsExactly("", "bindname2", " literal");
assertThat(pt.getAnchors()).containsOnly("bindname2");
assertThat(pt.getMissing()).isEmpty();
@ -76,7 +76,7 @@ public class ParsedStringTemplateTest {
@Test
public void testPositionalExpansionShouldBeValid() {
String multi = "A {bindname1} of {bindname2} sort.";
ParsedStringTemplate pt = new ParsedStringTemplate(multi, bindings);
ParsedTemplateString pt = new ParsedTemplateString(multi, bindings);
assertThat(pt.getSpans()).containsExactly("A ", "bindname1", " of ", "bindname2", " sort.");
assertThat(pt.getAnchors()).containsOnly("bindname1", "bindname2");
assertThat(pt.getMissing()).isEmpty();
@ -91,11 +91,11 @@ public class ParsedStringTemplateTest {
@Test
public void shouldMatchBasicCapturePoint() {
ParsedStringTemplate pt = new ParsedStringTemplate(
ParsedTemplateString pt = new ParsedTemplateString(
"select [u],[v as v1] from users where userid={userid}", Map.of("userid", "NumberNameToString()")
);
assertThat(pt.getAnchors()).containsExactly("userid");
assertThat(pt.getType()).isEqualTo(BindType.concat);
assertThat(pt.getType()).isEqualTo(ParsedSpanType.concat);
assertThat(pt.getCaptures()).containsExactly(CapturePoint.of("u"),CapturePoint.of("v","v1"));
}

6
virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/StringCompositorTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -26,13 +26,13 @@ public class StringCompositorTest {
@Test
public void testShouldMatchSpanOnly() {
ParsedStringTemplate pt = new ParsedStringTemplate("A\\{ {one}two", Map.of());
ParsedTemplateString pt = new ParsedTemplateString("A\\{ {one}two", Map.of());
assertThat(pt.getSpans()).containsExactly("A\\{ ", "one", "two");
}
@Test
public void testShouldNotMatchEscaped() {
ParsedStringTemplate pt = new ParsedStringTemplate("A\\{{B}C",Map.of());
ParsedTemplateString pt = new ParsedTemplateString("A\\{{B}C",Map.of());
assertThat(pt.getSpans()).containsExactly("A\\{","B","C");
}

10
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_charbuffer/CharBufferExtract.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -50,12 +50,12 @@ public class CharBufferExtract implements LongFunction<CharBuffer> {
public CharBufferExtract(Object initFunc, Object sizeFunc) {
CharBuffer image = null;
if (initFunc instanceof Number) {
int bufsize = ((Number) initFunc).intValue();
if (initFunc instanceof Number number) {
int bufsize = number.intValue();
this.image = new CharBufImage(bufsize).apply(1L);
} else {
LongFunction<String> bbfunc = VirtDataConversions.adaptFunction(initFunc, LongFunction.class, String.class);
this.image = CharBuffer.wrap(bbfunc.apply(0));
LongFunction<String> stringFunc = VirtDataConversions.adaptFunction(initFunc, LongFunction.class, String.class);
this.image = CharBuffer.wrap(stringFunc.apply(0));
}
this.imgsize = this.image.limit();