IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

partial variable capture work

Browse Source
This commit is contained in:
Jonathan Shook 2024-03-15 13:04:32 -05:00
parent 0d0efd12d6
commit f2f6649c93
12 changed files with 336 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4CqlOp.java
View File

@ -29,6 +29,7 @@ import io.nosqlbench.adapter.cqld4.exceptions.ExceededRetryReplaceException;
import io.nosqlbench.adapter.cqld4.exceptions.UnexpectedPagingException;
import io.nosqlbench.adapter.cqld4.instruments.CqlOpMetrics;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.*;
import io.nosqlbench.virtdata.core.templates.CapturePoint;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -49,7 +50,7 @@ import java.util.concurrent.*;
// TODO: add rows histogram resultSetSizeHisto
public abstract class Cqld4CqlOp implements CycleOp<List<Row>>, VariableCapture, OpGenerator, OpResultSize {
public abstract class Cqld4CqlOp implements CycleOp<List<Row>>, VariableCapture<List<Row>>, OpGenerator, OpResultSize {
private final static Logger logger = LogManager.getLogger(Cqld4CqlOp.class);
private final CqlSession session;
@ -165,8 +166,10 @@ public abstract class Cqld4CqlOp implements CycleOp<List<Row>>, VariableCapture,
return next;
}
public Map<String, ?> capture() {
throw new NotImplementedException("Not implemented for Cqld4CqlOp");
@Override
public List<?> capture(List<Row> input, List<CapturePoint> capturePoints) {
}
public abstract Statement<?> getStmt();

11
adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optypes/Cqld4RainbowTableOp.java
View File

@ -22,7 +22,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.*;
import java.util.Map;
// Need to create RainbowTableStatement
public class Cqld4RainbowTableOp implements CycleOp<ResultSet>, VariableCapture, OpGenerator, OpResultSize {
public class Cqld4RainbowTableOp implements CycleOp<ResultSet>, OpGenerator, OpResultSize {
// private final CqlSession session;
// private final RainbowTableStatement stmt;
@ -38,13 +38,4 @@ public class Cqld4RainbowTableOp implements CycleOp<ResultSet>, VariableCapture,
throw new RuntimeException("implement me");
}
@Override
public Map<String, ?> capture() {
throw new RuntimeException("implement me");
}
//
// public Cqld4RainbowTableOp(CqlSession session, RainbowTableStatement stmt, int maxpages, boolean retryreplace) {
// //
// }
}

26
adapters-api/src/main/java/io/nosqlbench/adapters/api/activityimpl/uniform/flowtypes/VariableCapture.java
View File

@ -16,9 +16,11 @@
package io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes;
import io.nosqlbench.virtdata.core.templates.CapturePoint;
import io.nosqlbench.virtdata.core.templates.ParsedTemplateString;
import io.nosqlbench.virtdata.library.basics.core.threadstate.SharedState;
import java.util.Map;
import java.util.List;
/**
* If an op implements VariableCapture, then it is known to be able to
@ -29,6 +31,24 @@ import java.util.Map;
* and to allow for auto documentation tha the feature is supported for
* a given adapter.
*/
public interface VariableCapture {
Map<String,?> capture();
public interface VariableCapture<I> {
List<?> capture(I input, List<CapturePoint> capturePoints);
default void applyCaptures(List<CapturePoint> capturePoints, List<?> values) {
for (int i = 0; i < capturePoints.size(); i++) {
CapturePoint cp = capturePoints.get(i);
String storeAs = cp.getStoredName();
CapturePoint.Scope storeScope = cp.getStoredScope();
Class<?> storedType = cp.getStoredType();
switch (storeScope) {
case stanza, container, thread ->
SharedState.tl_ObjectMap.get().put(storeAs, storedType.cast(values.get(i)));
case session ->
SharedState.gl_ObjectMap.put(storeAs, storedType.cast(values.get(i)));
}
}
}
}

17
engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/SimpleActivity.java
View File

@ -56,6 +56,7 @@ import io.nosqlbench.adapters.api.activityimpl.uniform.DryRunOpDispenserWrapper;
import io.nosqlbench.adapters.api.activityimpl.uniform.decorators.SyntheticOpTemplateProvider;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import io.nosqlbench.virtdata.core.templates.CapturePoint;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -451,6 +452,22 @@ public class SimpleActivity extends NBStatusComponent implements Activity, Invok
OpMapper<? extends Op> opMapper = adapter.getOpMapper();
OpDispenser<? extends Op> dispenser = opMapper.apply(pop);
List<CapturePoint> captures = pop.getCaptures();
if (!captures.isEmpty()) {
logger.debug("Creating test op for variable capture configuration (for " + pop.getName() + ")");
Op op = dispenser.apply(0);
if (op instanceof VariableCapture cap) {
if (op instanceof CycleOp<?> cycleOp) {
dispenser = new VarCapOpDispenserWrapper((DriverAdapter<Op,Object>) adapter, pop,
dispenser);
} else {
throw new RuntimeException("Unable to wrap op dispenser for variable capture because of " +
"the core op implementation does not return a result. Use CycleOp for this.");
}
}
}
String dryrunSpec = pop.takeStaticConfigOr("dryrun", "none");
if ("op".equalsIgnoreCase(dryrunSpec)) {
dispenser = new DryRunOpDispenserWrapper((DriverAdapter<Op,Object>)adapter, pop, dispenser);

43
engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/varcap/VarCapCycleOp.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityimpl.varcap;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.VariableCapture;
import io.nosqlbench.virtdata.core.templates.CapturePoint;
import java.util.List;
public class VarCapCycleOp<T> implements CycleOp<T> {
private final CycleOp<T> realOp;
private final List<CapturePoint> capturePointList;
public VarCapCycleOp(CycleOp<T> op, List<CapturePoint> varcaps) {
this.realOp = op;
this.capturePointList = varcaps;
}
@Override
public T apply(long value) {
T result = realOp.apply(value);
VariableCapture<T> capturer = ((VariableCapture<T>) this);
List<?> captured = capturer.capture(result, capturePointList);
capturer.applyCaptures(capturePointList,captured);
return result;
}
}

56
engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/varcap/VarCapOpDispenserWrapper.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.engine.api.activityimpl.varcap;
import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser;
import io.nosqlbench.adapters.api.activityimpl.OpDispenser;
import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.CycleOp;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op;
import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.VariableCapture;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import io.nosqlbench.virtdata.core.templates.CapturePoint;
import java.util.List;
import java.util.function.LongFunction;
public class VarCapOpDispenserWrapper extends BaseOpDispenser<Op, Object> {
private final OpDispenser<? extends Op> realDispenser;
private final LongFunction<CycleOp<?>> opFunc;
private final List<CapturePoint> capturePoints;
public VarCapOpDispenserWrapper(DriverAdapter<Op, Object> adapter, ParsedOp pop, OpDispenser<? extends Op> dispenser) {
super(adapter, pop);
this.realDispenser = dispenser;
this.capturePoints = pop.getCaptures();
Op exampleOp = realDispenser.apply(0L);
if (exampleOp instanceof CycleOp<?> cop) {
opFunc = l -> new VarCapCycleOp((CycleOp<?>)realDispenser.apply(l), capturePoints);
} else {
throw new RuntimeException("Invalid type for varcap:" + exampleOp.getClass().getCanonicalName());
}
}
@Override
public CycleOp<?> apply(long value) {
return opFunc.apply(value);
}
}

131
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/CapturePoint.java
View File

@ -19,28 +19,39 @@ package io.nosqlbench.virtdata.core.templates;
import java.util.Objects;
/**
* A capture point is a named variable which should be extracted from a payload or result type
* <p>A capture point is a named variable which should be extracted from a payload or result type
* using a native driver API. The result is meant to be provided to the NoSQLBench runtime
* during cycle execution, and stored in a scoped context of variables which can be re-used within
* other operations.
* </p>
* <p>
* <hr/>
*
* <H2>Format</H2>
*
* <pre>{@code
* select [username as u1] from users where userid={userid};
* }</pre>
*
* <p>
* In the example above, the span <em>[username as u1]</em> is recognized as a capture point.
* The name of the variable to be captured is <em>username</em>. It is to be captured under
* a different variable name <em>u1</em>.
* </p>
*
* <p>
* If the name is the same in both cases, i.e. the variable is named in the result as it
* should be known after extraction, then you can elide the <em>as u1</em> clause as in this example:
*
* <pre>{@code
* select [username] from users where userid={userid};
* }</pre>
* </p>
*
* <p>
* The scope of the captured
* </p>
*
* <p>
* During op mapping, any capture points are condensed down to the native driver vernacular by
* removing the square brackets from the op template. Thus, the result of parsing the above would
* yield a form compatible with a native driver. For example, converting to prepared statement form
@ -49,62 +60,142 @@ import java.util.Objects;
* <pre>{@code
* select username from users where userid=:userid
* }</pre>
*
* <p>
* For details on the <em>{userid}</em> form, see {@link BindPoint}
*/
public class CapturePoint {
private final String name;
private final String asName;
/**
* The name of the capture point as known by the original protocol or driver API
*/
private final String capturedName;
public CapturePoint(String name, String asName) {
this.name = name;
this.asName = asName;
/**
* The name that the captured value should be stored as
*/
private final String storedName;
private final Scope storedScope;
private final Class<?> storedType;
private final Class<?> elementType;
protected CapturePoint(
String capturedName,
String storedName,
Scope storedScope,
Class<?> storedType,
Class<?> elementType
) {
this.capturedName = capturedName;
this.storedName = storedName;
this.storedScope = storedScope;
this.storedType = storedType;
this.elementType = elementType;
}
public String getName() {
return name;
public String getCapturedName() {
return capturedName;
}
public String getStoredName() {
return storedName;
}
public Scope getStoredScope() {
return this.storedScope;
}
/**
* Create a CapturePoint with the specified anchorName, and an optional aliasName.
* If aliasName is null, then the anchorName is used as the alias.
*
* @param anchorName The name of the capture variable in the native form
* @param aliasName The name of the captured value as seen by consumers
* @param capturedName
* The name of the capture variable in the native form
* @param storedName
* The name of the captured value as seen by consumers
* @return A new CapturePoint
*/
public static CapturePoint of(String anchorName, String aliasName) {
Objects.requireNonNull(anchorName);
return new CapturePoint(anchorName, aliasName == null ? anchorName : aliasName);
public static CapturePoint of(String capturedName, String storedName, Scope scope, Class<?> storedType,
Class<?> elementType) {
Objects.requireNonNull(capturedName);
return new CapturePoint(capturedName, storedName == null ? capturedName : storedName, scope, storedType,
elementType);
}
/**
* Create a CapturePoint with the specified anchorName, and the same aliasName.
*
* @param anchorName The name of the capture variable in the native form and as seen by consumers.
* @param anchorName
* The name of the capture variable in the native form and as seen by consumers.
* @return A new CapturePoint
*/
public static CapturePoint of(String anchorName) {
Objects.requireNonNull(anchorName);
return new CapturePoint(anchorName, anchorName);
return new CapturePoint(anchorName, anchorName, Scope.stanza, Object.class, null);
}
@Override
public String toString() {
return "[" + getName() + (name.equals(asName) ? "" : " as " + asName) + "]";
StringBuilder sb = new StringBuilder();
sb.append("[").append(capturedName);
if (!capturedName.equals(storedName)) {
sb.append(" as:").append(storedName);
}
sb.append(" scope:").append(storedScope.name());
if (storedType != Object.class) {
sb.append(" type:").append(storedType.getCanonicalName());
}
sb.append("]");
return sb.toString();
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
CapturePoint that = (CapturePoint) o;
return Objects.equals(name, that.name) && Objects.equals(asName, that.asName);
if (!capturedName.equals(that.capturedName)) return false;
if (!storedName.equals(that.storedName)) return false;
if (storedScope != that.storedScope) return false;
return storedType.equals(that.storedType);
}
@Override
public int hashCode() {
return Objects.hash(name, asName);
int result = capturedName.hashCode();
result = 31 * result + storedName.hashCode();
result = 31 * result + storedScope.hashCode();
result = 31 * result + storedType.hashCode();
return result;
}
public Class<?> getStoredType() {
return this.storedType;
}
public static enum Scope {
/**
* The stanza scope includes the op sequence, but not the next iteration of an op sequence
*/
stanza,
/**
* Thread scope is equivalent to thread-local, although it may be implemented using a different mechanism with
* virtual threads
*/
thread,
/**
* Container scope is limited to the container within which an op is executed
*/
container,
/**
* Session scope includes all containers, threads, and stanzas within a session
*/
session
}
}

68
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/CapturePointParser.java
View File

@ -23,34 +23,70 @@ import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class CapturePointParser implements Function<String, CapturePointParser.Result> {
public class CapturePointParser implements Function<String, CapturePointParser.ParsedCapturePoint> {
public final static Pattern CAPTUREPOINT_PATTERN = Pattern.compile(
"(\\[(?<capture>\\w+[-_\\d\\w.]*)(\\s+[aA][sS]\\s+(?<alias>\\w+[-_\\d\\w.]*))?])"
public final static Pattern CAPTURE_PARAM_PATTERN = Pattern.compile(
"(\\s*(?<param>as|scope|type)\\s*:\\s*(?<value>[-_.a-zA-Z0-9<>]+))"
);
public final static Pattern CAPTURE_POINT_PATTERN = Pattern.compile(
"\\[\\s*(?<name>[a-z][_a-zA-Z0-9]*)(?<params>" + CAPTURE_PARAM_PATTERN.pattern() + "+)*]"
);
@Override
public Result apply(String template) {
public ParsedCapturePoint apply(String template) {
StringBuilder raw = new StringBuilder();
Matcher m = CAPTUREPOINT_PATTERN.matcher(template);
List<CapturePoint> captures = new ArrayList<>();
Matcher m = CAPTURE_POINT_PATTERN.matcher(template);
List<io.nosqlbench.virtdata.core.templates.CapturePoint> captures = new ArrayList<>();
while (m.find()) {
CapturePoint captured = CapturePoint.of(m.group("capture"), m.group("alias"));
String captureName = m.group("name");
String captureParams = m.group("params");
String storedName = captureName;
Class<?> storedClass = Object.class;
Class<?> elementType = null;
io.nosqlbench.virtdata.core.templates.CapturePoint.Scope storedScope = io.nosqlbench.virtdata.core.templates.CapturePoint.Scope.stanza;
if (captureParams != null) {
Matcher pfinder = CAPTURE_PARAM_PATTERN.matcher(captureParams);
while (pfinder.find()) {
String param = pfinder.group("param");
String value = pfinder.group("value");
switch (param) {
case "as":
storedName = value;
break;
case "scope":
storedScope = io.nosqlbench.virtdata.core.templates.CapturePoint.Scope.valueOf(value);
break;
case "type":
try {
storedClass = Class.forName(value);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
break;
}
}
}
io.nosqlbench.virtdata.core.templates.CapturePoint captured =
io.nosqlbench.virtdata.core.templates.CapturePoint.of(captureName, storedName, storedScope, storedClass,
null);
captures.add(captured);
m.appendReplacement(raw,captured.getName());
m.appendReplacement(raw, captured.getCapturedName());
}
m.appendTail(raw);
return new Result(raw.toString(),captures);
return new ParsedCapturePoint(raw.toString(), captures);
}
public final static class Result {
public final static class ParsedCapturePoint {
private final String rawTemplate;
private final List<CapturePoint> captures;
public Result(String rawTemplate, List<CapturePoint> captures) {
private final List<io.nosqlbench.virtdata.core.templates.CapturePoint> captures;
public ParsedCapturePoint(String rawTemplate, List<io.nosqlbench.virtdata.core.templates.CapturePoint> captures) {
this.rawTemplate = rawTemplate;
this.captures = captures;
}
@ -59,7 +95,7 @@ public class CapturePointParser implements Function<String, CapturePointParser.R
return this.rawTemplate;
}
public List<CapturePoint> getCaptures() {
public List<io.nosqlbench.virtdata.core.templates.CapturePoint> getCaptures() {
return this.captures;
}
@ -67,8 +103,8 @@ public class CapturePointParser implements Function<String, CapturePointParser.R
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Result result = (Result) o;
return Objects.equals(rawTemplate, result.rawTemplate) && Objects.equals(captures, result.captures);
ParsedCapturePoint capturePoint = (ParsedCapturePoint) o;
return Objects.equals(rawTemplate, capturePoint.rawTemplate) && Objects.equals(captures, capturePoint.captures);
}
@Override

2
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/ParsedTemplateString.java
View File

@ -141,7 +141,7 @@ public class ParsedTemplateString {
this.rawtemplate = rawtemplate;
CapturePointParser capturePointParser = new CapturePointParser();
CapturePointParser.Result captureData = capturePointParser.apply(rawtemplate);
CapturePointParser.ParsedCapturePoint captureData = capturePointParser.apply(rawtemplate);
this.captures.addAll(captureData.getCaptures());
BindPointParser bindPointParser = new BindPointParser();

22
virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/CapturePointParserTest.java
View File

@ -27,13 +27,27 @@ public class CapturePointParserTest {
@Test
public void testCapturePoint1() {
CapturePointParser cpp = new CapturePointParser();
CapturePointParser.Result result = cpp.apply("string with [capture1]");
assertThat(result).isEqualTo(
new CapturePointParser.Result(
CapturePointParser.ParsedCapturePoint capturePoint = cpp.apply("string with [capture1]");
assertThat(capturePoint).isEqualTo(
new CapturePointParser.ParsedCapturePoint(
"string with capture1",
List.of(CapturePoint.of("capture1"))
List.of(io.nosqlbench.virtdata.core.templates.CapturePoint.of("capture1"))
)
);
}
@Test
public void testCaptureStoredName() {
CapturePointParser cpp = new CapturePointParser();
CapturePointParser.ParsedCapturePoint capturePoint = cpp.apply("string with [capture1 as:captured-as-name scope:stanza]");
assertThat(capturePoint).isEqualTo(
new CapturePointParser.ParsedCapturePoint(
"string with capture1",
List.of(io.nosqlbench.virtdata.core.templates.CapturePoint.of("capture1", "captured-as-name",
io.nosqlbench.virtdata.core.templates.CapturePoint.Scope.stanza, Object.class, null))
)
);
}
}

9
virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/CapturePointTest.java
View File

@ -28,10 +28,11 @@ public class CapturePointTest {
public void testBasicCaptures() {
CapturePointParser cpp = new CapturePointParser();
assertThat(cpp.apply("test [point1] [point2 as alias3]")).isEqualTo(
new CapturePointParser.Result("test point1 point2",
new CapturePointParser.ParsedCapturePoint("test point1 point2",
List.of(
CapturePoint.of("point1"),
CapturePoint.of("point2","alias3")
io.nosqlbench.virtdata.core.templates.CapturePoint.of("point1"),
io.nosqlbench.virtdata.core.templates.CapturePoint.of("point2","alias3",
io.nosqlbench.virtdata.core.templates.CapturePoint.Scope.stanza,Object.class, null)
))
);
}
@ -40,7 +41,7 @@ public class CapturePointTest {
public void testBypass() {
CapturePointParser cpp = new CapturePointParser();
assertThat(cpp.apply("")).isEqualTo(
new CapturePointParser.Result("", List.of())
new CapturePointParser.ParsedCapturePoint("", List.of())
);
}

5
virtdata-api/src/test/java/io/nosqlbench/virtdata/core/templates/ParsedTemplateStringTest.java
View File

@ -92,11 +92,12 @@ public class ParsedTemplateStringTest {
@Test
public void shouldMatchBasicCapturePoint() {
ParsedTemplateString pt = new ParsedTemplateString(
"select [u],[v as v1] from users where userid={userid}", Map.of("userid", "NumberNameToString()")
"select [u],[v as:v1] from users where userid={userid}", Map.of("userid", "NumberNameToString()")
);
assertThat(pt.getAnchors()).containsExactly("userid");
assertThat(pt.getType()).isEqualTo(ParsedSpanType.concat);
assertThat(pt.getCaptures()).containsExactly(CapturePoint.of("u"),CapturePoint.of("v","v1"));
assertThat(pt.getCaptures()).containsExactly(CapturePoint.of("u"),CapturePoint.of("v","v1",
CapturePoint.Scope.stanza,Object.class,null));
}