IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

test classpath and loader fixes

Browse Source
This commit is contained in:
Jonathan Shook
2022-07-13 10:30:32 -05:00
parent 50e617233f
commit 0dc7a2f157
10 changed files with 268 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlKeyspace.java
View File

@@ -23,6 +23,7 @@ import java.util.Map;
public class CqlKeyspace implements Labeled {
String keyspaceName= "";
String refddl;
private String refReplDdl;
public CqlKeyspace() {
}
@@ -57,4 +58,12 @@ public class CqlKeyspace implements Labeled {
"keyspace", keyspaceName
);
}
public void setRefReplDdl(String refReplDdl) {
this.refReplDdl=refReplDdl;
}
public String getRefDdlWithReplFields(String replFields) {
return refddl.replace(refReplDdl,replFields);
}
}

41
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModel.java
View File

@@ -23,13 +23,12 @@ public class CqlModel {
private final Supplier<List<String>> errors;
Map<String, CqlKeyspace> keyspaces = new LinkedHashMap<>();
Map<String, Map<String, CqlTable>> tables = new LinkedHashMap<>();
Map<String, Map<String, CqlType>> types = new LinkedHashMap<>();
transient
CqlKeyspace keyspace = null;
transient
CqlTable table;
transient CqlKeyspace keyspace = null;
transient CqlTable table;
transient CqlType udt;
public CqlModel(Supplier<List<String>> errorSource) {
@@ -118,4 +117,36 @@ public class CqlModel {
public void addClusteringColumn(String ccolumn) {
table.addClusteringColumn(ccolumn);
}
public void setReplicationText(String repldata) {
keyspace.setRefReplDdl(repldata);
}
public void newType() {
udt = new CqlType();
}
public void addTypeField(String name, String typedef) {
udt.addField(name, typedef);
}
public void saveType(String keyspace, String name, String refddl) {
udt.setKeyspace(keyspace);
udt.setRefddl(refddl);
udt.setName(name);
Map<String, CqlType> ksTypes = this.types.computeIfAbsent(keyspace, ks -> new LinkedHashMap<>());
ksTypes.put(udt.getName(),udt);
udt=null;
}
public List<CqlType> getTypes() {
ArrayList<CqlType> list = new ArrayList<>();
for (Map<String, CqlType> cqlTypesByKeyspace : types.values()) {
for (CqlType cqlType : cqlTypesByKeyspace.values()) {
list.add(cqlType);
}
}
return list;
}
}

48
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModelBuilder.java
View File

@@ -59,6 +59,12 @@ public class CqlModelBuilder extends CqlParserBaseListener {
);
}
@Override
public void exitReplicationList(CqlParser.ReplicationListContext ctx) {
String repldata = textOf(ctx);
model.setReplicationText(repldata);
}
@Override
public void enterCreateTable(CqlParser.CreateTableContext ctx) {
model.newTable();
@@ -74,11 +80,15 @@ public class CqlModelBuilder extends CqlParserBaseListener {
if (ctx.singlePrimaryKey()!=null) {
model.addPartitionKey(ctx.singlePrimaryKey().column().getText());
} else if (ctx.compositeKey()!=null) {
for (CqlParser.PartitionKeyContext pkctx : ctx.compositeKey().partitionKeyList().partitionKey()) {
model.addPartitionKey(pkctx.column().getText());
if (ctx.compositeKey().partitionKeyList()!=null) {
for (CqlParser.PartitionKeyContext pkctx : ctx.compositeKey().partitionKeyList().partitionKey()) {
model.addPartitionKey(pkctx.column().getText());
}
}
for (CqlParser.ClusteringKeyContext ccol : ctx.compositeKey().clusteringKeyList().clusteringKey()) {
model.addClusteringColumn(ccol.column().getText());
if (ctx.compositeKey().clusteringKeyList()!=null) {
for (CqlParser.ClusteringKeyContext ccol : ctx.compositeKey().clusteringKeyList().clusteringKey()) {
model.addClusteringColumn(ccol.column().getText());
}
}
} else if (ctx.compoundKey()!=null) {
model.addClusteringColumn(ctx.compoundKey().partitionKey().column().getText());
@@ -88,6 +98,36 @@ public class CqlModelBuilder extends CqlParserBaseListener {
}
}
@Override
public void enterCreateType(CqlParser.CreateTypeContext ctx) {
model.newType();
}
@Override
public void exitCreateType(CqlParser.CreateTypeContext ctx) {
String keyspace = ctx.keyspace().getText();
String name = ctx.type_().getText();
String refddl = textOf(ctx);
model.saveType(keyspace,name,refddl);
}
// HERE consider building hierarchic type model
@Override
public void exitTypeMemberColumnList(CqlParser.TypeMemberColumnListContext ctx) {
List<CqlParser.ColumnContext> columns = ctx.column();
List<CqlParser.DataTypeContext> dataTypes = ctx.dataType();
for (int idx = 0; idx < columns.size(); idx++) {
model.addTypeField(
columns.get(idx).getText(),
dataTypes.get(idx).getText()
);
}
// dataTypes.get(0).dataType().get(0).dataType().get(0)
}
@Override
public void exitSinglePrimaryKey(CqlParser.SinglePrimaryKeyContext ctx) {
super.exitSinglePrimaryKey(ctx);

16
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlTable.java
View File

@@ -21,6 +21,7 @@ import io.nosqlbench.nb.api.labels.Labeled;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
public class CqlTable implements Labeled {
@@ -28,7 +29,8 @@ public class CqlTable implements Labeled {
String keyspace = "";
List<CqlColumnDef> coldefs = new ArrayList<>();
String refddl;
private String refddl;
List<String> partitionKeys = new ArrayList<>();
List<String> clusteringColumns = new ArrayList<>();
@@ -122,7 +124,15 @@ public class CqlTable implements Labeled {
}
public CqlColumnDef getColumnDefForName(String colname) {
return coldefs.stream().filter(c -> c.getName().equalsIgnoreCase(colname))
.findFirst().orElseThrow();
Optional<CqlColumnDef> def = coldefs
.stream()
.filter(c -> c.getName().equalsIgnoreCase(colname))
.findFirst();
if (!def.isPresent()) {
throw new RuntimeException("Unable to find column definition in table '" +
this.getTableName() + "' for column '" + colname + "'");
}
return def.orElseThrow();
}
}

54
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlType.java Normal file
View File

@@ -0,0 +1,54 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.cqlast;
import java.util.LinkedHashMap;
import java.util.Map;
public class CqlType {
private String keyspace;
private String name;
private String refddl;
private final Map<String,String> fields = new LinkedHashMap<>();
public void setKeyspace(String keyspace) {
this.keyspace = keyspace;
}
public void setName(String name) {
this.name = name;
}
public void setRefddl(String ddl) {
this.refddl = ddl;
}
public String getKeyspace() {
return keyspace;
}
public String getName() {
return this.name;
}
public void addField(String name, String typedef) {
this.fields.put(name, typedef);
}
public Map<String, String> getFields() {
return fields;
}
}

7
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/BindingsAccumulator.java
View File

@@ -17,6 +17,8 @@
package io.nosqlbench.converters.cql.exporters;
import io.nosqlbench.converters.cql.cqlast.CqlColumnDef;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.LinkedHashMap;
import java.util.List;
@@ -24,6 +26,7 @@ import java.util.Map;
import java.util.Optional;
public class BindingsAccumulator {
private final static Logger logger = LogManager.getLogger("CQL-GENERATOR");
private final NamingFolio namer;
private final List<BindingsLibrary> libraries;
@@ -44,7 +47,9 @@ public class BindingsAccumulator {
return newBinding;
}
}
throw new RuntimeException("Unable to find a binding for column def '" + def + "'");
logger.error("Unable to find a binding for column def '" + def + "', installing 'TBD' place-holder until this is supported. This workload will not be functional until then.");
return new Binding(name, "TBD");
// throw new RuntimeException("Unable to find a binding for column def '" + def + "'");
}
private void registerBinding(Binding newBinding) {

3
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlLiteralFormat.java
View File

@@ -34,7 +34,8 @@ public enum CqlLiteralFormat {
DECIMAL,
DOUBLE,
FLOAT,
UNKNOWN;
UNKNOWN,
TIMESTAMP;
private final Function<String, String> literalFormat;
CqlLiteralFormat() {

34
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlWorkloadExporter.java
View File

@@ -48,6 +48,7 @@ import java.util.stream.Collectors;
public class CqlWorkloadExporter {
private final static Logger logger = LogManager.getLogger(CqlWorkloadExporter.class);
public final static String DEFAULT_NAMING_TEMPLATE = "[OPTYPE-][COLUMN-][TYPEDEF-][TABLE!]-[KEYSPACE]";
public static final String DEFAULT_REPLICATION = "\n 'class': 'SimpleStrategy',\n 'replication_factor': 'TEMPLATE(rf:1)'\n";
private final BindingsLibrary defaultBindings = new DefaultCqlBindings();
@@ -120,16 +121,30 @@ public class CqlWorkloadExporter {
namer.populate(model);
Map<String, Object> workload = new LinkedHashMap<>();
workload.put("description", "Auto-generated workload from source schema.");
workload.put("scenarios", genScenarios(model));
workload.put("bindings", bindingsMap);
Map<String, Object> blocks = new LinkedHashMap<>();
workload.put("blocks", blocks);
blocks.put("schema", genSchemaBlock(model));
blocks.put("truncate", genTruncateBlock(model));
blocks.put("rampup", genRampupBlock(model));
blocks.put("main", genMainBlock(model));
bindingsMap.putAll(bindings.getAccumulatedBindings());
return workload;
}
private Map<String, Object> genScenarios(CqlModel model) {
return Map.of(
"default", Map.of(
"schema", "run driver=cql tags=block:schema threads===UNDEF cycles===UNDEF",
"rampup", "run driver=cql tags=block:rampup threads=auto cycles===TEMPLATE(rampup-cycles,10000)",
"main", "run driver=cql tags=block:main threads=auto cycles===TEMPLATE(main-cycles,10000)"
),
"truncate", "run driver=cql tags=block:truncate threads===UNDEF cycles===UNDEF"
);
}
private Map<String, Object> genMainBlock(CqlModel model) {
Map<String, String> mainOpTemplates = new LinkedHashMap<>();
@@ -230,7 +245,8 @@ public class CqlWorkloadExporter {
.setDefaultFlowStyle(FlowStyle.BLOCK)
.setIndent(2)
.setDefaultScalarStyle(ScalarStyle.PLAIN)
.setMaxSimpleKeyLength(100)
.setMaxSimpleKeyLength(1000)
.setWidth(100)
.setSplitLines(true)
.setIndentWithIndicator(true)
.setMultiLineFlow(true)
@@ -258,12 +274,26 @@ public class CqlWorkloadExporter {
return Map.of();
}
private Map<String, Object> genTruncateBlock(CqlModel model) {
Map<String, Object> truncateblock = new LinkedHashMap<>();
Map<String, Object> ops = new LinkedHashMap<>();
truncateblock.put("ops", ops);
for (CqlTable table : model.getAllTables()) {
ops.put(
namer.nameFor(table, "optype", "truncate"),
"truncate " + table.getKeySpace() + "." + table.getTableName() + ";"
);
}
return truncateblock;
}
private Map<String, Object> genSchemaBlock(CqlModel model) {
Map<String, Object> schemablock = new LinkedHashMap<>();
Map<String, Object> ops = new LinkedHashMap<>();
for (CqlKeyspace ks : model.getKeyspaces().values()) {
ops.put("create-keyspace-" + ks.getKeyspaceName(), ks.getRefddl());
ops.put("create-keyspace-" + ks.getKeyspaceName(), ks.getRefDdlWithReplFields(DEFAULT_REPLICATION));
}
for (String ksname : model.getTablesByKeyspace().keySet()) {
for (CqlTable cqltable : model.getTablesByKeyspace().get(ksname).values()) {

13
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/parser/CqlModelParser.java
View File

@@ -19,6 +19,7 @@ package io.nosqlbench.converters.cql.parser;
import io.nosqlbench.converters.cql.cqlast.CQBErrorListener;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlModelBuilder;
import io.nosqlbench.converters.cql.cqlast.CqlType;
import io.nosqlbench.converters.cql.generated.CqlLexer;
import io.nosqlbench.converters.cql.generated.CqlParser;
import org.antlr.v4.runtime.CharStreams;
@@ -30,6 +31,7 @@ import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
public class CqlModelParser {
@@ -45,6 +47,15 @@ public class CqlModelParser {
}
}
public static CqlType parseCqlType(String input) {
CqlModel parsed = parse(input, null);
List<CqlType> types = parsed.getTypes();
if (types.size()!=1) {
throw new RuntimeException("error parsing typedef");
}
return types.get(0);
}
public static CqlModel parse(String input, Path origin) {
try {
@@ -59,7 +70,7 @@ public class CqlModelParser {
parser.addParseListener(cqlModelBuilder);
parser.addErrorListener(errorListener);
CqlParser.RootContext keyspaceParser = parser.root();
parser.root();
CqlModel model = cqlModelBuilder.getModel();
if (model.getErrors().size()>0) {

60
adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/udts/ToUdt.java Normal file
View File

@@ -0,0 +1,60 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.datamappers.functions.udts;
import com.datastax.oss.driver.api.core.data.UdtValue;
import com.datastax.oss.driver.api.core.type.DataType;
import com.datastax.oss.driver.api.core.type.UserDefinedType;
import com.datastax.oss.driver.internal.core.type.UserDefinedTypeBuilder;
import io.nosqlbench.converters.cql.cqlast.CqlType;
import io.nosqlbench.converters.cql.parser.CqlModelParser;
import java.util.function.LongFunction;
/**
* map<A,B>
* where:
* A := (f1 text, f2 text, f3 int)
* AND
* B := (f1 text, f2 text)
*/
public class ToUdt implements LongFunction<UdtValue> {
private final String spec;
private final CqlType typeinfo;
private final UserDefinedType udt;
public ToUdt(String spec) {
this.spec=spec;
typeinfo = CqlModelParser.parseCqlType(spec);
UserDefinedTypeBuilder builder = new UserDefinedTypeBuilder(typeinfo.getKeyspace(), typeinfo.getName());
typeinfo.getFields().forEach((name,typedef) -> {
DataType dataType = resolveDataType(typedef);
builder.withField(name,dataType);
});
this.udt = builder.build();
}
private DataType resolveDataType(String typedef) {
return null;
}
@Override
public UdtValue apply(long value) {
return null;
}
}