diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlKeyspace.java b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlKeyspace.java index e129c3808..f994f6ac6 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlKeyspace.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlKeyspace.java @@ -23,6 +23,7 @@ import java.util.Map; public class CqlKeyspace implements Labeled { String keyspaceName= ""; String refddl; + private String refReplDdl; public CqlKeyspace() { } @@ -57,4 +58,12 @@ public class CqlKeyspace implements Labeled { "keyspace", keyspaceName ); } + + public void setRefReplDdl(String refReplDdl) { + this.refReplDdl=refReplDdl; + } + + public String getRefDdlWithReplFields(String replFields) { + return refddl.replace(refReplDdl,replFields); + } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModel.java b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModel.java index 56cfd86bc..140babf59 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModel.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModel.java @@ -23,13 +23,12 @@ public class CqlModel { private final Supplier> errors; Map keyspaces = new LinkedHashMap<>(); - Map> tables = new LinkedHashMap<>(); + Map> types = new LinkedHashMap<>(); - transient - CqlKeyspace keyspace = null; - transient - CqlTable table; + transient CqlKeyspace keyspace = null; + transient CqlTable table; + transient CqlType udt; public CqlModel(Supplier> errorSource) { @@ -118,4 +117,36 @@ public class CqlModel { public void addClusteringColumn(String ccolumn) { table.addClusteringColumn(ccolumn); } + + public void setReplicationText(String repldata) { + keyspace.setRefReplDdl(repldata); + } + + public void newType() { + udt = new CqlType(); + } + + public void addTypeField(String name, String typedef) { + udt.addField(name, typedef); + } + + public void saveType(String keyspace, String name, String refddl) { + udt.setKeyspace(keyspace); + udt.setRefddl(refddl); + udt.setName(name); + Map ksTypes = this.types.computeIfAbsent(keyspace, ks -> new LinkedHashMap<>()); + ksTypes.put(udt.getName(),udt); + udt=null; + } + + public List getTypes() { + ArrayList list = new ArrayList<>(); + for (Map cqlTypesByKeyspace : types.values()) { + for (CqlType cqlType : cqlTypesByKeyspace.values()) { + list.add(cqlType); + } + } + return list; + + } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModelBuilder.java b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModelBuilder.java index b9c2bd206..c0e5e3f6e 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModelBuilder.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModelBuilder.java @@ -59,6 +59,12 @@ public class CqlModelBuilder extends CqlParserBaseListener { ); } + @Override + public void exitReplicationList(CqlParser.ReplicationListContext ctx) { + String repldata = textOf(ctx); + model.setReplicationText(repldata); + } + @Override public void enterCreateTable(CqlParser.CreateTableContext ctx) { model.newTable(); @@ -74,11 +80,15 @@ public class CqlModelBuilder extends CqlParserBaseListener { if (ctx.singlePrimaryKey()!=null) { model.addPartitionKey(ctx.singlePrimaryKey().column().getText()); } else if (ctx.compositeKey()!=null) { - for (CqlParser.PartitionKeyContext pkctx : ctx.compositeKey().partitionKeyList().partitionKey()) { - model.addPartitionKey(pkctx.column().getText()); + if (ctx.compositeKey().partitionKeyList()!=null) { + for (CqlParser.PartitionKeyContext pkctx : ctx.compositeKey().partitionKeyList().partitionKey()) { + model.addPartitionKey(pkctx.column().getText()); + } } - for (CqlParser.ClusteringKeyContext ccol : ctx.compositeKey().clusteringKeyList().clusteringKey()) { - model.addClusteringColumn(ccol.column().getText()); + if (ctx.compositeKey().clusteringKeyList()!=null) { + for (CqlParser.ClusteringKeyContext ccol : ctx.compositeKey().clusteringKeyList().clusteringKey()) { + model.addClusteringColumn(ccol.column().getText()); + } } } else if (ctx.compoundKey()!=null) { model.addClusteringColumn(ctx.compoundKey().partitionKey().column().getText()); @@ -88,6 +98,36 @@ public class CqlModelBuilder extends CqlParserBaseListener { } } + + @Override + public void enterCreateType(CqlParser.CreateTypeContext ctx) { + model.newType(); + } + + @Override + public void exitCreateType(CqlParser.CreateTypeContext ctx) { + String keyspace = ctx.keyspace().getText(); + String name = ctx.type_().getText(); + String refddl = textOf(ctx); + model.saveType(keyspace,name,refddl); + } + + + // HERE consider building hierarchic type model + @Override + public void exitTypeMemberColumnList(CqlParser.TypeMemberColumnListContext ctx) { + List columns = ctx.column(); + List dataTypes = ctx.dataType(); + for (int idx = 0; idx < columns.size(); idx++) { + model.addTypeField( + columns.get(idx).getText(), + dataTypes.get(idx).getText() + ); + } + +// dataTypes.get(0).dataType().get(0).dataType().get(0) + } + @Override public void exitSinglePrimaryKey(CqlParser.SinglePrimaryKeyContext ctx) { super.exitSinglePrimaryKey(ctx); diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlTable.java b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlTable.java index 173963b41..adfedfcec 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlTable.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlTable.java @@ -21,6 +21,7 @@ import io.nosqlbench.nb.api.labels.Labeled; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; public class CqlTable implements Labeled { @@ -28,7 +29,8 @@ public class CqlTable implements Labeled { String keyspace = ""; List coldefs = new ArrayList<>(); - String refddl; + private String refddl; + List partitionKeys = new ArrayList<>(); List clusteringColumns = new ArrayList<>(); @@ -122,7 +124,15 @@ public class CqlTable implements Labeled { } public CqlColumnDef getColumnDefForName(String colname) { - return coldefs.stream().filter(c -> c.getName().equalsIgnoreCase(colname)) - .findFirst().orElseThrow(); + Optional def = coldefs + .stream() + .filter(c -> c.getName().equalsIgnoreCase(colname)) + .findFirst(); + if (!def.isPresent()) { + throw new RuntimeException("Unable to find column definition in table '" + + this.getTableName() + "' for column '" + colname + "'"); + } + return def.orElseThrow(); } + } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlType.java b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlType.java new file mode 100644 index 000000000..ffe139d19 --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlType.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.converters.cql.cqlast; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class CqlType { + private String keyspace; + private String name; + private String refddl; + private final Map fields = new LinkedHashMap<>(); + + public void setKeyspace(String keyspace) { + this.keyspace = keyspace; + } + public void setName(String name) { + this.name = name; + } + + public void setRefddl(String ddl) { + this.refddl = ddl; + } + + public String getKeyspace() { + return keyspace; + } + + public String getName() { + return this.name; + } + + public void addField(String name, String typedef) { + this.fields.put(name, typedef); + } + + public Map getFields() { + return fields; + } +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/BindingsAccumulator.java b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/BindingsAccumulator.java index 6b5d8b107..b78b7d230 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/BindingsAccumulator.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/BindingsAccumulator.java @@ -17,6 +17,8 @@ package io.nosqlbench.converters.cql.exporters; import io.nosqlbench.converters.cql.cqlast.CqlColumnDef; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.LinkedHashMap; import java.util.List; @@ -24,6 +26,7 @@ import java.util.Map; import java.util.Optional; public class BindingsAccumulator { + private final static Logger logger = LogManager.getLogger("CQL-GENERATOR"); private final NamingFolio namer; private final List libraries; @@ -44,7 +47,9 @@ public class BindingsAccumulator { return newBinding; } } - throw new RuntimeException("Unable to find a binding for column def '" + def + "'"); + logger.error("Unable to find a binding for column def '" + def + "', installing 'TBD' place-holder until this is supported. This workload will not be functional until then."); + return new Binding(name, "TBD"); +// throw new RuntimeException("Unable to find a binding for column def '" + def + "'"); } private void registerBinding(Binding newBinding) { diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlLiteralFormat.java b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlLiteralFormat.java index 6b45ec993..ab19c4ac1 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlLiteralFormat.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlLiteralFormat.java @@ -34,7 +34,8 @@ public enum CqlLiteralFormat { DECIMAL, DOUBLE, FLOAT, - UNKNOWN; + UNKNOWN, + TIMESTAMP; private final Function literalFormat; CqlLiteralFormat() { diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlWorkloadExporter.java b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlWorkloadExporter.java index c4eeac95e..fe7dd6ddb 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlWorkloadExporter.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlWorkloadExporter.java @@ -48,6 +48,7 @@ import java.util.stream.Collectors; public class CqlWorkloadExporter { private final static Logger logger = LogManager.getLogger(CqlWorkloadExporter.class); public final static String DEFAULT_NAMING_TEMPLATE = "[OPTYPE-][COLUMN-][TYPEDEF-][TABLE!]-[KEYSPACE]"; + public static final String DEFAULT_REPLICATION = "\n 'class': 'SimpleStrategy',\n 'replication_factor': 'TEMPLATE(rf:1)'\n"; private final BindingsLibrary defaultBindings = new DefaultCqlBindings(); @@ -120,16 +121,30 @@ public class CqlWorkloadExporter { namer.populate(model); Map workload = new LinkedHashMap<>(); + workload.put("description", "Auto-generated workload from source schema."); + workload.put("scenarios", genScenarios(model)); workload.put("bindings", bindingsMap); Map blocks = new LinkedHashMap<>(); workload.put("blocks", blocks); blocks.put("schema", genSchemaBlock(model)); + blocks.put("truncate", genTruncateBlock(model)); blocks.put("rampup", genRampupBlock(model)); blocks.put("main", genMainBlock(model)); bindingsMap.putAll(bindings.getAccumulatedBindings()); return workload; } + private Map genScenarios(CqlModel model) { + return Map.of( + "default", Map.of( + "schema", "run driver=cql tags=block:schema threads===UNDEF cycles===UNDEF", + "rampup", "run driver=cql tags=block:rampup threads=auto cycles===TEMPLATE(rampup-cycles,10000)", + "main", "run driver=cql tags=block:main threads=auto cycles===TEMPLATE(main-cycles,10000)" + ), + "truncate", "run driver=cql tags=block:truncate threads===UNDEF cycles===UNDEF" + ); + } + private Map genMainBlock(CqlModel model) { Map mainOpTemplates = new LinkedHashMap<>(); @@ -230,7 +245,8 @@ public class CqlWorkloadExporter { .setDefaultFlowStyle(FlowStyle.BLOCK) .setIndent(2) .setDefaultScalarStyle(ScalarStyle.PLAIN) - .setMaxSimpleKeyLength(100) + .setMaxSimpleKeyLength(1000) + .setWidth(100) .setSplitLines(true) .setIndentWithIndicator(true) .setMultiLineFlow(true) @@ -258,12 +274,26 @@ public class CqlWorkloadExporter { return Map.of(); } + private Map genTruncateBlock(CqlModel model) { + Map truncateblock = new LinkedHashMap<>(); + Map ops = new LinkedHashMap<>(); + truncateblock.put("ops", ops); + + for (CqlTable table : model.getAllTables()) { + ops.put( + namer.nameFor(table, "optype", "truncate"), + "truncate " + table.getKeySpace() + "." + table.getTableName() + ";" + ); + } + return truncateblock; + } + private Map genSchemaBlock(CqlModel model) { Map schemablock = new LinkedHashMap<>(); Map ops = new LinkedHashMap<>(); for (CqlKeyspace ks : model.getKeyspaces().values()) { - ops.put("create-keyspace-" + ks.getKeyspaceName(), ks.getRefddl()); + ops.put("create-keyspace-" + ks.getKeyspaceName(), ks.getRefDdlWithReplFields(DEFAULT_REPLICATION)); } for (String ksname : model.getTablesByKeyspace().keySet()) { for (CqlTable cqltable : model.getTablesByKeyspace().get(ksname).values()) { diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/parser/CqlModelParser.java b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/parser/CqlModelParser.java index a0d1bbcc6..35bd5fe17 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/parser/CqlModelParser.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/parser/CqlModelParser.java @@ -19,6 +19,7 @@ package io.nosqlbench.converters.cql.parser; import io.nosqlbench.converters.cql.cqlast.CQBErrorListener; import io.nosqlbench.converters.cql.cqlast.CqlModel; import io.nosqlbench.converters.cql.cqlast.CqlModelBuilder; +import io.nosqlbench.converters.cql.cqlast.CqlType; import io.nosqlbench.converters.cql.generated.CqlLexer; import io.nosqlbench.converters.cql.generated.CqlParser; import org.antlr.v4.runtime.CharStreams; @@ -30,6 +31,7 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; public class CqlModelParser { @@ -45,6 +47,15 @@ public class CqlModelParser { } } + public static CqlType parseCqlType(String input) { + CqlModel parsed = parse(input, null); + List types = parsed.getTypes(); + if (types.size()!=1) { + throw new RuntimeException("error parsing typedef"); + } + return types.get(0); + } + public static CqlModel parse(String input, Path origin) { try { @@ -59,7 +70,7 @@ public class CqlModelParser { parser.addParseListener(cqlModelBuilder); parser.addErrorListener(errorListener); - CqlParser.RootContext keyspaceParser = parser.root(); + parser.root(); CqlModel model = cqlModelBuilder.getModel(); if (model.getErrors().size()>0) { diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/udts/ToUdt.java b/adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/udts/ToUdt.java new file mode 100644 index 000000000..224f5c53f --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/udts/ToUdt.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.datamappers.functions.udts; + +import com.datastax.oss.driver.api.core.data.UdtValue; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.UserDefinedType; +import com.datastax.oss.driver.internal.core.type.UserDefinedTypeBuilder; +import io.nosqlbench.converters.cql.cqlast.CqlType; +import io.nosqlbench.converters.cql.parser.CqlModelParser; + +import java.util.function.LongFunction; + +/** + * map + * where: + * A := (f1 text, f2 text, f3 int) + * AND + * B := (f1 text, f2 text) + */ +public class ToUdt implements LongFunction { + + private final String spec; + private final CqlType typeinfo; + private final UserDefinedType udt; + public ToUdt(String spec) { + this.spec=spec; + typeinfo = CqlModelParser.parseCqlType(spec); + UserDefinedTypeBuilder builder = new UserDefinedTypeBuilder(typeinfo.getKeyspace(), typeinfo.getName()); + typeinfo.getFields().forEach((name,typedef) -> { + DataType dataType = resolveDataType(typedef); + builder.withField(name,dataType); + }); + this.udt = builder.build(); + } + + private DataType resolveDataType(String typedef) { + + return null; + } + + @Override + public UdtValue apply(long value) { + return null; + } +}