IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

checkpoint on progress, naming indirection, primary key models, where predicates

Browse Source
This commit is contained in:
Jonathan Shook 2022-07-12 21:20:51 -05:00
parent 2df9e6b044
commit c683652acb
15 changed files with 737 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlColumnDef.java
View File

@ -16,13 +16,19 @@
package io.nosqlbench.converters.cql.cqlast;
public class CqlColumnDef {
String name;
String type;
import io.nosqlbench.nb.api.labels.Labeled;
public CqlColumnDef(String type, String name) {
this.type = type;
this.name = name;
import java.util.Map;
public class CqlColumnDef implements Labeled {
private String table;
private String keyspace;
private final String name;
private final String type;
public CqlColumnDef(String colname, String typedef) {
this.type = typedef;
this.name = colname;
}
public String getName() {
@ -33,8 +39,34 @@ public class CqlColumnDef {
return type;
}
public String getTable() {
return table;
}
public String getKeyspace() {
return keyspace;
}
@Override
public String toString() {
return " " + this.name + " " + this.type + ",";
return getLabels().toString();
}
@Override
public Map<String, String> getLabels() {
return Map.of(
"column", name,
"typedef", type,
"table", table,
"keyspace", keyspace
);
}
public void setKeyspace(String keyspace) {
this.keyspace = keyspace;
}
public void setTable(String table) {
this.table = table;
}
}

13
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlKeyspace.java
View File

@ -16,7 +16,11 @@
package io.nosqlbench.converters.cql.cqlast;
public class CqlKeyspace {
import io.nosqlbench.nb.api.labels.Labeled;
import java.util.Map;
public class CqlKeyspace implements Labeled {
String keyspaceName= "";
String refddl;
@ -46,4 +50,11 @@ public class CqlKeyspace {
public String getRefddl() {
return refddl;
}
@Override
public Map<String, String> getLabels() {
return Map.of(
"keyspace", keyspaceName
);
}
}

33
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModel.java
View File

@ -16,9 +16,7 @@
package io.nosqlbench.converters.cql.cqlast;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.function.Supplier;
public class CqlModel {
@ -59,14 +57,17 @@ public class CqlModel {
public void saveTable(String keyspace, String text, String refddl) {
table.setKeyspace(keyspace);
table.setName(text);
table.setTable(text);
table.setRefDdl(refddl);
this.tables.computeIfAbsent(keyspace, ks->new LinkedHashMap<>()).put(text, table);
table = null;
}
public void saveColumnDefinition(String coltype, String colname) {
table.addcolumnDef(coltype, colname);
public void saveColumnDefinition(String colname, String coltype, boolean isPrimaryKey) {
this.table.addcolumnDef(colname, coltype);
if (isPrimaryKey) {
this.table.addPartitionKey(colname);
}
}
public Map<String, CqlKeyspace> getKeyspaces() {
@ -97,4 +98,24 @@ public class CqlModel {
}
return sb.toString();
}
/**
* Get all the keyspace names which have been referenced in any way, whether or not
* this was in a keyspace definition or some other DDL like table or udt names.
* @return A list of all known keyspace names
*/
public Set<String> getAllKnownKeyspaceNames() {
Set<String> ksnames = new LinkedHashSet<>();
ksnames.addAll(this.keyspaces.keySet());
ksnames.addAll(this.tables.keySet());
return ksnames;
}
public void addPartitionKey(String partitionKey) {
table.addPartitionKey(partitionKey);
}
public void addClusteringColumn(String ccolumn) {
table.addClusteringColumn(ccolumn);
}
}

35
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModelBuilder.java
View File

@ -64,6 +64,35 @@ public class CqlModelBuilder extends CqlParserBaseListener {
model.newTable();
}
@Override
public void exitPrimaryKeyColumn(CqlParser.PrimaryKeyColumnContext ctx) {
super.exitPrimaryKeyColumn(ctx);
}
@Override
public void exitPrimaryKeyDefinition(CqlParser.PrimaryKeyDefinitionContext ctx) {
if (ctx.singlePrimaryKey()!=null) {
model.addPartitionKey(ctx.singlePrimaryKey().column().getText());
} else if (ctx.compositeKey()!=null) {
for (CqlParser.PartitionKeyContext pkctx : ctx.compositeKey().partitionKeyList().partitionKey()) {
model.addPartitionKey(pkctx.column().getText());
}
for (CqlParser.ClusteringKeyContext ccol : ctx.compositeKey().clusteringKeyList().clusteringKey()) {
model.addClusteringColumn(ccol.column().getText());
}
} else if (ctx.compoundKey()!=null) {
model.addClusteringColumn(ctx.compoundKey().partitionKey().column().getText());
for (CqlParser.ClusteringKeyContext ccol : ctx.compoundKey().clusteringKeyList().clusteringKey()) {
model.addClusteringColumn(ccol.column().getText());
}
}
}
@Override
public void exitSinglePrimaryKey(CqlParser.SinglePrimaryKeyContext ctx) {
super.exitSinglePrimaryKey(ctx);
}
@Override
public void exitCreateTable(CqlParser.CreateTableContext ctx) {
model.saveTable(
@ -87,7 +116,11 @@ public class CqlModelBuilder extends CqlParserBaseListener {
@Override
public void exitColumnDefinition(CqlParser.ColumnDefinitionContext ctx) {
model.saveColumnDefinition(ctx.dataType().getText(),ctx.column().getText());
model.saveColumnDefinition(
ctx.column().getText(),
ctx.dataType().getText(),
ctx.primaryKeyColumn()!=null
);
}
@Override

65
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlTable.java
View File

@ -16,15 +16,21 @@
package io.nosqlbench.converters.cql.cqlast;
import io.nosqlbench.nb.api.labels.Labeled;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class CqlTable {
String name = "";
public class CqlTable implements Labeled {
String table = "";
String keyspace = "";
List<CqlColumnDef> coldefs = new ArrayList();
List<CqlColumnDef> coldefs = new ArrayList<>();
String refddl;
List<String> partitionKeys = new ArrayList<>();
List<String> clusteringColumns = new ArrayList<>();
public CqlTable() {
}
@ -33,17 +39,20 @@ public class CqlTable {
this.coldefs.add(cqlField);
}
public void setName(String tableName) {
this.name = tableName;
public void setTable(String tableName) {
this.table = tableName;
for (CqlColumnDef coldef : coldefs) {
coldef.setTable(tableName);
}
}
public void addcolumnDef(String type, String fieldName) {
coldefs.add(new CqlColumnDef(type, fieldName));
public void addcolumnDef(String colname, String typedef) {
coldefs.add(new CqlColumnDef(colname, typedef));
}
@Override
public String toString() {
return "cql table: '" + this.name + "':\n"
return "cql table: '" + this.table + "':\n"
+ this.coldefs.stream()
.map(Object::toString)
.map(s -> " " +s)
@ -55,11 +64,14 @@ public class CqlTable {
}
public String getTableName() {
return this.name;
return this.table;
}
public void setKeyspace(String keyspace) {
this.keyspace=keyspace;
for (CqlColumnDef coldef : coldefs) {
coldef.setKeyspace(keyspace);
}
}
public String getRefDdl() {
@ -78,4 +90,39 @@ public class CqlTable {
public String getKeySpace() {
return this.keyspace;
}
@Override
public Map<String, String> getLabels() {
return Map.of(
"keyspace", this.keyspace,
"table", this.table
);
}
public void addPartitionKey(String pkey) {
this.partitionKeys.add(pkey);
}
public void addClusteringColumn(String ccol) {
this.clusteringColumns.add(ccol);
}
public List<String> getPartitionKeys() {
return this.partitionKeys;
}
public List<String> getClusteringColumns() {
return this.clusteringColumns;
}
public String typedefForColumn(String colname) {
return coldefs.stream()
.filter(c -> c.getName().equalsIgnoreCase(colname))
.map(CqlColumnDef::getType).findFirst().orElseThrow();
}
public CqlColumnDef getColumnDefForName(String colname) {
return coldefs.stream().filter(c -> c.getName().equalsIgnoreCase(colname))
.findFirst().orElseThrow();
}
}

101
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlWorkloadExporter.java
View File

@ -33,13 +33,10 @@ import org.snakeyaml.engine.v2.common.ScalarStyle;
import org.snakeyaml.engine.v2.representer.BaseRepresenter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
/**
@ -49,29 +46,37 @@ import java.util.stream.Collectors;
* provided elements.
*/
public class CqlWorkloadExporter {
private final static Logger logger = LogManager.getLogger(CqlWorkloadExporter.class);
public final static String DEFAULT_NAMING_TEMPLATE = "[OPTYPE-][COLUMN-][TYPEDEF][-TABLE!][-KEYSPACE]";
private final Map<String, String> defaultBindings = new DefaultCqlBindings();
private final NamingFolio namer;
private final CqlModel model;
public CqlWorkloadExporter(CqlModel model) {
namer = new NamingFolio(DEFAULT_NAMING_TEMPLATE);
this.model = model;
}
public CqlWorkloadExporter(String ddl, Path srcpath) {
namer = new NamingFolio(DEFAULT_NAMING_TEMPLATE);
this.model = CqlModelParser.parse(ddl, srcpath);
}
public CqlWorkloadExporter(String ddl) {
namer = new NamingFolio(DEFAULT_NAMING_TEMPLATE);
this.model = CqlModelParser.parse(ddl, null);
}
public CqlWorkloadExporter(Path path) {
namer = new NamingFolio(DEFAULT_NAMING_TEMPLATE);
this.model = CqlModelParser.parse(path);
}
public static void main(String[] args) {
logger.info("running CQL workload exporter with args:" + Arrays.toString(args));
if (args.length == 0) {
throw new RuntimeException("Usage example: PROG filepath.cql filepath.yaml");
}
@ -83,12 +88,18 @@ public class CqlWorkloadExporter {
throw new RuntimeException("File '" + srcpath + "' does not exist.");
}
Path target = Path.of(srcpath.toString().replace("\\.cql", "\\.yaml"));
Path target = null;
if (args.length == 2) {
target = Path.of(args[1]);
logger.info("using output path as '" + target + "'");
} else {
target = Path.of(srcpath.toString().replace(".cql", ".yaml"));
logger.info("assumed output path as '" + target + "'");
}
if (!target.toString().endsWith(".yaml")) {
throw new RuntimeException("Target file must end in .yaml");
throw new RuntimeException("Target file must end in .yaml, but it is '" + target + "'");
}
if (Files.exists(target) && !target.toString().startsWith("_")) {
throw new RuntimeException("Target file '" + target + "' exists. Please remove it first or use a different target file name.");
@ -97,9 +108,9 @@ public class CqlWorkloadExporter {
CqlWorkloadExporter exporter = new CqlWorkloadExporter(srcpath);
String workload = exporter.getWorkloadAsYaml();
try {
Files.write(
Files.writeString(
target,
workload.getBytes(StandardCharsets.UTF_8),
workload,
StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING
);
} catch (IOException e) {
@ -108,16 +119,15 @@ public class CqlWorkloadExporter {
}
public Map<String, Object> getWorkload() {
Map<String, Object> workload = new LinkedHashMap<>();
namer.populate(model);
for (CqlKeyspace ks : model.getKeyspaces().values()) {
workload.put("bindings", getDefaultCqlBindings());
Map<String, Object> blocks = new LinkedHashMap<>();
workload.put("blocks", blocks);
blocks.put("schema", genSchemaBlock(model));
blocks.put("rampup", genRampupBlock(model));
blocks.put("main", genMainBlock(model));
}
Map<String, Object> workload = new LinkedHashMap<>();
workload.put("bindings", defaultBindings);
Map<String, Object> blocks = new LinkedHashMap<>();
workload.put("blocks", blocks);
blocks.put("schema", genSchemaBlock(model));
blocks.put("rampup", genRampupBlock(model));
blocks.put("main", genMainBlock(model));
return workload;
}
@ -127,17 +137,14 @@ public class CqlWorkloadExporter {
mainOpTemplates.putAll(
model.getAllTables()
.stream()
.collect(Collectors.toMap(
t -> "insert-" + t.getKeySpace()+"__"+t.getTableName(),
this::genUpsertTemplate)
)
.collect(Collectors.toMap(namer::nameFor, this::genUpsertTemplate))
);
mainOpTemplates.putAll(
model.getAllTables()
.stream()
.collect(Collectors.toMap(
t -> "select-" + t.getKeySpace()+"__"+t.getTableName(),
t -> namer.nameFor(t, "optype", "select"),
this::genSelectTemplate)
));
@ -150,27 +157,39 @@ public class CqlWorkloadExporter {
Map<String, String> rampupOpTemplates = model.getAllTables()
.stream()
.collect(Collectors.toMap(
t -> "insert-" + t.getKeySpace()+"__"+t.getTableName(),
this::genUpsertTemplate)
t -> namer.nameFor(t, "optype", "rampup-insert"),
this::genUpsertTemplate
)
);
return Map.of("ops", rampupOpTemplates);
}
private String genSelectTemplate(CqlTable table) {
List<CqlColumnDef> cdefs = table.getColumnDefinitions();
return "select * from " + table.getKeySpace() + "." + table.getTableName() +
"\n WHERE " + genPredicateTemplate(table);
}
private String genPredicateTemplate(CqlTable table) {
return table.getColumnDefinitions()
.stream()
.map(this::genPredicatePart)
.collect(Collectors.joining("\n AND "))
+ ";";
StringBuilder sb = new StringBuilder();
List<CqlColumnDef> pkeys = new ArrayList<>();
for (String pkey : table.getPartitionKeys()) {
CqlColumnDef coldef = table.getColumnDefForName(pkey);
pkeys.add(coldef);
}
for (String ccol : table.getClusteringColumns()) {
CqlColumnDef coldef = table.getColumnDefForName(ccol);
pkeys.add(coldef);
}
pkeys.stream().map(this::genPredicatePart)
.forEach(p -> {
sb.append(p).append("\n AND ");
});
sb.setLength(sb.length() - "\n AND ".length());
return sb.toString();
}
private String genPredicatePart(CqlColumnDef def) {
@ -182,15 +201,20 @@ public class CqlWorkloadExporter {
logger.warn("Unknown literal format for " + typeName);
}
return def.getName() + "=" + cqlLiteralFormat.format("{" + def.getName() + "}");
return def.getName() + "=" + cqlLiteralFormat.format("{" + namer.nameFor(def) + "}");
}
private String genUpsertTemplate(CqlTable table) {
List<CqlColumnDef> cdefs = table.getColumnDefinitions();
return "insert into " + table.getKeySpace() + "." + table.getTableName() + "\n ( "
+ cdefs.stream().map(cd -> cd.getName()).collect(Collectors.joining(" , ")) +
" )\n values\n (" + cdefs.stream().map(cd -> cd.getName()).collect(Collectors.joining("},{", "{", "}"))
+ cdefs.stream().map(cd -> cd.getName())
.collect(Collectors.joining(" , ")) +
" )\n values\n (" +
cdefs
.stream()
.map(cd -> namer.nameFor(cd))
.collect(Collectors.joining("},{", "{", "}"))
+ ");";
}
@ -207,6 +231,7 @@ public class CqlWorkloadExporter {
.build();
BaseRepresenter r;
Dump dump = new Dump(dumpSettings);
Map<String, Object> workload = getWorkload();
return dump.dumpToString(workload);
}
@ -243,10 +268,4 @@ public class CqlWorkloadExporter {
return schemablock;
}
private Map<String, Object> getDefaultCqlBindings() {
return Map.of(
"text", "NumberNameToString()",
"timestamp", "yaddayaddayadda"
);
}
}

54
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/DefaultCqlBindings.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters;
import java.util.LinkedHashMap;
import java.util.Map;
public class DefaultCqlBindings extends LinkedHashMap<String,String> {{
putAll(Map.of(
"text","NumberNameToString();",
"int","ToString();",
"bigint", "ToLong();",
"blob", "ByteBufferSizedHashed(30);",
"boolean", "ToBoolean();",
"date", "ToDate();",
"decimal", "ToBigDecimal();",
"double", "ToDouble()",
"duration", "ToCqlDuration();")
);
putAll(Map.of(
"float","ToFloat()",
"frozen<list<int>>","ListSizedHashed(HashRange(3,7),ToInt()));",
"list<text>","ListStepped(Identity(),NumberNameToString(),NumberNameToString())",
"map<text,text>","MapSized(3, Combinations('A-Z;0-9'), NumberNameToString(), ToString());",
"set<text>","SetSized(HashRange(3,4),Identity(),NumberNameToString()));",
"smallint","ToShort();")
);
putAll(Map.of(
"time","StartingEpochMillis('2022-01-01 00:00:00Z');",
"timestamp","ToDate();",
"timeuuid","ToTimeUUID();",
"tinyint","ToByte();",
"uuid","ToUUID();",
"varint","ToBigDecimal();",
"inet","ToInetAddress();"
));
put(
"id_int","ToString()"
);
}}

148
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/ElementNamer.java Normal file
View File

@ -0,0 +1,148 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters;
import io.nosqlbench.nb.api.labels.Labeled;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class ElementNamer implements Function<Map<String, String>, String> {
public final static String _DEFAULT_TEMPLATE = "[PREFIX-][OPTYPE-][KEYSPACE__][TABLE][-DATATYPE]";
// for convenient reference
public final static String PREFIX = "PREFIX";
public final static String OPTYPE = "OPTYPE";
public final static String KEYSPACE = "KEYSPACE";
public final static String TABLE = "TABLE";
public final static String DATATYPE = "DATATYPE";
private final List<Section> sections = new ArrayList<>();
private final String spec;
private final List<Function<String, String>> transformers = new ArrayList<>();
public ElementNamer(String template, List<Function<String,String>> transformers) {
this.spec = template;
this.transformers.addAll(transformers);
Pattern pattern = Pattern.compile("(?<prefix>[^\\]]+)?\\[(?<section>(?<pre>.*?)(?<name>[A-Z]+)(?<required>!)?(?<post>.*?))?]");
Matcher scanner = pattern.matcher(template);
while (scanner.find()) {
if (scanner.group("section")!=null) {
Section section = new Section(
scanner.group("name").toLowerCase(),
scanner.group("pre") +
scanner.group("name")
+ scanner.group("post"),
scanner.group("required") != null);
sections.add(section);
}
if (scanner.group("prefix")!=null) {
String prefix = scanner.group("prefix");
sections.add(new Section(null, prefix, true));
}
}
}
public ElementNamer(String template) {
this(template, List.of());
}
public ElementNamer() {
this(_DEFAULT_TEMPLATE, List.of());
}
/**
* For each section in the Element Namer's ordered templates,
* if the labels contain a value for it, substitute the value
* for the named label into the section where the field is named in upper-case,
* including all the surrounding non-character literals.
*
* @param labels Metadata for the element to be named.
* @return A formatted string, with the sections added which are defined.
*/
@Override
public String apply(Map<String, String> labels) {
StringBuilder sb = new StringBuilder();
for (Section section : sections) {
String appender = section.apply(labels);
sb.append(appender);
}
String value = sb.toString();
for (Function<String, String> transformer : transformers) {
value = transformer.apply(value);
}
return value;
}
public String apply(Labeled element, String... keysAndValues) {
LinkedHashMap<String, String> mylabels = new LinkedHashMap<>();
for (int idx = 0; idx < keysAndValues.length; idx += 2) {
mylabels.put(keysAndValues[idx], keysAndValues[idx + 1]);
}
mylabels.putAll(element.getLabels());
return apply(mylabels);
}
private final static class Section implements Function<Map<String, String>, String> {
String name;
String template;
boolean required;
public Section(String name, String template, boolean required) {
this.name = (name!=null ? name.toLowerCase() : null);
this.template = template.toLowerCase();
this.required = required;
}
@Override
public String apply(Map<String, String> labels) {
if (name==null) {
return template;
} else if (labels.containsKey(name)) {
return template.replace(name, labels.get(name));
} else if (labels.containsKey(name.toUpperCase())) {
return template.replace(name, labels.get(name.toUpperCase()));
} else if (required) {
throw new RuntimeException("Section label '" + name + "' was not provided for template, but it is required.");
} else {
return "";
}
}
@Override
public String toString() {
return "Section{" +
"name='" + name + '\'' +
", template='" + template + '\'' +
", required=" + required +
'}';
}
}
@Override
public String toString() {
return "ElementNamer: " + this.spec + "]\n" +
"sections=" + sections +
'}';
}
}

23
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/LabeledElement.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters;
import java.util.Map;
public interface LabeledElement {
Map<String, String> getLabels();
}

82
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/NamingFolio.java Normal file
View File

@ -0,0 +1,82 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters;
import io.nosqlbench.converters.cql.cqlast.CqlColumnDef;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlTable;
import io.nosqlbench.nb.api.labels.Labeled;
import java.util.*;
/**
* This will be a pre-built inverted index of all field which need to have bindings assigned.
* A field reference is presumed to be unique within the scope from which the traversal to
* the working set has a single path.
*
* // name -> type -> table -> keyspace -> namespace
*/
public class NamingFolio {
private final Map<String, Labeled> graph = new LinkedHashMap<>();
private final ElementNamer namer;
public final static String DEFAULT_NAMER_SPEC = "[COLUMN][-TYPEDEF]_[TABLE][-KEYSPACE]";
public NamingFolio(String namerspec) {
this.namer = new ElementNamer(
namerspec,
List.of(s -> s.toLowerCase().replaceAll("[^a-zA-Z0-9_-]", ""))
);
}
public NamingFolio() {
this.namer = new ElementNamer(DEFAULT_NAMER_SPEC);
}
public void addFieldRef(Map<String, String> labels) {
String name = namer.apply(labels);
graph.put(name, Labeled.forMap(labels));
}
public void addFieldRef(String column, String typedef, String table, String keyspace) {
addFieldRef(Map.of("column", column, "typedef", typedef, "table", table, "keyspace", keyspace));
}
/**
* This will eventually elide extraneous fields according to knowledge of all known names
* by name, type, table, keyspace. For now it just returns everything in fully qualified form.
*/
public String nameFor(Labeled labeled, String... fields) {
Map<String, String> labelsPlus = labeled.getLabelsAnd(fields);
String name = namer.apply(labelsPlus);
return name;
}
public void populate(CqlModel model) {
for (CqlTable table : model.getAllTables()) {
for (CqlColumnDef coldef : table.getColumnDefinitions()) {
addFieldRef(coldef.getLabels());
}
}
}
public Set<String> getNames() {
return new LinkedHashSet<>(graph.keySet());
}
}

1
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/parser/CqlModelParser.java
View File

@ -55,7 +55,6 @@ public class CqlModelParser {
CommonTokenStream tokens = new CommonTokenStream(lexer);
CqlParser parser = new CqlParser(tokens);
CqlModelBuilder cqlModelBuilder = new CqlModelBuilder(errorListener);
parser.addParseListener(cqlModelBuilder);
parser.addErrorListener(errorListener);

35
adapter-cqld4/src/main/resources/activities/examples/bindings/cqltypes.yaml Normal file
View File

@ -0,0 +1,35 @@
blocks:
schema:
ops:
create-keyspace: |
create keyspace if not exists <<keyspace:baselines>>
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 'TEMPLATE(rf,1)'}
AND durable_writes = true;
create-table: |
CREATE TABLE baselines.alltypes (
id text PRIMARY KEY,
f_bigint bigint,
f_blob blob,
f_boolean boolean,
f_date date,
f_decimal decimal,
f_double double,
f_duration duration,
f_float float,
f_frozen frozen<list<int>>,
f_list list<text>,
f_map map<text, text>,
f_set set<text>,
f_smallint smallint,
f_text text,
f_time time,
f_timestamp timestamp,
f_timeuuid timeuuid,
f_tinyint tinyint,
f_uuid uuid,
f_varchar text,
f_varint varint,
f_ascii ascii,
f_inet inet,
f_int int
)

103
adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/ElementNamerTest.java Normal file
View File

@ -0,0 +1,103 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters;
import io.nosqlbench.nb.api.labels.Labeled;
import org.junit.jupiter.api.Test;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
public class ElementNamerTest {
@Test
public void testNonRequiredFields() {
ElementNamer namer = new ElementNamer("[ABC---][,deFGH][__IJ__]");
assertThat(namer.apply(Map.of())).isEqualTo("");
}
@Test
public void testLiteralTweens() {
ElementNamer namer = new ElementNamer("[ABC---]!-23[,deFGH])(*&[__IJ__]");
assertThat(namer.apply(Map.of("abc","123"))).isEqualTo("123---!-23)(*&");
}
@Test
public void testPartialFields() {
ElementNamer namer = new ElementNamer("[ABC---][,deFGH][__IJ__]");
assertThat(namer.apply(Map.of("abc", "base"))).isEqualTo("base---");
}
@Test
public void testLabeledFields() {
ElementNamer namer = new ElementNamer("[ABC---][,deFGH][__IJ__]");
Labeled mylabeled = new Labeled() {
@Override
public Map<String, String> getLabels() {
return Map.of("ij", "eyejay");
}
};
assertThat(namer.apply(mylabeled, "abc", "base")).isEqualTo("base---__eyejay__");
}
@Test
public void testCasedSectionName() {
ElementNamer namer = new ElementNamer("[ABC---][,deFGH][__IJ__]");
assertThat(namer.apply(
Map.of(
"abc", "1111",
"fgh", "2222",
"IJ", "3333"
))
).isEqualTo("1111---,de2222__3333__");
}
@Test
public void testRequiredFieldsPresent() {
ElementNamer namer = new ElementNamer("[ABC!---!]");
Labeled mylabeled = new Labeled() {
@Override
public Map<String, String> getLabels() {
return Map.of("ij", "eyejay");
}
};
assertThat(namer.apply(Map.of(
"abc", "WOOT"
))).isEqualTo("WOOT---!");
}
@Test
public void testRequiredFieldsMissing() {
ElementNamer namer = new ElementNamer("[ABC!---!]");
Labeled mylabeled = new Labeled() {
@Override
public Map<String, String> getLabels() {
return Map.of("ij", "eyejay");
}
};
assertThatThrownBy(() -> namer.apply(Map.of(
"not_the", "right_label"
))).isInstanceOf(RuntimeException.class);
}
}

38
adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/NamingFolioTest.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters;
import org.junit.jupiter.api.Test;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
public class NamingFolioTest {
@Test
public void testBindingFolio() {
NamingFolio folio = new NamingFolio();
folio.addFieldRef(Map.of("column","c1","typedef","t1","table","tb1","keyspace","ks1"));
assertThat(folio.getNames()).containsExactly("c1-t1-tb1-ks1");
folio.addFieldRef("c2","t2","tb2","ks2");
assertThat(folio.getNames()).containsExactly(
"c1-t1-tb1-ks1","c2-t2-tb2-ks2"
);
}
}

26
nb-api/src/main/java/io/nosqlbench/nb/api/labels/Labeled.java
View File

@ -16,8 +16,34 @@
package io.nosqlbench.nb.api.labels;
import java.util.LinkedHashMap;
import java.util.Map;
public interface Labeled {
Map<String, String> getLabels();
default Map<String, String> getLabelsAnd(String... keyvalues) {
LinkedHashMap<String, String> map = new LinkedHashMap<>(getLabels());
for (int idx = 0; idx < keyvalues.length; idx+=2) {
map.put(keyvalues[0],keyvalues[1]);
}
return map;
}
static MapLabels forMap(Map<String,String> labels) {
return new MapLabels(labels);
}
class MapLabels implements Labeled {
private final Map<String, String> labels;
public MapLabels(Map<String,String> labels) {
this.labels = labels;
}
@Override
public Map<String, String> getLabels() {
return labels;
}
}
}