IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

checkpoint: exporter.yaml config and many transformers added (see the file)

Browse Source
This commit is contained in:
Jonathan Shook 2022-07-15 19:44:58 -05:00
parent 2af5f2c39a
commit fb200e0b58
40 changed files with 1002 additions and 335 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
adapter-cqld4/pom.xml
View File

@ -147,6 +147,7 @@
<resource>
<directory>src/main/java</directory>
<includes>
<include>**/*.conf</include>
<include>**/*.md</include>
<include>**/*.yaml</include>
<include>**/*.txt</include>

4
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CQBErrorListener.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CGErrorListener.java
View File

@ -28,12 +28,12 @@ import java.util.List;
import java.util.Locale;
import java.util.function.Supplier;
public class CQBErrorListener extends BaseErrorListener implements Supplier<List<String>> {
public class CGErrorListener extends BaseErrorListener implements Supplier<List<String>> {
List<String> errors = new ArrayList<>();
private final Path origin;
public CQBErrorListener(Path origin) {
public CGErrorListener(Path origin) {
this.origin = origin;
}

23
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/ColType.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.cqlast;
public enum ColType {
PartitionComponent,
ClusteringComponent,
NonKeyComponent
}

4
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlColumnDef.java
View File

@ -28,6 +28,8 @@ public class CqlColumnDef implements NBNamedElement, Labeled {
private String keyspace;
private String name;
private String type;
private int position;
private ColType coltype;
public CqlColumnDef(String colname, String typedef, String refColumnDdl) {
this.type = typedef;
@ -35,7 +37,7 @@ public class CqlColumnDef implements NBNamedElement, Labeled {
this.refDefinitionDdl = refColumnDdl;
}
public String getRefddl() {
public String getDefinitionDdl() {
return refDefinitionDdl;
}

39
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlKeyspace.java
View File

@ -18,31 +18,24 @@ package io.nosqlbench.converters.cql.cqlast;
import io.nosqlbench.api.config.NBNamedElement;
import io.nosqlbench.api.labels.Labeled;
import io.nosqlbench.converters.cql.exporters.CGKeyspaceStats;
import java.util.HashMap;
import java.util.Map;
public class CqlKeyspace implements NBNamedElement, Labeled {
String keyspaceName= "";
String refddl;
private String refReplDdl;
Map<String,String> keyspaceAttributes = new HashMap<String,String>();
public Map<String, String> getKeyspaceAttributes() {
return keyspaceAttributes;
}
public void setKeyspaceAttributes(Map<String, String> keyspaceAttributes) {
this.keyspaceAttributes = keyspaceAttributes;
}
CGKeyspaceStats stats;
public CqlKeyspace() {
}
public void setKeyspaceName(String name) {
this.keyspaceName=name;
public void setKeyspaceName(String newname) {
if (this.refddl!=null) {
this.refddl = refddl.replaceAll(this.keyspaceName, newname);
}
this.keyspaceName=newname;
}
public String getName() {
@ -72,11 +65,23 @@ public class CqlKeyspace implements NBNamedElement, Labeled {
);
}
public void setRefReplDdl(String refReplDdl) {
this.refReplDdl=refReplDdl;
public void setRefReplDdl(String newRefReplDdl) {
if (this.refddl!=null) {
this.refddl=this.refddl.replaceAll(this.refReplDdl,newRefReplDdl);
}
this.refReplDdl=newRefReplDdl;
}
public String getRefDdlWithReplFields(String replFields) {
return refddl.replace(refReplDdl,replFields);
refddl.replace(refReplDdl,replFields);
return refddl;
}
public String getReplRefDdl() {
return this.refReplDdl;
}
public void setStats(CGKeyspaceStats ksstats) {
this.stats=ksstats;
}
}

163
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModel.java
View File

@ -16,23 +16,93 @@
package io.nosqlbench.converters.cql.cqlast;
import io.nosqlbench.converters.cql.exporters.CGKeyspaceStats;
import io.nosqlbench.converters.cql.exporters.CGSchemaStats;
import io.nosqlbench.converters.cql.exporters.CGTableStats;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.*;
import java.util.function.Supplier;
/**
* This model contains definition level details for schema elements which are parsed from the
* Antlr4 CQL grammar.
* Because keyspace, table, column, and type elements are handled sometimes in different ways,
* these are stored in separate data structures.
* When you see a *refddl or similar field, this is a copy of the text image from the original
* parsed syntax. These are used for populating schema blocks without doing a full parse.
* If you update either the refddl or the actual AST level elements for any of the types in this
* model, you are required to update the other version along with it, using string substitution
* if necessary.
*/
public class CqlModel {
private final static Logger logger = LogManager.getLogger(CqlModel.class);
private final Supplier<List<String>> errors;
Map<String, CqlKeyspace> keyspaces = new LinkedHashMap<>();
Map<String, Map<String, CqlTable>> tables = new LinkedHashMap<>();
Map<String, CqlKeyspace> keyspaceDefs = new LinkedHashMap<>();
Map<String, Map<String, CqlTable>> tableDefs = new LinkedHashMap<>();
Map<String, Map<String, CqlType>> types = new LinkedHashMap<>();
CGSchemaStats schemaStats = null;
public CGSchemaStats getKeyspaceAttributes() {
return schemaStats;
}
public void setKeyspaceAttributes(CGSchemaStats schemaStats) {
this.schemaStats = schemaStats;
for (String statsKeyspacename : schemaStats.getKeyspaces().keySet()) {
CGKeyspaceStats keyspaceStats = schemaStats.getKeyspace(statsKeyspacename);
if (keyspaceDefs.containsKey(statsKeyspacename)) {
logger.debug("setting keyspace stats for '" + statsKeyspacename + "'");
keyspaceDefs.get(statsKeyspacename).setStats(keyspaceStats);
} else {
logger.debug(" skipping keyspace stats for '" + statsKeyspacename + "'");
}
for (String statsTableName : keyspaceStats.getKeyspaceTables().keySet()) {
CGTableStats tableStats = keyspaceStats.getKeyspaceTables().get(statsTableName);
Map<String, CqlTable> modelTables = tableDefs.get(statsKeyspacename);
if (modelTables!=null) {
CqlTable modelTable = modelTables.get(statsTableName);
if (modelTable!=null) {
logger.debug("setting table stats for '" + statsKeyspacename+"."+statsTableName+"'");
modelTable.setTableAttributes(tableStats);
} else {
logger.debug(" skipping table stats for '" + statsKeyspacename + "."+statsTableName+"'");
}
} else {
logger.debug(" SKIPPING stats for all tables in keyspace '" + statsKeyspacename + "'");
}
}
}
// schemaStats.getKeyspaces().forEach((ksname, ksstats) -> {
// CqlKeyspace modelKs = getKeyspacesByName().get(ksname);
// if (modelKs!=null) {
// modelKs.setStats(ksstats);
// ksstats.getKeyspaceTables().forEach((tbname, tbstats) -> {
// Map<String, CqlTable> tabledefs = tableDefs.get(ksname);
// if (tabledefs!=null) {
// for (CqlTable tabledef : tabledefs.values()) {
// tabledef.setTableAttributes(tbstats);
// }
// }
// });
//
// }
// });
}
transient CqlKeyspace keyspace = null;
transient CqlTable table;
transient CqlType udt;
public boolean hasStats() {
return keyspaces.size()>0 && keyspaces.values().iterator().next().getKeyspaceAttributes().size()!=0;
return schemaStats!=null;
}
public CqlModel(Supplier<List<String>> errorSource) {
this.errors = errorSource;
}
@ -48,7 +118,7 @@ public class CqlModel {
public void saveKeyspace(String text,String refddl) {
keyspace.setKeyspaceName(text);
keyspace.setRefDdl(refddl);
this.keyspaces.put(text, keyspace);
this.keyspaceDefs.put(text, keyspace);
keyspace=null;
}
@ -60,7 +130,7 @@ public class CqlModel {
table.setKeyspace(keyspace);
table.setName(text);
table.setRefDdl(refddl);
this.tables.computeIfAbsent(keyspace, ks->new LinkedHashMap<>()).put(text, table);
this.tableDefs.computeIfAbsent(keyspace, ks->new LinkedHashMap<>()).put(text, table);
table = null;
}
@ -72,37 +142,38 @@ public class CqlModel {
}
public Map<String, CqlKeyspace> getKeyspacesByName() {
return keyspaces;
return keyspaceDefs;
}
public List<CqlKeyspace> getKeyspaces() {
return new ArrayList<>(this.keyspaces.values());
public List<CqlKeyspace> getKeyspaceDefs() {
return new ArrayList<>(this.keyspaceDefs.values());
}
public Map<String, Map<String, CqlTable>> getTablesByNameByKeyspace() {
return tables;
return tableDefs;
}
public List<CqlTable> getTablesForKeyspace(String ksname) {
Map<String, CqlTable> tables = this.tables.get(ksname);
Map<String, CqlTable> tables = this.tableDefs.get(ksname);
if (tables!=null) {
return new ArrayList<>(tables.values());
}
return List.of();
}
public List<CqlTable> getTables() {
return tables.values().stream().flatMap(m->m.values().stream()).toList();
public List<CqlTable> getTableDefs() {
return tableDefs.values().stream().flatMap(m->m.values().stream()).toList();
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
for (String ks : keyspaces.keySet()) {
CqlKeyspace keyspace = keyspaces.get(ks);
for (String ks : keyspaceDefs.keySet()) {
CqlKeyspace keyspace = keyspaceDefs.get(ks);
sb.append("keyspace '").append(keyspace.getName()).append("':\n");
sb.append(keyspace).append("\n");
tables.getOrDefault(ks,Map.of()).values().stream()
tableDefs.getOrDefault(ks,Map.of()).values().stream()
.forEach(table -> {
sb.append("table '").append(table.getName()).append("':\n");
sb.append(table);
@ -118,8 +189,8 @@ public class CqlModel {
*/
public Set<String> getAllKnownKeyspaceNames() {
Set<String> ksnames = new LinkedHashSet<>();
ksnames.addAll(this.keyspaces.keySet());
ksnames.addAll(this.tables.keySet());
ksnames.addAll(this.keyspaceDefs.keySet());
ksnames.addAll(this.tableDefs.keySet());
return ksnames;
}
@ -161,4 +232,62 @@ public class CqlModel {
}
return list;
}
public void removeKeyspaceDef(String ksname) {
this.keyspaceDefs.remove(ksname);
}
public void removeTablesForKeyspace(String ksname) {
this.tableDefs.remove(ksname);
}
public void removeTypesForKeyspace(String name) {
this.types.remove(name);
}
public String getSummaryLine() {
return "keyspaces: " + keyspaceDefs.size() + ", tables: " + getTableDefs().size() +
", columns: " + getTableDefs().stream().mapToInt(t -> t.getColumnDefinitions().size()).sum() +
", types: " + getTypes().size();
}
public void renamekeyspace(String keyspaceName, String newKeyspaceName) {
if (this.keyspaceDefs.containsKey(keyspaceName)) {
CqlKeyspace keyspace = this.keyspaceDefs.remove(keyspaceName);
keyspace.setKeyspaceName(newKeyspaceName);
this.keyspaceDefs.put(newKeyspaceName, keyspace);
}
if (this.tableDefs.containsKey(keyspaceName)) {
Map<String, CqlTable> tablesForKeyspace = this.tableDefs.remove(keyspaceName);
if (tablesForKeyspace!=null) {
for (CqlTable table : tablesForKeyspace.values()) {
table.setKeyspace(newKeyspaceName);
}
}
this.tableDefs.put(newKeyspaceName, tablesForKeyspace);
}
if (this.types.containsKey(keyspaceName)) {
Map<String, CqlType> typesForKeyspace = this.types.remove(keyspaceName);
if (typesForKeyspace!=null) {
for (CqlType cqltype : typesForKeyspace.values()) {
cqltype.setKeyspace(newKeyspaceName);
}
}
this.types.put(newKeyspaceName,typesForKeyspace);
}
}
public void renameTable(String keyspaceName, String tableName, String newTableName) {
Map<String, CqlTable> tablesInKeyspace = tableDefs.get(keyspaceName);
CqlTable table = tablesInKeyspace.remove(tableName);
table.setName(newTableName);
tablesInKeyspace.put(newTableName,table);
}
public void renameType(String keyspaceName, String typeName, String newTypeName) {
Map<String,CqlType> typesInKeyspace = types.get(keyspaceName);
CqlType cqlType = typesInKeyspace.remove(typeName);
cqlType.setName(newTypeName);
typesInKeyspace.put(newTypeName,cqlType);
}
}

19
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModelBuilder.java
View File

@ -22,26 +22,37 @@ import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.misc.Interval;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.ParseTree;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.List;
public class CqlModelBuilder extends CqlParserBaseListener {
private final static Logger logger = LogManager.getLogger(CqlModelBuilder.class);
private final CQBErrorListener errorListener;
private final CGErrorListener errorListener;
private final CqlModel model;
private long counted;
public CqlModelBuilder(CQBErrorListener errorListener) {
public CqlModelBuilder(CGErrorListener errorListener) {
this.errorListener = errorListener;
this.model = new CqlModel(errorListener);
}
@Override
public void exitEveryRule(ParserRuleContext ctx) {
if ((counted++&0b11111111111111)==0b10000000000000) {
logger.trace("parsed " + counted + " elements...");
}
}
@Override
public void visitErrorNode(ErrorNode node) {
System.out.println("error parsing: " + node.toString());
ParseTree parent = node.getParent();
String errorNodeType = parent.getClass().getSimpleName();
System.out.println("error type: " + errorNodeType);
System.out.println("source interval: " + node.getSourceInterval());
// System.out.println("error type: " + errorNodeType);
// System.out.println("source interval: " + node.getSourceInterval());
super.visitErrorNode(node);
}

35
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlTable.java
View File

@ -18,15 +18,20 @@ package io.nosqlbench.converters.cql.cqlast;
import io.nosqlbench.api.config.NBNamedElement;
import io.nosqlbench.api.labels.Labeled;
import io.nosqlbench.converters.cql.exporters.CGTableStats;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
public class CqlTable implements NBNamedElement, Labeled {
String name = "";
String keyspace = "";
List<CqlColumnDef> coldefs = new ArrayList<>();
Map<String, String> tableAttributes = new HashMap<String, String>();
CGTableStats tableAttributes = null;
List<String> partitionKeys = new ArrayList<>();
List<String> clusteringColumns = new ArrayList<>();
private String refddl;
@ -34,11 +39,11 @@ public class CqlTable implements NBNamedElement, Labeled {
public CqlTable() {
}
public Map<String, String> getTableAttributes() {
public CGTableStats getTableAttributes() {
return tableAttributes;
}
public void setTableAttributes(Map<String, String> tableAttributes) {
public void setTableAttributes(CGTableStats tableAttributes) {
this.tableAttributes = tableAttributes;
}
@ -74,11 +79,18 @@ public class CqlTable implements NBNamedElement, Labeled {
return this.name;
}
public void setKeyspace(String keyspace) {
this.keyspace = keyspace;
public void setKeyspace(String newKsName) {
for (CqlColumnDef coldef : coldefs) {
coldef.setKeyspace(keyspace);
if (coldef.getDefinitionDdl()!=null) {
coldef.setDefinitionRefDdl(coldef.getDefinitionDdl().replaceAll(keyspace,newKsName));
}
}
if (this.refddl!=null) {
this.refddl = this.refddl.replaceAll(this.keyspace,newKsName);
}
this.keyspace = newKsName;
}
public String getRefDdl() {
@ -89,10 +101,6 @@ public class CqlTable implements NBNamedElement, Labeled {
this.refddl = refddl;
}
public String getRefddl() {
return refddl;
}
public String getKeySpace() {
return this.keyspace;
@ -134,6 +142,13 @@ public class CqlTable implements NBNamedElement, Labeled {
return def.orElseThrow();
}
public void renameColumns(Function<String,String> renamer) {
for (CqlColumnDef coldef : coldefs) {
coldef.setName(renamer.apply(coldef.getName()));
}
}
public List<CqlColumnDef> getNonKeyColumnDefinitions() {
return coldefs.stream()
.filter(n -> !partitionKeys.contains(n.getName()))

11
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlType.java
View File

@ -27,8 +27,11 @@ public class CqlType implements NBNamedElement {
private String refddl;
private final Map<String,String> fields = new LinkedHashMap<>();
public void setKeyspace(String keyspace) {
this.keyspace = keyspace;
public void setKeyspace(String newksname) {
this.keyspace = newksname;
if (refddl!=null) {
this.refddl = this.refddl.replaceAll(this.keyspace,newksname);
}
}
public void setName(String name) {
this.name = name;
@ -53,4 +56,8 @@ public class CqlType implements NBNamedElement {
public Map<String, String> getFields() {
return fields;
}
public String getRefDdl() {
return this.refddl;
}
}

4
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/DefaultCqlBindings.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CGDefaultCqlBindings.java
View File

@ -29,11 +29,11 @@ import java.io.InputStream;
import java.util.Map;
import java.util.Optional;
public class DefaultCqlBindings implements BindingsLibrary {
public class CGDefaultCqlBindings implements BindingsLibrary {
private final Map<String, String> bindings;
public DefaultCqlBindings() {
public CGDefaultCqlBindings() {
String yamlContent = NBIO.all()
.name("bindings")
.extension("yaml", "yml")

8
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/ElementNamer.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CGElementNamer.java
View File

@ -26,7 +26,7 @@ import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class ElementNamer implements Function<Map<String, String>, String> {
public class CGElementNamer implements Function<Map<String, String>, String> {
public final static String _DEFAULT_TEMPLATE = "[PREFIX-][OPTYPE-][KEYSPACE__][TABLE][-DATATYPE]";
@ -41,7 +41,7 @@ public class ElementNamer implements Function<Map<String, String>, String> {
private final String spec;
private final List<Function<String, String>> transformers = new ArrayList<>();
public ElementNamer(String template, List<Function<String,String>> transformers) {
public CGElementNamer(String template, List<Function<String,String>> transformers) {
this.spec = template;
this.transformers.addAll(transformers);
Pattern pattern = Pattern.compile("(?<prefix>[^\\]]+)?\\[(?<section>(?<pre>.*?)(?<name>[A-Z]+)(?<required>!)?(?<post>.*?))?]");
@ -63,11 +63,11 @@ public class ElementNamer implements Function<Map<String, String>, String> {
}
}
public ElementNamer(String template) {
public CGElementNamer(String template) {
this(template, List.of());
}
public ElementNamer() {
public CGElementNamer() {
this(_DEFAULT_TEMPLATE, List.of());
}

12
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlKeyspaceStats.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CGKeyspaceStats.java
View File

@ -19,12 +19,12 @@ package io.nosqlbench.converters.cql.exporters;
import java.util.HashMap;
import java.util.Map;
public class CqlKeyspaceStats {
public class CGKeyspaceStats {
String keyspaceName;
Map<String,String> keyspaceAttributes = new HashMap<String,String>();
Map<String, CqlTableStats> keyspaceTables = new HashMap<String, CqlTableStats>();
Map<String, CGTableStats> keyspaceTables = new HashMap<String, CGTableStats>();
public String getKeyspaceName() {
return keyspaceName;
}
@ -49,19 +49,19 @@ public class CqlKeyspaceStats {
this.keyspaceAttributes.put(attributeName, attributeVal);
}
public Map<String, CqlTableStats> getKeyspaceTables() {
public Map<String, CGTableStats> getKeyspaceTables() {
return keyspaceTables;
}
public CqlTableStats getKeyspaceTable(String tableName) {
public CGTableStats getKeyspaceTable(String tableName) {
return keyspaceTables.get(tableName);
}
public void setKeyspaceTables(Map<String, CqlTableStats> keyspaceTables) {
public void setKeyspaceTables(Map<String, CGTableStats> keyspaceTables) {
this.keyspaceTables = keyspaceTables;
}
public void setKeyspaceTable(String tableName, CqlTableStats tableAttributes) {
public void setKeyspaceTable(String tableName, CGTableStats tableAttributes) {
this.keyspaceTables.put(tableName, tableAttributes);
}

12
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlLiteralFormat.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CGLiteralFormat.java
View File

@ -22,7 +22,7 @@ import java.util.Locale;
import java.util.Optional;
import java.util.function.Function;
public enum CqlLiteralFormat {
public enum CGLiteralFormat {
TEXT(v -> "\""+v+"\""),
ASCII(v -> "\""+v+"\""),
VARCHAR(v -> "\""+v+"\""),
@ -42,23 +42,23 @@ public enum CqlLiteralFormat {
VARINT;
private final Function<String, String> literalFormat;
CqlLiteralFormat() {
CGLiteralFormat() {
this.literalFormat=v->v;
}
CqlLiteralFormat(Function<String,String> modifier) {
CGLiteralFormat(Function<String,String> modifier) {
this.literalFormat = modifier;
}
public static String formatBindType(CqlColumnDef cd, String valueref) {
return CqlLiteralFormat.valueOfCqlType(cd.getTrimmedTypedef()).orElse(CqlLiteralFormat.UNKNOWN).format(valueref);
return CGLiteralFormat.valueOfCqlType(cd.getTrimmedTypedef()).orElse(CGLiteralFormat.UNKNOWN).format(valueref);
}
public String format(String value) {
return this.literalFormat.apply(value);
}
public static Optional<CqlLiteralFormat> valueOfCqlType(String typename) {
for (CqlLiteralFormat value : CqlLiteralFormat.values()) {
public static Optional<CGLiteralFormat> valueOfCqlType(String typename) {
for (CGLiteralFormat value : CGLiteralFormat.values()) {
if (typename.toUpperCase(Locale.ROOT).equals(value.toString().toUpperCase(Locale.ROOT))) {
return Optional.of(value);
}

12
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlSchemaStats.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CGSchemaStats.java
View File

@ -19,22 +19,22 @@ package io.nosqlbench.converters.cql.exporters;
import java.util.HashMap;
import java.util.Map;
public class CqlSchemaStats {
Map<String, CqlKeyspaceStats> keyspaces = new HashMap<String, CqlKeyspaceStats>();
public class CGSchemaStats {
Map<String, CGKeyspaceStats> keyspaces = new HashMap<String, CGKeyspaceStats>();
public Map<String, CqlKeyspaceStats> getKeyspaces() {
public Map<String, CGKeyspaceStats> getKeyspaces() {
return keyspaces;
}
public void setKeyspaces(Map<String, CqlKeyspaceStats> keyspaces) {
public void setKeyspaces(Map<String, CGKeyspaceStats> keyspaces) {
this.keyspaces = keyspaces;
}
public CqlKeyspaceStats getKeyspace(String keyspaceName) {
public CGKeyspaceStats getKeyspace(String keyspaceName) {
return keyspaces.get(keyspaceName);
}
public void setKeyspace(CqlKeyspaceStats keyspace) {
public void setKeyspace(CGKeyspaceStats keyspace) {
this.keyspaces.put(keyspace.getKeyspaceName(), keyspace);
}

6
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlTableStats.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CGTableStats.java
View File

@ -19,7 +19,7 @@ package io.nosqlbench.converters.cql.exporters;
import java.util.HashMap;
import java.util.Map;
public class CqlTableStats {
public class CGTableStats {
String tableName;
Map<String,String> attributes = new HashMap<String,String>();
@ -48,4 +48,8 @@ public class CqlTableStats {
attributes.put(attributeName, attributeVal);
}
public int size() {
return getAttributes().size();
}
}

223
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlWorkloadExporter.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CGWorkloadExporter.java
View File

@ -23,9 +23,7 @@ import io.nosqlbench.converters.cql.cqlast.CqlKeyspace;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlTable;
import io.nosqlbench.converters.cql.exporters.binders.*;
import io.nosqlbench.converters.cql.exporters.transformers.CqlModelFixup;
import io.nosqlbench.converters.cql.exporters.transformers.RatioCalculator;
import io.nosqlbench.converters.cql.exporters.transformers.StatsEnhancer;
import io.nosqlbench.converters.cql.exporters.transformers.CGTransformersInit;
import io.nosqlbench.converters.cql.parser.CqlModelParser;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -35,6 +33,7 @@ import org.snakeyaml.engine.v2.common.FlowStyle;
import org.snakeyaml.engine.v2.common.NonPrintableStyle;
import org.snakeyaml.engine.v2.common.ScalarStyle;
import org.snakeyaml.engine.v2.representer.BaseRepresenter;
import org.yaml.snakeyaml.Yaml;
import java.io.IOException;
import java.nio.file.Files;
@ -42,7 +41,6 @@ import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* The unit of generation is simply everything that is provided to the exporter together.
@ -52,49 +50,39 @@ import java.util.stream.Collectors;
*
* @see <a href="https://cassandra.apache.org/doc/trunk/cassandra/cql/index.html">Apache Cassandra CQL Docs</a>
*/
public class CqlWorkloadExporter {
private final static Logger logger = LogManager.getLogger(CqlWorkloadExporter.class);
public final static String DEFAULT_NAMING_TEMPLATE = "[OPTYPE-][COLUMN-][TYPEDEF-][TABLE!]-[KEYSPACE]";
public static final String DEFAULT_REPLICATION = """
'class': 'SimpleStrategy',
'replication_factor': 'TEMPLATE(rf:1)'
""";
public class CGWorkloadExporter {
private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.class);
private final BindingsLibrary defaultBindings = new DefaultCqlBindings();
private final BindingsLibrary defaultBindings = new CGDefaultCqlBindings();
private final NamingFolio namer = new NamingFolio(DEFAULT_NAMING_TEMPLATE);
private final BindingsAccumulator bindings = new BindingsAccumulator(namer, List.of(defaultBindings));
private NamingFolio namer;
private BindingsAccumulator bindings = new BindingsAccumulator(namer, List.of(defaultBindings));
private CqlModel model;
private final Map<String, String> bindingsMap = new LinkedHashMap<>();
private final int DEFAULT_RESOLUTION = 10000;
String replication;
String namingTemplate;
private List<String> includedKeyspaces;
private double partitionMultiplier;
public CqlWorkloadExporter(CqlModel model, List<Function<CqlModel, CqlModel>> transformers) {
public CGWorkloadExporter(CqlModel model, CGTransformersInit transformers) {
this.model = model;
for (Function<CqlModel, CqlModel> transformer : transformers) {
this.model = transformer.apply(this.model);
for (Function<CqlModel, CqlModel> transformer : transformers.get()) {
CqlModel modified = transformer.apply(this.model);
model = modified;
}
}
public CqlWorkloadExporter(String ddl, Path srcpath, List<Function<CqlModel, CqlModel>> transformers) {
this.model = CqlModelParser.parse(ddl, srcpath);
for (Function<CqlModel, CqlModel> transformer : transformers) {
this.model = transformer.apply(this.model);
}
public CGWorkloadExporter(String ddl, Path srcpath, CGTransformersInit transformers) {
this(CqlModelParser.parse(ddl, srcpath), transformers);
}
public CqlWorkloadExporter(String ddl, List<Function<CqlModel, CqlModel>> transformers) {
this.model = CqlModelParser.parse(ddl, null);
for (Function<CqlModel, CqlModel> transformer : transformers) {
this.model = transformer.apply(this.model);
}
public CGWorkloadExporter(String ddl, CGTransformersInit transformers) {
this(ddl, null, transformers);
}
public CqlWorkloadExporter(Path path, List<Function<CqlModel, CqlModel>> transformers) {
this.model = CqlModelParser.parse(path);
for (Function<CqlModel, CqlModel> transformer : transformers) {
this.model = transformer.apply(this.model);
}
public CGWorkloadExporter(Path path, CGTransformersInit transformers) {
this(CqlModelParser.parse(path), transformers);
}
public static void main(String[] args) {
@ -128,40 +116,61 @@ public class CqlWorkloadExporter {
throw new RuntimeException("Target file '" + target + "' exists. Please remove it first or use a different target file name.");
}
CqlSchemaStats schemaStats = null;
if (args.length == 3) {
Path statspath = Path.of(args[2]);
Yaml yaml = new Yaml();
Path cfgpath = Path.of("exporter.yaml");
CGWorkloadExporter exporter;
if (Files.exists(cfgpath)) {
try {
CqlSchemaStatsParser parser = new CqlSchemaStatsParser();
schemaStats = parser.parse(statspath);
CGTransformersInit transformers = new CGTransformersInit();
String configfile = Files.readString(cfgpath);
Map cfgmap = yaml.loadAs(configfile, Map.class);
if (cfgmap.containsKey("model_transformers")) {
transformers.accept((List<Map<String, ?>>) cfgmap.get("model_transformers"));
}
exporter = new CGWorkloadExporter(srcpath, transformers);
String defaultNamingTemplate = cfgmap.get("naming_template").toString();
exporter.setNamingTemplate(defaultNamingTemplate);
String partition_multipler = cfgmap.get("partition_multiplier").toString();
exporter.setPartitionMultiplier(Double.parseDouble(partition_multipler));
String workload = exporter.getWorkloadAsYaml();
try {
Files.writeString(
target,
workload,
StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING
);
} catch (IOException e) {
e.printStackTrace();
}
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
List<Function<CqlModel, CqlModel>> transformers = List.of(
new CqlModelFixup(), // elide UDTs in lieu of blobs for now
new StatsEnhancer(schemaStats), // add keyspace, schema and table stats from nodetool
new RatioCalculator() // Normalize read and write fraction over total ops in unit interval
);
CqlWorkloadExporter exporter = new CqlWorkloadExporter(srcpath, transformers);
String workload = exporter.getWorkloadAsYaml();
try {
Files.writeString(
target,
workload,
StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING
);
} catch (IOException e) {
e.printStackTrace();
}
}
private void setPartitionMultiplier(double multipler) {
this.partitionMultiplier = multipler;
}
private void setIncludedKeyspaces(List<String> includeKeyspaces) {
this.includedKeyspaces = includeKeyspaces;
}
public void setNamingTemplate(String namingTemplate) {
this.namingTemplate = namingTemplate;
this.namer = new NamingFolio(namingTemplate);
this.bindings = new BindingsAccumulator(namer, List.of(defaultBindings));
}
public Map<String, Object> getWorkload() {
namer.populate(model);
namer.informNamerOfAllKnownNames(model);
Map<String, Object> workload = new LinkedHashMap<>();
workload.put("description", "Auto-generated workload from source schema.");
@ -169,7 +178,9 @@ public class CqlWorkloadExporter {
workload.put("bindings", bindingsMap);
Map<String, Object> blocks = new LinkedHashMap<>();
workload.put("blocks", blocks);
blocks.put("schema", genSchemaBlock(model));
blocks.put("schema-keyspaces", genKeyspacesSchemaBlock(model));
blocks.put("schema-tables", genTablesSchemaBlock(model));
blocks.put("schema-types", genTypesSchemaBlock(model));
blocks.put("truncate", genTruncateBlock(model));
blocks.put("rampup", genRampupBlock(model));
blocks.put("main", genMainBlock(model));
@ -177,10 +188,17 @@ public class CqlWorkloadExporter {
return workload;
}
private Map<String, Object> genTypesSchemaBlock(CqlModel model) {
return Map.of();
}
private Map<String, Object> genScenarios(CqlModel model) {
return Map.of(
"default", Map.of(
"schema", "run driver=cql tags=block:schema threads===UNDEF cycles===UNDEF",
"schema", "run driver=cql tags=block:'schema-.*' threads===UNDEF cycles===UNDEF",
"schema-keyspaces", "run driver=cql tags=block:'schema-keyspaces' threads===UNDEF cycles===UNDEF",
"schema-tables", "run driver=cql tags=block:'schema-tables' threads===UNDEF cycles===UNDEF",
"schema-types", "run driver=cql tags=block:'schema-types' threads===UNDEF cycles===UNDEF",
"rampup", "run driver=cql tags=block:rampup threads=auto cycles===TEMPLATE(rampup-cycles,10000)",
"main", "run driver=cql tags=block:main threads=auto cycles===TEMPLATE(main-cycles,10000)"
),
@ -191,7 +209,7 @@ public class CqlWorkloadExporter {
private Map<String, Object> genMainBlock(CqlModel model) {
Map<String, Object> mainOpTemplates = new LinkedHashMap<>();
for (CqlTable table : model.getTables()) {
for (CqlTable table : model.getTableDefs()) {
if (!isCounterTable(table)) {
@ -226,7 +244,8 @@ public class CqlWorkloadExporter {
Optional<String> selectTemplate = this.genSelectTemplate(table);
if (selectTemplate.isPresent()) {
mainOpTemplates.put(namer.nameFor(table, "optype", "select"),
Map.of("stmt", selectTemplate.get(),
Map.of(
"stmt", selectTemplate.get(),
"ratio", readRatioFor(table))
);
} else {
@ -245,18 +264,18 @@ public class CqlWorkloadExporter {
private int readRatioFor(CqlTable table) {
if (table.getTableAttributes().size()==0) {
if (table.getTableAttributes()==null ||table.getTableAttributes().size() == 0) {
return 1;
}
double weighted_reads = Double.parseDouble(table.getTableAttributes().get("weighted_reads"));
double weighted_reads = Double.parseDouble(table.getTableAttributes().getAttribute("weighted_reads"));
return (int) (weighted_reads * DEFAULT_RESOLUTION);
}
private int writeRatioFor(CqlTable table) {
if (table.getTableAttributes().size()==0) {
if (table.getTableAttributes()==null ||table.getTableAttributes().size() == 0) {
return 1;
}
double weighted_writes = Double.parseDouble(table.getTableAttributes().get("weighted_writes"));
double weighted_writes = Double.parseDouble(table.getTableAttributes().getAttribute("weighted_writes"));
return (int) (weighted_writes * DEFAULT_RESOLUTION);
}
@ -265,7 +284,7 @@ public class CqlWorkloadExporter {
Map<String, String> rampupOpTemplates = new LinkedHashMap<>();
for (CqlTable table : model.getTables()) {
for (CqlTable table : model.getTableDefs()) {
if (!isCounterTable(table)) {
Optional<String> insert = genInsertTemplate(table);
if (insert.isPresent()) {
@ -348,26 +367,48 @@ public class CqlWorkloadExporter {
.append(", ");
}
}
sb.setLength(sb.length() - ", ".length());
if (sb.length() > 0) {
sb.setLength(sb.length() - ", ".length());
} else {
logger.debug("no assignments in this?\n" + table.getRefDdl());
}
return sb.toString();
}
private Optional<String> genInsertTemplate(CqlTable table) {
try {
List<CqlColumnDef> cdefs = table.getColumnDefinitions();
return Optional.of("insert into " +
table.getKeySpace() + "." + table.getName() + "\n" +
" ( " + cdefs.stream().map(CqlColumnDef::getName)
.collect(Collectors.joining(" , ")) +
" )\n values\n (" +
cdefs
.stream()
.map(cd -> {
Binding binding = bindings.forColumn(cd);
return "{" + binding.name() + "}";
})
.collect(Collectors.joining(","))
+ ");");
return Optional.of("""
insert into KEYSPACE.TABLE\n
( FIELDNAMES )
VALUES
( BINDINGS );
"""
.replace("KEYSPACE", table.getKeySpace())
.replace("TABLE", table.getName())
.replace("FIELDNAMES",
String.join(", ",
table.getColumnDefinitions().stream()
.map(CqlColumnDef::getName).toList()))
.replaceAll("BINDINGS",
String.join(", ",
table.getColumnDefinitions().stream()
.map(c -> "{" + bindings.forColumn(c).name() + "}").toList())));
//
// List<CqlColumnDef> cdefs = table.getColumnDefinitions();
// return Optional.of("insert into " +
// table.getKeySpace() + "." + table.getName() + "\n" +
// " ( " + cdefs.stream().map(CqlColumnDef::getName)
// .collect(Collectors.joining(" , ")) +
// " )\n values\n (" +
// cdefs
// .stream()
// .map(cd -> {
// Binding binding = bindings.forColumn(cd);
// return "{" + binding.name() + "}";
// })
// .collect(Collectors.joining(","))
// + ");");
} catch (UnresolvedBindingException ube) {
return Optional.empty();
}
@ -412,7 +453,7 @@ public class CqlWorkloadExporter {
Map<String, Object> ops = new LinkedHashMap<>();
truncateblock.put("ops", ops);
for (CqlTable table : model.getTables()) {
for (CqlTable table : model.getTableDefs()) {
ops.put(
namer.nameFor(table, "optype", "truncate"),
"truncate " + table.getKeySpace() + "." + table.getName() + ";"
@ -421,16 +462,25 @@ public class CqlWorkloadExporter {
return truncateblock;
}
private Map<String, Object> genSchemaBlock(CqlModel model) {
private Map<String, Object> genKeyspacesSchemaBlock(CqlModel model) {
Map<String, Object> schemablock = new LinkedHashMap<>();
Map<String, Object> ops = new LinkedHashMap<>();
for (CqlKeyspace ks : model.getKeyspacesByName().values()) {
ops.put("create-keyspace-" + ks.getName(), ks.getRefDdlWithReplFields(DEFAULT_REPLICATION));
ops.put("create-keyspace-" + ks.getName(), ks.getRefddl() + ";");
}
schemablock.put("ops", ops);
return schemablock;
}
private Map<String, Object> genTablesSchemaBlock(CqlModel model) {
Map<String, Object> schemablock = new LinkedHashMap<>();
Map<String, Object> ops = new LinkedHashMap<>();
for (String ksname : model.getTablesByNameByKeyspace().keySet()) {
for (CqlTable cqltable : model.getTablesByNameByKeyspace().get(ksname).values()) {
ops.put("create-table-" + ksname + "." + cqltable.getName(), cqltable.getRefDdl());
ops.put("create-table-" + ksname + "." + cqltable.getName(), cqltable.getRefDdl() + ":");
}
}
@ -438,4 +488,5 @@ public class CqlWorkloadExporter {
return schemablock;
}
}

12
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/binders/NamingFolio.java
View File

@ -19,7 +19,7 @@ package io.nosqlbench.converters.cql.exporters.binders;
import io.nosqlbench.converters.cql.cqlast.CqlColumnDef;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlTable;
import io.nosqlbench.converters.cql.exporters.ElementNamer;
import io.nosqlbench.converters.cql.exporters.CGElementNamer;
import io.nosqlbench.api.labels.Labeled;
import java.util.*;
@ -41,19 +41,19 @@ import java.util.*;
public class NamingFolio {
private final Map<String, Labeled> graph = new LinkedHashMap<>();
private final ElementNamer namer;
private final CGElementNamer namer;
public final static String DEFAULT_NAMER_SPEC = "[COLUMN]-[TYPEDEF-][TABLE][-KEYSPACE]";
NamingStyle namingStyle = NamingStyle.SymbolicType;
public NamingFolio(String namerspec) {
this.namer = new ElementNamer(
this.namer = new CGElementNamer(
namerspec,
List.of(s -> s.toLowerCase().replaceAll("[^a-zA-Z0-9_-]", ""))
);
}
public NamingFolio() {
this.namer = new ElementNamer(DEFAULT_NAMER_SPEC);
this.namer = new CGElementNamer(DEFAULT_NAMER_SPEC);
}
public void addFieldRef(Map<String, String> labels) {
@ -76,8 +76,8 @@ public class NamingFolio {
}
public void populate(CqlModel model) {
for (CqlTable table : model.getTables()) {
public void informNamerOfAllKnownNames(CqlModel model) {
for (CqlTable table : model.getTableDefs()) {
for (CqlColumnDef coldef : table.getColumnDefinitions()) {
addFieldRef(coldef.getLabels());
}

59
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/exporter.yaml Normal file
View File

@ -0,0 +1,59 @@
text_transformers:
- class: CGRegexReplacer
config:
replacers:
- /a/b/
model_transformers:
# filters in or out keyspaces
- class: CGKeyspaceFilter
config:
include:
- dba_info
- vzw_order
- mcs_or_prod
- prod_parallel_test
- vzw_common
- vzw_soe
# replaces the replication settings with the provided values here,
# specifed as a text block to be put inside the curly braces
- class: CGReplicationSettingInjector
config:
replication_fields: |
'class': 'SimpleStrategy',
'replication_factor': 'TEMPLATE(rf:1)'
# This is more appropriately handled in the scenario selections, which now
# contain all or partial schema blocks
# # Removes Keyspace DDL statements
# - class: CGKeySpaceDDLRemover
# Adds IF NOT EXIST to all DDL
- class: CGIfNotExistsInjector
# Replaces UDTs with blobs until we have full UDT generation capability
- class: CGUdtReplacer
# Reads a configured file path containing nodetool histogram stats output
- class: CGGenStatsInjector
config:
path: histogram
# Uses nodetool histogram stats to weight reads and writes over all ops
- class: CGRatioCalculator
# This needs support of the AST, without relying on the refddl text images
# In other words, it doesn't work reliably with the substitution methods
# Don't enable it until the next revision
# # replaces names of keyspaces, tables, and columns with generated values
# - class: CGNameObfuscator
naming_template: "[OPTYPE-][COLUMN-][TYPEDEF-][TABLE!]-[KEYSPACE]"
partition_multiplier: 30

21
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/NameRemapper.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/CGCachingNameRemapper.java
View File

@ -17,30 +17,45 @@
package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.api.config.NBNamedElement;
import io.nosqlbench.converters.cql.cqlast.CqlTable;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_string.Combinations;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Function;
import java.util.function.LongFunction;
public class NameRemapper {
public class CGCachingNameRemapper {
private final LongFunction<String> namefunc;
private final Map<String,String> remapped = new HashMap<>();
private long index=0;
public NameRemapper() {
public CGCachingNameRemapper() {
this.namefunc = new Combinations("a-z;a-z;a-z;a-z;a-z;a-z;");
}
public NameRemapper(LongFunction<String> function) {
public CGCachingNameRemapper(LongFunction<String> function) {
this.namefunc = function;
}
public synchronized String nameForType(String type, String originalName) {
String canonical = type+"__"+originalName;
return getOrCreateName(canonical);
}
public synchronized String nameFor(NBNamedElement element) {
String canonical = element.getClass().getSimpleName()+"--"+element.getName();
return getOrCreateName(canonical);
}
private String getOrCreateName(String canonical) {
if (!remapped.containsKey(canonical)) {
String newname = namefunc.apply(index++);
remapped.put(canonical,newname);
}
return remapped.get(canonical);
}
public Function<String, String> mapperForType(CqlTable cqlTable) {
return in -> this.nameForType(cqlTable.getClass().getSimpleName(),in);
}
}

58
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/CGGenStatsInjector.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.exporters.CGSchemaStats;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Map;
public class CGGenStatsInjector implements CGModelTransformer, CGTransformerConfigType {
private CGSchemaStats schemaStats = null;
public CGGenStatsInjector() {
}
@Override
public CqlModel apply(CqlModel model) {
if (schemaStats != null) {
model.setKeyspaceAttributes(schemaStats);
}
return model;
}
@Override
public void accept(Map<String, ?> config) {
String histogramPath = config.get("path").toString();
if (histogramPath != null) {
CGSchemaStats schemaStats = null;
Path statspath = Path.of(histogramPath);
try {
CqlSchemaStatsParser parser = new CqlSchemaStatsParser();
schemaStats = parser.parse(statspath);
this.schemaStats = schemaStats;
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
} else schemaStats = null;
}
}

52
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/CGIfNotExistsInjector.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.converters.cql.cqlast.CqlKeyspace;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlTable;
import io.nosqlbench.converters.cql.cqlast.CqlType;
public class CGIfNotExistsInjector implements CGModelTransformer {
@Override
public CqlModel apply(CqlModel model) {
for (CqlKeyspace keyspace : model.getKeyspaceDefs()) {
keyspace.setRefDdl(keyspace.getRefddl().replaceAll(
"(?m)(?s)(?i)(\\s*CREATE (TABLE|KEYSPACE|TYPE) +)(?!IF NOT EXISTS)",
"$1IF NOT EXISTS "
));
}
for (CqlTable table : model.getTableDefs()) {
String refddl = table.getRefDdl();
String replaced = refddl.replaceAll(
"(?m)(?s)(?i)(\\s*CREATE (TABLE|KEYSPACE|TYPE) +)(?!IF NOT EXISTS)",
"$1IF NOT EXISTS "
);
table.setRefDdl(replaced);
}
for (CqlType type : model.getTypes()) {
type.setRefddl(type.getRefDdl().replaceAll(
"(?m)(?s)(?i)(\\s*CREATE (TABLE|KEYSPACE|TYPE) +)(?!IF NOT EXISTS)",
"$1IF NOT EXISTS "
));
}
return model;
}
}

35
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/CGKeySpaceDDLRemover.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.converters.cql.cqlast.CqlKeyspace;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import java.util.List;
public class CGKeySpaceDDLRemover implements CGModelTransformer {
private List<String> includes;
@Override
public CqlModel apply(CqlModel model) {
for (CqlKeyspace keyspace : model.getKeyspaceDefs()) {
model.removeKeyspaceDef(keyspace.getName());
}
return model;
}
}

63
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/CGKeyspaceFilter.java Normal file
View File

@ -0,0 +1,63 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.converters.cql.cqlast.CqlKeyspace;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfigType {
private List<Pattern> patterns = List.of(Pattern.compile(".*"));
private final static Logger logger = LogManager.getLogger(CGKeyspaceFilter.class);
@Override
public CqlModel apply(CqlModel model) {
for (CqlKeyspace keyspace : model.getKeyspaceDefs()) {
boolean included = false;
for (Pattern pattern : patterns) {
if (pattern.matcher(keyspace.getName()).matches()) {
included=true;
break;
}
}
if (!included) {
logger.info("removing keyspaces, tables and types for non-included keyspace '" + keyspace.getName() +"'");
model.removeKeyspaceDef(keyspace.getName());
model.removeTablesForKeyspace(keyspace.getName());
model.removeTypesForKeyspace(keyspace.getName());
} else {
logger.info("including keyspace '" + keyspace.getName()+"'");
}
}
return model;
}
@Override
public void accept(Map<String, ?> cfgmap) {
List<String> includes = (List<String>) cfgmap.get("include");
this.patterns = includes.stream()
.map(Pattern::compile)
.toList();
}
}

26
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/CGModelTransformer.java Normal file
View File

@ -0,0 +1,26 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import java.util.function.Function;
public interface CGModelTransformer extends Function<CqlModel,CqlModel> {
@Override
CqlModel apply(CqlModel model);
}

60
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/CGNameObfuscator.java Normal file
View File

@ -0,0 +1,60 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Create random but stable names for all elements.
* Use a deterministic method for creating names.
* once an element is named, use the same name throughout
* prefix each element type with a code for the type
*/
package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlTable;
import io.nosqlbench.converters.cql.cqlast.CqlType;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public class CGNameObfuscator implements CGModelTransformer {
private final static Logger logger = LogManager.getLogger(CGNameObfuscator.class);
private final CGCachingNameRemapper remapper = new CGCachingNameRemapper();
private Object keyspaceName;
@Override
public CqlModel apply(CqlModel model) {
for (String keyspaceName : model.getAllKnownKeyspaceNames()) {
String newKeyspaceName = remapper.nameForType("keyspace",keyspaceName);
model.renamekeyspace(keyspaceName,newKeyspaceName);
}
for (CqlTable cqlTable : model.getTableDefs()) {
String tablename = cqlTable.getName();
String newTableName = remapper.nameFor(cqlTable);
model.renameTable(cqlTable.getKeySpace(), tablename, newTableName);
cqlTable.renameColumns(remapper.mapperForType(cqlTable));
}
for (CqlType type : model.getTypes()) {
String typeName = type.getName();
String newTypeName = remapper.nameFor(type);
model.renameType(type.getKeyspace(),typeName,newTypeName);
}
return model;
}
}

30
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/RatioCalculator.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/CGRatioCalculator.java
View File

@ -18,14 +18,14 @@ package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlTable;
import io.nosqlbench.converters.cql.exporters.CGTableStats;
import java.util.function.Function;
public class RatioCalculator implements Function<CqlModel,CqlModel> {
public class CGRatioCalculator implements CGModelTransformer {
@Override
public CqlModel apply(CqlModel model) {
if (!model.hasStats()) {
// TODO: True this up
return model;
}
double totalReads = 0.0d;
@ -33,30 +33,34 @@ public class RatioCalculator implements Function<CqlModel,CqlModel> {
double totalSpace = 0.0d;
double totalOps=0.0d;
for (CqlTable table : model.getTables()) {
String local_read_count = table.getTableAttributes().get("Local read count");
for (CqlTable table : model.getTableDefs()) {
CGTableStats tableAttributes = table.getTableAttributes();
if (tableAttributes==null) {
continue;
}
String local_read_count = tableAttributes.getAttribute("Local read count");
double reads = Double.parseDouble(local_read_count);
totalReads+=reads;
totalOps+=reads;
String local_write_count = table.getTableAttributes().get("Local write count");
String local_write_count = table.getTableAttributes().getAttribute("Local write count");
double writes = Double.parseDouble(local_write_count);
totalWrites += writes;
totalOps+=writes;
String space_used_total = table.getTableAttributes().get("Space used (total)");
String space_used_total = table.getTableAttributes().getAttribute("Space used (total)");
double space = Double.parseDouble(space_used_total);
totalSpace+=space;
}
for (CqlTable table : model.getTables()) {
double reads = Double.parseDouble(table.getTableAttributes().get("Local read count"));
double writes = Double.parseDouble(table.getTableAttributes().get("Local write count"));
for (CqlTable table : model.getTableDefs()) {
double reads = Double.parseDouble(table.getTableAttributes().getAttribute("Local read count"));
double writes = Double.parseDouble(table.getTableAttributes().getAttribute("Local write count"));
table.getTableAttributes().put("weighted_reads", String.valueOf(reads / totalOps));
table.getTableAttributes().put("weighted_writes", String.valueOf(writes / totalOps));
table.getTableAttributes().setAttribute("weighted_reads", String.valueOf(reads / totalOps));
table.getTableAttributes().setAttribute("weighted_writes", String.valueOf(writes / totalOps));
table.getTableAttributes().put("weighted_space", String.valueOf(Double.parseDouble(table.getTableAttributes().get("Space used (total)")) / totalReads));
table.getTableAttributes().setAttribute("weighted_space", String.valueOf(Double.parseDouble(table.getTableAttributes().getAttribute("Space used (total)")) / totalReads));
}
return model;

41
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/CGReplicationSettingInjector.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.converters.cql.cqlast.CqlKeyspace;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import java.util.Map;
public class CGReplicationSettingInjector implements CGModelTransformer, CGTransformerConfigType {
private String replicationFields;
@Override
public CqlModel apply(CqlModel model) {
for (CqlKeyspace keyspace : model.getKeyspaceDefs()) {
keyspace.setRefReplDdl(this.replicationFields);
}
return model;
}
@Override
public void accept(Map<String, ?> stringMap) {
if (stringMap.containsKey("replication_fields")) {
this.replicationFields = stringMap.get("replication_fields").toString();
}
}
}

23
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/CGTransformerConfigType.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters.transformers;
import java.util.Map;
import java.util.function.Consumer;
public interface CGTransformerConfigType extends Consumer<Map<String, ?>> {
}

87
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/CGTransformersInit.java Normal file
View File

@ -0,0 +1,87 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters.transformers;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Supplier;
public class CGTransformersInit implements Consumer<List<Map<String, ?>>>, Supplier<List<CGModelTransformer>> {
private final static Logger logger = LogManager.getLogger(CGTransformersInit.class);
private List<CGModelTransformer> transformers = new ArrayList<>();
public CGTransformersInit() {
}
@Override
public void accept(List<Map<String, ?>> configs) {
List<CGModelTransformer> transformers = new ArrayList<>();
for (Map<String, ?> cfgmap : configs) {
// Instantiate Transformer
String classname = cfgmap.get("class").toString();
if (!classname.contains(".")) {
String newname = CGNameObfuscator.class.getPackageName() + "." + classname;
logger.info("qualified transformer '" + classname + "' as '" + newname + "'");
classname = newname;
}
Class<?> txclass = null;
CGModelTransformer transformer = null;
try {
txclass = Class.forName(classname);
Constructor<?> ctor = txclass.getConstructor();
Object instance = ctor.newInstance();
if (instance instanceof CGModelTransformer t) {
transformer = t;
} else {
throw new RuntimeException("Object " + instance.getClass().getName() + " is not a " + CGModelTransformer.class.getName());
}
} catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException e) {
throw new RuntimeException(e);
}
// Configure Transformer IFF ...
if (transformer instanceof CGTransformerConfigType configurable) {
Object cfgvalues = cfgmap.get("config");
if (cfgvalues instanceof Map txconfigmap) {
configurable.accept((txconfigmap));
logger.info("configured transformer with " + txconfigmap);
}
}
transformers.add(transformer);
}
this.transformers.addAll(transformers);
}
@Override
public List<CGModelTransformer> get() {
return this.transformers;
}
}

11
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/CqlModelFixup.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/CGUdtReplacer.java
View File

@ -21,23 +21,22 @@ import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlTable;
import java.util.List;
import java.util.function.Function;
public class CqlModelFixup implements Function<CqlModel,CqlModel> {
public class CGUdtReplacer implements CGModelTransformer {
@Override
public CqlModel apply(CqlModel model) {
List<String> toReplace = model.getTypes().stream().map(t -> t.getKeyspace() + "." + t.getName()).toList();
for (CqlTable table : model.getTables()) {
for (CqlTable table : model.getTableDefs()) {
for (CqlColumnDef coldef : table.getColumnDefinitions()) {
String coldefDdl = coldef.getRefddl();
String coldefDdl = coldef.getDefinitionDdl();
for (String searchFor : toReplace) {
if (coldefDdl.contains(searchFor)) {
String typedef = coldef.getType();
coldef.setType("blob");
String replaced = coldef.getRefddl().replace(typedef, "blob");
String replaced = coldef.getDefinitionDdl().replace(typedef, "blob");
coldef.setDefinitionRefDdl(replaced);
table.setRefDdl(table.getRefddl().replace(typedef,"blob"));
table.setRefDdl(table.getRefDdl().replace(typedef,"blob"));
}
}
}

19
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlSchemaStatsParser.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/CqlSchemaStatsParser.java
View File

@ -14,8 +14,11 @@
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters;
package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.converters.cql.exporters.CGKeyspaceStats;
import io.nosqlbench.converters.cql.exporters.CGSchemaStats;
import io.nosqlbench.converters.cql.exporters.CGTableStats;
import org.apache.commons.math4.util.Pair;
import java.io.BufferedReader;
@ -28,12 +31,12 @@ public class CqlSchemaStatsParser {
private static final String KEYSPACE = "Keyspace";
private static final String TABLE = "Table";
CqlSchemaStats stats = null;
CqlKeyspaceStats currentKeyspace = null;
CqlTableStats currentTable = null;
CGSchemaStats stats = null;
CGKeyspaceStats currentKeyspace = null;
CGTableStats currentTable = null;
public CqlSchemaStats parse(Path statspath) throws IOException {
this.stats = new CqlSchemaStats();
public CGSchemaStats parse(Path statspath) throws IOException {
this.stats = new CGSchemaStats();
BufferedReader reader = Files.newBufferedReader(statspath);
String currentLine = reader.readLine(); //ignore 1st line
while((currentLine = reader.readLine()) != null) {
@ -66,7 +69,7 @@ public class CqlSchemaStatsParser {
private boolean evalForTable(String currentLine) {
if (currentLine.startsWith(TABLE)) {
writeCurrentTable();
currentTable = new CqlTableStats();
currentTable = new CGTableStats();
currentTable.setTableName(currentLine.split(":")[1].trim());
return true;
}
@ -77,7 +80,7 @@ public class CqlSchemaStatsParser {
if (currentLine.startsWith(KEYSPACE)) {
writeCurrentTable();
writeCurrentKeyspace();
currentKeyspace = new CqlKeyspaceStats();
currentKeyspace = new CGKeyspaceStats();
currentKeyspace.setKeyspaceName(currentLine.split(":")[1].trim());
currentTable = null;
return true;

61
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/Obfuscator.java
View File

@ -1,61 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Create random but stable names for all elements.
* Use a deterministic method for creating names.
* once an element is named, use the same name throughout
* prefix each element type with a code for the type
*/
package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.converters.cql.cqlast.CqlColumnDef;
import io.nosqlbench.converters.cql.cqlast.CqlKeyspace;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlTable;
import java.util.function.Function;
public class Obfuscator implements Function<CqlModel,CqlModel> {
private final NameRemapper remapper = new NameRemapper();
@Override
public CqlModel apply(CqlModel model) {
for (CqlKeyspace keyspace : model.getKeyspaces()) {
String ksname = keyspace.getName();
String ksnewname = remapper.nameFor(keyspace);
keyspace.setKeyspaceName(ksnewname);
keyspace.setRefDdl();
for (CqlTable cqlTable : model.getTablesForKeyspace(keyspace.getName())) {
String tablename = cqlTable.getName();
String tbnewname = remapper.nameFor(cqlTable);
cqlTable.setName(tbnewname);
for (CqlColumnDef coldef : cqlTable.getColumnDefinitions()) {
String colname = coldef.getName();
String replacement = remapper.nameFor(coldef);
coldef.setName(replacement);
}
}
}
}
}

55
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/StatsEnhancer.java
View File

@ -1,55 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlTable;
import io.nosqlbench.converters.cql.exporters.CqlKeyspaceStats;
import io.nosqlbench.converters.cql.exporters.CqlSchemaStats;
import java.util.Map;
import java.util.function.Function;
public class StatsEnhancer implements Function<CqlModel, CqlModel> {
private final CqlSchemaStats schemaStats;
public StatsEnhancer(CqlSchemaStats schemaStats) {
this.schemaStats = schemaStats;
}
@Override
public CqlModel apply(CqlModel model) {
if (schemaStats != null) {
//TODO: rewrite this in something resembling an efficient way
CqlKeyspaceStats ksStats = null;
for (String ksName : model.getKeyspacesByName().keySet()) {
if ((ksStats = schemaStats.getKeyspace(ksName)) != null) {
model.getKeyspacesByName().get(ksName).setKeyspaceAttributes(ksStats.getKeyspaceAttributes());
Map<String, CqlTable> ksTables = model.getTablesByNameByKeyspace().get(ksName);
for (String tableName : ksTables.keySet()) {
if (ksStats.getKeyspaceTable(tableName) != null) {
model.getTablesByNameByKeyspace().get(ksName).get(tableName)
.setTableAttributes(ksStats.getKeyspaceTable(tableName).getAttributes());
}
}
}
}
}
return model;
}
}

2
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/grammars/CqlLexer.g4
View File

@ -219,7 +219,7 @@ fragment HEX_DIGIT: [0-9A-F];
fragment DEC_DIGIT: [0-9];
fragment EXPONENT_NUM_PART: 'E' '-'? DEC_DIGIT+;
fragment EXPONENT_NUM_PART: 'E' ('-'|'+') ? DEC_DIGIT+;

1
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/grammars/CqlParser.g4
View File

@ -755,6 +755,7 @@ decimalLiteral
floatLiteral
: DECIMAL_LITERAL
| FLOAT_LITERAL
| REAL_LITERAL
;
stringLiteral

10
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/parser/CqlModelParser.java
View File

@ -16,7 +16,7 @@
package io.nosqlbench.converters.cql.parser;
import io.nosqlbench.converters.cql.cqlast.CQBErrorListener;
import io.nosqlbench.converters.cql.cqlast.CGErrorListener;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlModelBuilder;
import io.nosqlbench.converters.cql.cqlast.CqlType;
@ -41,7 +41,11 @@ public class CqlModelParser {
public static CqlModel parse(Path path) {
try {
String ddl = Files.readString(path);
return parse(ddl, path);
logger.info("read " + ddl.length() + " character DDL file, parsing");
CqlModel parsed = parse(ddl, path);
logger.info("parsed cql model: " + parsed.getSummaryLine());
return parsed;
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -60,7 +64,7 @@ public class CqlModelParser {
try {
CodePointCharStream cstream = CharStreams.fromString(input);
CQBErrorListener errorListener = new CQBErrorListener(origin);
CGErrorListener errorListener = new CGErrorListener(origin);
CqlLexer lexer = new CqlLexer(cstream);
CommonTokenStream tokens = new CommonTokenStream(lexer);

9
adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/cql/parser/CqlParserHarnessTest.java
View File

@ -16,13 +16,13 @@
package io.nosqlbench.converters.cql.cql.parser;
import io.nosqlbench.converters.cql.exporters.CqlWorkloadExporter;
import io.nosqlbench.converters.cql.exporters.CGWorkloadExporter;
import io.nosqlbench.converters.cql.exporters.transformers.CGTransformersInit;
import io.nosqlbench.converters.cql.parser.CqlModelParser;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@ -47,7 +47,8 @@ public class CqlParserHarnessTest {
@Test
public void testAllTypes() {
CqlWorkloadExporter exporter = new CqlWorkloadExporter(Path.of("src/test/resources/testschemas/cql_alltypes.cql"), List.of());
CGWorkloadExporter exporter = new CGWorkloadExporter(Path.of("src/test/resources/testschemas/cql_alltypes.cql"), new CGTransformersInit());
exporter.setNamingTemplate("[OPTYPE-][COLUMN-][TYPEDEF-][TABLE!]-[KEYSPACE]");
var data = exporter.getWorkloadAsYaml();
}
@ -55,7 +56,7 @@ public class CqlParserHarnessTest {
@Disabled
@Test
public void testGenBasicWorkload() {
CqlWorkloadExporter exporter = new CqlWorkloadExporter(ddl, List.of());
CGWorkloadExporter exporter = new CGWorkloadExporter(ddl, new CGTransformersInit());
assertThatThrownBy(() -> exporter.getWorkloadAsYaml()).isInstanceOf(RuntimeException.class);
}

16
adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/ElementNamerTest.java → adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/CGElementNamerTest.java
View File

@ -25,30 +25,30 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
public class ElementNamerTest {
public class CGElementNamerTest {
@Test
public void testNonRequiredFields() {
ElementNamer namer = new ElementNamer("[ABC---][,deFGH][__IJ__]");
CGElementNamer namer = new CGElementNamer("[ABC---][,deFGH][__IJ__]");
assertThat(namer.apply(Map.of())).isEqualTo("");
}
@Test
public void testLiteralTweens() {
ElementNamer namer = new ElementNamer("[ABC---]!-23[,deFGH])(*&[__IJ__]");
CGElementNamer namer = new CGElementNamer("[ABC---]!-23[,deFGH])(*&[__IJ__]");
assertThat(namer.apply(Map.of("abc","123"))).isEqualTo("123---!-23)(*&");
}
@Test
public void testPartialFields() {
ElementNamer namer = new ElementNamer("[ABC---][,deFGH][__IJ__]");
CGElementNamer namer = new CGElementNamer("[ABC---][,deFGH][__IJ__]");
assertThat(namer.apply(Map.of("abc", "base"))).isEqualTo("base---");
}
@Test
public void testLabeledFields() {
ElementNamer namer = new ElementNamer("[ABC---][,deFGH][__IJ__]");
CGElementNamer namer = new CGElementNamer("[ABC---][,deFGH][__IJ__]");
Labeled mylabeled = new Labeled() {
@Override
public Map<String, String> getLabels() {
@ -60,7 +60,7 @@ public class ElementNamerTest {
@Test
public void testCasedSectionName() {
ElementNamer namer = new ElementNamer("[ABC---][,deFGH][__IJ__]");
CGElementNamer namer = new CGElementNamer("[ABC---][,deFGH][__IJ__]");
assertThat(namer.apply(
Map.of(
"abc", "1111",
@ -72,7 +72,7 @@ public class ElementNamerTest {
@Test
public void testRequiredFieldsPresent() {
ElementNamer namer = new ElementNamer("[ABC!---!]");
CGElementNamer namer = new CGElementNamer("[ABC!---!]");
Labeled mylabeled = new Labeled() {
@Override
public Map<String, String> getLabels() {
@ -87,7 +87,7 @@ public class ElementNamerTest {
@Test
public void testRequiredFieldsMissing() {
ElementNamer namer = new ElementNamer("[ABC!---!]");
CGElementNamer namer = new CGElementNamer("[ABC!---!]");
Labeled mylabeled = new Labeled() {
@Override
public Map<String, String> getLabels() {

8
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java
View File

@ -43,7 +43,9 @@ public class StatementsLoader {
StmtsDocList layered = new StmtsDocList(rawDocList);
transformer.checkpointAccesses().forEach((k,v) -> {
layered.addTemplateVariable(k,v);
params.remove(k);
if (params.containsKey(k)) {
params.remove(k);
}
});
return layered;
}
@ -59,7 +61,9 @@ public class StatementsLoader {
StmtsDocList layered = new StmtsDocList(rawStmtsDocList);
transformer.checkpointAccesses().forEach((k,v) -> {
layered.addTemplateVariable(k,v);
params.remove(k);
if (params.containsKey(k)) {
params.remove(k);
}
});
return layered;
}

2
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -227,7 +227,7 @@ public class NBCLI implements Function<String[], Integer> {
}
if (args.length > 0 && args[0].toLowerCase().equals("cqlgen")) {
String exporterImpl = "io.nosqlbench.converters.cql.exporters.CqlWorkloadExporter";
String exporterImpl = "io.nosqlbench.converters.cql.exporters.CGWorkloadExporter";
String[] exporterArgs = Arrays.copyOfRange(args, 1, args.length);
try {
Class<?> genclass = Class.forName(exporterImpl);