IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

cqlgen checkpoint: hierarchic model, ref checking, builder handles transients, transformers are named

Browse Source
This commit is contained in:
Jonathan Shook 2022-07-29 20:53:40 -05:00
parent f3c0d001aa
commit fab797fd02
39 changed files with 1104 additions and 551 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/BindingsLibrary.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.cqlgen.api;
import io.nosqlbench.cqlgen.binders.Binding;
import io.nosqlbench.cqlgen.model.CqlColumnDef;
import io.nosqlbench.cqlgen.model.CqlColumnBase;
import java.util.Optional;
@ -26,5 +26,5 @@ import java.util.Optional;
* to map a column definition to a binding function.
*/
public interface BindingsLibrary {
Optional<Binding> resolveBindingsFor(CqlColumnDef def);
Optional<Binding> resolveBindingsFor(CqlColumnBase def);
}

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGModelTransformer.java
View File

@ -16,6 +16,7 @@
package io.nosqlbench.cqlgen.api;
import io.nosqlbench.api.config.NBNamedElement;
import io.nosqlbench.cqlgen.model.CqlModel;
import java.util.function.Function;
@ -25,7 +26,8 @@ import java.util.function.Function;
* The type and order of transformers is important, as one transformer may be responsible
* for preparing the model for one or more downstream transformers.
*/
public interface CGModelTransformer extends Function<CqlModel,CqlModel> {
public interface CGModelTransformer extends Function<CqlModel,CqlModel>, NBNamedElement {
@Override
CqlModel apply(CqlModel model);
void setName(String name);
}

51
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CqlModelInfo.java
View File

@ -1,51 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.api;
import io.nosqlbench.cqlgen.core.CGSchemaStats;
import io.nosqlbench.cqlgen.model.CqlKeyspace;
import io.nosqlbench.cqlgen.model.CqlTable;
import io.nosqlbench.cqlgen.model.CqlType;
import java.util.List;
import java.util.Map;
import java.util.Set;
public interface CqlModelInfo {
CGSchemaStats getStats();
boolean hasStats();
Map<String, CqlKeyspace> getKeyspacesByName();
List<CqlKeyspace> getKeyspaceDefs();
Map<String, Map<String, CqlTable>> getTableDefsByKeyspaceThenTable();
List<CqlTable> getTablesForKeyspace(String ksname);
List<CqlTable> getTableDefs();
Set<String> getAllKnownKeyspaceNames();
List<CqlType> getTypeDefs();
String getSummaryLine();
Map<String, Map<String, CqlType>> getTypesByKeyspaceThenName();
}

12
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/BindingsAccumulator.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.cqlgen.binders;
import io.nosqlbench.cqlgen.api.BindingsLibrary;
import io.nosqlbench.cqlgen.model.CqlColumnDef;
import io.nosqlbench.cqlgen.model.CqlColumnBase;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -42,10 +42,10 @@ public class BindingsAccumulator {
this.libraries = libraries;
}
public Binding forColumn(CqlColumnDef def, String... prefixes) {
public Binding forColumn(CqlColumnBase def, String... prefixes) {
return forColumn(def,Map.of(), prefixes);
}
public Binding forColumn(CqlColumnDef def, Map<String,String> extra, String... prefixes) {
public Binding forColumn(CqlColumnBase def, Map<String,String> extra, String... prefixes) {
for (BindingsLibrary library : libraries) {
Optional<Binding> optionalBinding = switch (namingStyle) {
case FullyQualified -> this.resolveFullyQualifiedBinding(def, extra);
@ -72,11 +72,11 @@ public class BindingsAccumulator {
}
private Optional<Binding> resolvedCondensedBinding(CqlColumnDef def, Map<String,String> extra) {
private Optional<Binding> resolvedCondensedBinding(CqlColumnBase def, Map<String,String> extra) {
throw new RuntimeException("Implement me!");
}
private Optional<Binding> resolveSymbolicBinding(CqlColumnDef def, Map<String,String> extra) {
private Optional<Binding> resolveSymbolicBinding(CqlColumnBase def, Map<String,String> extra) {
for (BindingsLibrary library : libraries) {
Optional<Binding> binding = library.resolveBindingsFor(def);
if (binding.isPresent()) {
@ -90,7 +90,7 @@ public class BindingsAccumulator {
}
private Optional<Binding> resolveFullyQualifiedBinding(CqlColumnDef def, Map<String,String> extra) {
private Optional<Binding> resolveFullyQualifiedBinding(CqlColumnBase def, Map<String,String> extra) {
for (BindingsLibrary library : libraries) {
Optional<Binding> bindingRecipe = library.resolveBindingsFor(def);
if (bindingRecipe.isPresent()) {

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/NamingFolio.java
View File

@ -16,7 +16,7 @@
package io.nosqlbench.cqlgen.binders;
import io.nosqlbench.cqlgen.model.CqlColumnDef;
import io.nosqlbench.cqlgen.model.CqlColumnBase;
import io.nosqlbench.cqlgen.model.CqlModel;
import io.nosqlbench.cqlgen.model.CqlTable;
import io.nosqlbench.cqlgen.core.CGElementNamer;
@ -84,7 +84,7 @@ public class NamingFolio {
public void informNamerOfAllKnownNames(CqlModel model) {
for (CqlTable table : model.getTableDefs()) {
for (CqlColumnDef coldef : table.getColumnDefinitions()) {
for (CqlColumnBase coldef : table.getColumnDefs()) {
addFieldRef(coldef.getLabels());
}
}

6
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/UnresolvedBindingException.java
View File

@ -16,12 +16,12 @@
package io.nosqlbench.cqlgen.binders;
import io.nosqlbench.cqlgen.model.CqlColumnDef;
import io.nosqlbench.cqlgen.model.CqlColumnBase;
public class UnresolvedBindingException extends RuntimeException {
private final CqlColumnDef def;
private final CqlColumnBase def;
public UnresolvedBindingException(CqlColumnDef def) {
public UnresolvedBindingException(CqlColumnBase def) {
this.def = def;
}

10
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGColumnRebinder.java
View File

@ -16,9 +16,10 @@
package io.nosqlbench.cqlgen.core;
import io.nosqlbench.cqlgen.model.CqlColumnDef;
import io.nosqlbench.cqlgen.binders.Binding;
import io.nosqlbench.cqlgen.binders.BindingsAccumulator;
import io.nosqlbench.cqlgen.model.CqlTableColumn;
import io.nosqlbench.cqlgen.model.FieldPosition;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -35,16 +36,15 @@ public class CGColumnRebinder {
this.quantizerDigits = quantizerDigits;
}
public Binding forColumn(CqlColumnDef cdef) {
if (cdef.isPartitionKey()) {
public Binding forColumn(CqlTableColumn cdef) {
if (cdef.getPosition()== FieldPosition.Partitioning) {
return dividedBinding(cdef);
} else {
return accumulator.forColumn(cdef);
}
}
private Binding dividedBinding(CqlColumnDef column) {
private Binding dividedBinding(CqlTableColumn column) {
CGTableStats stats = column.getTable().getTableAttributes();
if (stats == null) {
return accumulator.forColumn(column);

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java
View File

@ -16,7 +16,7 @@
package io.nosqlbench.cqlgen.core;
import io.nosqlbench.cqlgen.model.CqlColumnDef;
import io.nosqlbench.cqlgen.model.CqlColumnBase;
import io.nosqlbench.cqlgen.binders.Binding;
import io.nosqlbench.cqlgen.api.BindingsLibrary;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
@ -58,7 +58,7 @@ public class CGDefaultCqlBindings implements BindingsLibrary {
}
@Override
public Optional<Binding> resolveBindingsFor(CqlColumnDef def) {
public Optional<Binding> resolveBindingsFor(CqlColumnBase def) {
String typedef = def.getTrimmedTypedef();
String recipe = bindings.get(def.getTrimmedTypedef());
Binding optionalBinding = new Binding(typedef, recipe);

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGLiteralFormat.java
View File

@ -16,7 +16,7 @@
package io.nosqlbench.cqlgen.core;
import io.nosqlbench.cqlgen.model.CqlColumnDef;
import io.nosqlbench.cqlgen.model.CqlColumnBase;
import java.util.Locale;
import java.util.Optional;
@ -49,7 +49,7 @@ public enum CGLiteralFormat {
this.literalFormat = modifier;
}
public static String formatBindType(CqlColumnDef cd, String valueref) {
public static String formatBindType(CqlColumnBase cd, String valueref) {
return CGLiteralFormat.valueOfCqlType(cd.getTrimmedTypedef()).orElse(CGLiteralFormat.UNKNOWN).format(valueref);
}

92
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java
View File

@ -126,7 +126,7 @@ public class CGWorkloadExporter implements BundledApp {
Yaml yaml = new Yaml();
CGWorkloadExporter exporter;
Content<?> cqlgencfg = NBIO.local().prefix("cqlgen").name("cqlgen").extension("conf").one();
Content<?> cqlgencfg = NBIO.local().prefix("cqlgen").name("cqlgen").extension("conf").first().orElseThrow();
if (cqlgencfg == null) {
throw new RuntimeException("Unable to load cqlgen.conf");
}
@ -161,13 +161,18 @@ public class CGWorkloadExporter implements BundledApp {
String partition_multipler = cfgmap.get("partition_multiplier").toString();
setPartitionMultiplier(Double.parseDouble(partition_multipler));
configureTimeouts(cfgmap.get("timeouts"));
configureBlocks(cfgmap.get("blockplan"));
configureQuantizerDigits(cfgmap.get("quantizer_digits"));
this.model = CqlModelParser.parse(ddl, srcpath);
List<String> errorlist = model.getReferenceErrors();
if (errorlist.size()>0) {
for (String error : errorlist) {
logger.error(error);
}
throw new RuntimeException("there were " + errorlist.size() + " reference errors in the model.");
}
this.model = modelTransformers.apply(this.model);
String workload = getWorkloadAsYaml();
@ -360,7 +365,7 @@ public class CGWorkloadExporter implements BundledApp {
where PREDICATE
LIMIT;
"""
.replace("KEYSPACE", table.getKeySpace())
.replace("KEYSPACE", table.getKeyspace().getName())
.replace("TABLE", table.getName())
.replace("PREDICATE", genPredicateTemplate(table, -1))
.replace("LIMIT", genLimitSyntax(table));
@ -390,7 +395,7 @@ public class CGWorkloadExporter implements BundledApp {
where PREDICATE
LIMIT;
"""
.replace("KEYSPACE", table.getKeySpace())
.replace("KEYSPACE", table.getKeyspace().getName())
.replace("TABLE", table.getName())
.replace("PREDICATE", genPredicateTemplate(table, 0))
.replace("LIMIT", genLimitSyntax(table));
@ -430,15 +435,15 @@ public class CGWorkloadExporter implements BundledApp {
VALUES
( BINDINGS );
"""
.replace("KEYSPACE", table.getKeySpace())
.replace("KEYSPACE", table.getKeyspace().getName())
.replace("TABLE", table.getName())
.replace("FIELDNAMES",
String.join(", ",
table.getColumnDefinitions().stream()
.map(CqlColumnDef::getName).toList()))
table.getColumnDefs().stream()
.map(CqlTableColumn::getName).toList()))
.replaceAll("BINDINGS",
String.join(", ",
table.getColumnDefinitions().stream()
table.getColumnDefs().stream()
.map(c -> binder.forColumn(c))
.map(c -> "{" + c.getName() + "}").toList()));
}
@ -463,7 +468,7 @@ public class CGWorkloadExporter implements BundledApp {
private boolean isCounterTable(CqlTable table) {
return table.getColumnDefinitions().stream()
return table.getColumnDefs().stream()
.anyMatch(cd -> cd.getTrimmedTypedef().equalsIgnoreCase("counter"));
}
@ -505,13 +510,13 @@ public class CGWorkloadExporter implements BundledApp {
private String genPredicateTemplate(CqlTable table, int keycount) {
StringBuilder sb = new StringBuilder();
LinkedList<CqlColumnDef> pkeys = new LinkedList<>();
LinkedList<CqlTableColumn> pkeys = new LinkedList<>();
for (String pkey : table.getPartitionKeys()) {
CqlColumnDef coldef = table.getColumnDefForName(pkey);
CqlTableColumn coldef = table.getColumnDefForName(pkey);
pkeys.push(coldef);
}
for (String ccol : table.getClusteringColumns()) {
CqlColumnDef coldef = table.getColumnDefForName(ccol);
CqlTableColumn coldef = table.getColumnDefForName(ccol);
pkeys.push(coldef);
}
@ -541,7 +546,7 @@ public class CGWorkloadExporter implements BundledApp {
return sb.toString();
}
private String genPredicatePart(CqlColumnDef def) {
private String genPredicatePart(CqlTableColumn def) {
String typeName = def.getTrimmedTypedef();
Binding binding = binder.forColumn(def);
return def.getName() + "={" + binding.getName() + "}";
@ -553,7 +558,7 @@ public class CGWorkloadExporter implements BundledApp {
set ASSIGNMENTS
where PREDICATES;
"""
.replaceAll("KEYSPACE", table.getKeySpace())
.replaceAll("KEYSPACE", table.getKeyspace().getName())
.replaceAll("TABLE", table.getName())
.replaceAll("PREDICATES", genPredicateTemplate(table, 0))
.replaceAll("ASSIGNMENTS", genAssignments(table));
@ -561,7 +566,7 @@ public class CGWorkloadExporter implements BundledApp {
private String genAssignments(CqlTable table) {
StringBuilder sb = new StringBuilder();
for (CqlColumnDef coldef : table.getNonKeyColumnDefinitions()) {
for (CqlTableColumn coldef : table.getNonKeyColumnDefinitions()) {
if (coldef.isCounter()) {
sb.append(coldef.getName()).append("=")
.append(coldef.getName()).append("+").append("{").append(binder.forColumn(coldef).getName()).append("}")
@ -693,7 +698,7 @@ public class CGWorkloadExporter implements BundledApp {
Map<String, Object> schemablock = new LinkedHashMap<>();
Map<String, Object> ops = new LinkedHashMap<>();
for (CqlKeyspace ks : model.getKeyspacesByName().values()) {
for (CqlKeyspaceDef ks : model.getKeyspaceDefs()) {
ops.put(
namer.nameFor(ks, "optype", "create", "blockname", blockname),
Map.of(
@ -712,22 +717,21 @@ public class CGWorkloadExporter implements BundledApp {
Map<String, Object> ops = new LinkedHashMap<>();
blockdata.put("ops", ops);
for (String keyspace : model.getTypesByKeyspaceThenName().keySet()) {
for (CqlType type : model.getTypesByKeyspaceThenName().get(keyspace).values()) {
ops.put(
namer.nameFor(type, "optype", "create", "blockname", blockname),
Map.of(
"simple", genTypeDDL(type),
"timeout", timeouts.get("create")
)
);
}
}
model.getTypeDefs().forEach(type -> {
ops.put(
namer.nameFor(type,"optype","create","blockname",blockname),
Map.of(
"simple",genTypeDDL(type),
"timeout",timeouts.get("create")
)
);
});
return blockdata;
}
private String genKeyspaceDDL(CqlKeyspace keyspace) {
private String genKeyspaceDDL(CqlKeyspaceDef keyspace) {
return """
create keyspace KEYSPACE
with replication = {REPLICATION}DURABLEWRITES?;
@ -742,17 +746,15 @@ public class CGWorkloadExporter implements BundledApp {
Map<String, Object> schemablock = new LinkedHashMap<>();
Map<String, Object> ops = new LinkedHashMap<>();
for (String ksname : model.getTableDefsByKeyspaceThenTable().keySet()) {
for (CqlTable cqltable : model.getTableDefsByKeyspaceThenTable().get(ksname).values()) {
ops.put(
namer.nameFor(cqltable, "optype", "create", "blockname", blockname),
Map.of(
"simple", genTableDDL(cqltable),
"timeout", timeouts.get("create")
)
);
}
}
model.getTableDefs().forEach(table -> {
ops.put(
namer.nameFor(table, "optype","create","blockname",blockname),
Map.of(
"simple",genTableDDL(table),
"timeout",timeouts.get("create")
)
);
});
schemablock.put("ops", ops);
return schemablock;
@ -765,10 +767,10 @@ public class CGWorkloadExporter implements BundledApp {
TYPEDEF
);
"""
.replace("KEYSPACE", type.getKeyspace())
.replace("KEYSPACE", type.getKeyspace().getName())
.replace("TYPENAME", type.getName())
.replace("TYPEDEF", type.getFields().entrySet().stream()
.map(entry -> entry.getKey() + " " + entry.getValue()).collect(Collectors.joining(",\n")));
.replace("TYPEDEF", type.getColumnDefs().stream()
.map(def -> def.getName() + " " + def.getTypedef()).collect(Collectors.joining(",\n")));
}
private Object genTableDDL(CqlTable cqltable) {
@ -782,7 +784,7 @@ public class CGWorkloadExporter implements BundledApp {
primary key (PRIMARYKEY)
)CLUSTERING;
"""
.replace("KEYSPACE", cqltable.getKeySpace())
.replace("KEYSPACE", cqltable.getKeyspace().getName())
.replace("TABLE", cqltable.getName())
.replace("COLUMN_DEFS", genTableColumnDDL(cqltable))
.replace("PRIMARYKEY", genPrimaryKeyDDL(cqltable))
@ -823,7 +825,7 @@ public class CGWorkloadExporter implements BundledApp {
}
private String genTableColumnDDL(CqlTable cqltable) {
return cqltable.getColumnDefinitions().stream()
return cqltable.getColumnDefs().stream()
.map(cd -> cd.getName() + " " + cd.getTrimmedTypedef())
.collect(Collectors.joining(",\n"));
}

81
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlColumnDef.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlColumnBase.java
View File

@ -21,42 +21,42 @@ import io.nosqlbench.api.labels.Labeled;
import java.util.Map;
public class CqlColumnDef implements NBNamedElement, Labeled {
private CqlTable table;
private String keyspace;
private String name;
private String type;
private final int position;
/**
* Not anchored to a parent, as it could be a table or a type.
* All access to these must be through their parent element.
*/
public abstract class CqlColumnBase implements NBNamedElement, Labeled {
public CqlColumnDef(CqlTable table, int position, String colname, String typedef) {
this.table = table;
this.position = position;
this.type = typedef;
private String name;
private String typedef;
private FieldPosition position;
public CqlColumnBase(String colname, String typedef) {
this.typedef = typedef;
this.name = colname;
}
public void setPosition(FieldPosition position) {
this.position = position;
}
public FieldPosition getPosition() {
return this.position;
}
public void setTypeDef(String type) {
this.type = type;
this.typedef = type;
}
public String getName() {
return name;
}
public String getType() {
return type;
public String getTypedef() {
return typedef;
}
public String getTrimmedTypedef() {
return type.replaceAll(" ", "");
}
public String getTableName() {
return table.getName();
}
public String getKeyspace() {
return keyspace;
return typedef.replaceAll(" ", "");
}
@Override
@ -68,21 +68,11 @@ public class CqlColumnDef implements NBNamedElement, Labeled {
public Map<String, String> getLabels() {
return Map.of(
"name", name,
"typedef", type,
"table", table.getName(),
"keyspace", keyspace,
"typedef", typedef,
"type", "column"
);
}
public void setKeyspace(String keyspace) {
this.keyspace = keyspace;
}
public void setTable(CqlTable table) {
this.table = table;
}
public boolean isCounter() {
return getTrimmedTypedef().equalsIgnoreCase("counter");
}
@ -95,27 +85,10 @@ public class CqlColumnDef implements NBNamedElement, Labeled {
return getName() + " " + getTrimmedTypedef();
}
public boolean isPartitionKey() {
return table.isPartitionKey(position);
}
public boolean isLastPartitionKey() {
return table.isLastPartitionKey(position);
}
public boolean isClusteringColumn() {
return table.isClusteringColumn(position);
}
public boolean isLastClusteringColumn() {
return table.isLastClusteringColumn(position);
}
public CqlTable getTable() {
return this.table;
}
public String getFullName() {
return getKeyspace() + "." + getTable().getName() + "." + getName() + "(column)";
return getParentFullName() + "." + getName();
}
protected abstract String getParentFullName();
}

68
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlKeyspace.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlKeyspaceDef.java
View File

@ -16,19 +16,31 @@
package io.nosqlbench.cqlgen.model;
import com.datastax.oss.driver.internal.core.util.Strings;
import io.nosqlbench.api.config.NBNamedElement;
import io.nosqlbench.api.labels.Labeled;
import io.nosqlbench.cqlgen.core.CGKeyspaceStats;
import java.util.Map;
import java.util.*;
public class CqlKeyspace implements NBNamedElement, Labeled {
public class CqlKeyspaceDef implements NBNamedElement, Labeled {
String keyspaceName= "";
CGKeyspaceStats stats;
private boolean isDurableWrites;
private String replicationData;
private final List<CqlTable> tableDefs = new ArrayList<>();
private final List<CqlType> typeDefs = new ArrayList<>();
/**
* Has this been populated by keyspace definition? If false, it is only
* here because it was vivified by a reference.
*/
private transient boolean defined;
public CqlKeyspace() {
public CqlKeyspaceDef() {
}
public CqlKeyspaceDef(String ksname) {
setKeyspaceName(ksname);
}
public void setKeyspaceName(String newname) {
@ -39,7 +51,6 @@ public class CqlKeyspace implements NBNamedElement, Labeled {
return this.keyspaceName;
}
@Override
public String toString() {
return "CqlKeyspace{" +
@ -77,4 +88,53 @@ public class CqlKeyspace implements NBNamedElement, Labeled {
public String getReplicationData() {
return this.replicationData;
}
public CqlTable getTable(String table) {
return this.tableDefs.stream().filter(t -> t.getName().equals(table)).findAny().orElse(null);
}
public void addTable(CqlTable table) {
table.setKeyspace(this);
this.tableDefs.add(table);
}
public List<CqlType> getTypeDefs() {
return this.typeDefs;
}
public List<CqlTable> getTableDefs() {
return this.tableDefs;
}
public void removeTable(CqlTable table) {
this.tableDefs.remove(table.getName());
}
public void getReferenceErrors(List<String> errors) {
if (!defined) {
errors.add("keyspace " + this.getName() + " was referenced but not defined.");
}
for (CqlType typedef : typeDefs) {
typedef.getReferenceErrors(errors);
}
for (CqlTable value : tableDefs) {
value.getReferenceErrors(errors);
}
}
public void setDefined() {
if (this.keyspaceName==null) {
throw new RuntimeException("nuh uh");
}
this.defined=true;
}
public void validate() {
Strings.requireNotEmpty(this.keyspaceName, "keyspace name");
}
public void addType(CqlType usertype) {
this.typeDefs.add(usertype);
}
}

276
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModel.java
View File

@ -16,14 +16,14 @@
package io.nosqlbench.cqlgen.model;
import io.nosqlbench.cqlgen.api.CqlModelInfo;
import io.nosqlbench.cqlgen.core.CGKeyspaceStats;
import io.nosqlbench.cqlgen.core.CGSchemaStats;
import io.nosqlbench.cqlgen.core.CGTableStats;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
/**
@ -45,25 +45,22 @@ import java.util.function.Supplier;
* name without requiring a fully-interconnected keyspace->table->column object graph.
* </p>
*/
public class CqlModel implements CqlModelInfo {
public class CqlModel {
private final static Logger logger = LogManager.getLogger(CqlModel.class);
private final Supplier<List<String>> errors;
private final Map<String, CqlKeyspace> keyspaceDefs = new LinkedHashMap<>();
private final Map<String, Map<String, CqlTable>> tableDefs = new LinkedHashMap<>();
private final Map<String, Map<String, CqlType>> typeDefs = new LinkedHashMap<>();
private final List<CqlKeyspaceDef> keyspaceDefs = new ArrayList();
private CGSchemaStats schemaStats = null;
private ComputedSchemaStats computedSchemaStats;
private Map<String,CqlKeyspaceDef> ksNameCache;
@Override
public CGSchemaStats getStats() {
return schemaStats;
}
@Override
public boolean hasStats() {
return schemaStats!=null;
return schemaStats != null;
}
public ComputedSchemaStats getComputedStats() {
@ -74,34 +71,29 @@ public class CqlModel implements CqlModelInfo {
this.schemaStats = schemaStats;
for (String statsKeyspacename : schemaStats.getKeyspaces().keySet()) {
CGKeyspaceStats keyspaceStats = schemaStats.getKeyspace(statsKeyspacename);
if (keyspaceDefs.containsKey(statsKeyspacename)) {
CqlKeyspaceDef ksdef = getKeyspace(statsKeyspacename);
if (ksdef !=null) {
logger.debug("setting keyspace stats for '" + statsKeyspacename + "'");
keyspaceDefs.get(statsKeyspacename).setStats(keyspaceStats);
ksdef.setStats(keyspaceStats);
keyspaceStats.getKeyspaceTables().forEach((tbname, tbstats) -> {
CqlTable table = ksdef.getTable(tbname);
if (table != null) {
table.setStats(tbstats);
} else {
logger.debug(" skipping table '" + statsKeyspacename + "." + tbname + ", since it was not found in the model.");
}
});
} else {
logger.debug(" skipping keyspace stats for '" + statsKeyspacename + "'");
}
for (String statsTableName : keyspaceStats.getKeyspaceTables().keySet()) {
CGTableStats tableStats = keyspaceStats.getKeyspaceTables().get(statsTableName);
Map<String, CqlTable> modelTables = tableDefs.get(statsKeyspacename);
if (modelTables != null) {
CqlTable modelTable = modelTables.get(statsTableName);
if (modelTable != null) {
logger.debug("setting table stats for '" + statsKeyspacename + "." + statsTableName + "'");
modelTable.setTableAttributes(tableStats);
} else {
logger.debug(" skipping table stats for '" + statsKeyspacename + "." + statsTableName + "'");
}
} else {
logger.debug(" SKIPPING stats for all tables in keyspace '" + statsKeyspacename + "'");
}
}
}
}
transient CqlKeyspace keyspace = null;
transient CqlTable table;
transient CqlType udt;
private CqlKeyspaceDef getKeyspace(String ksname) {
return this.keyspaceDefs.stream().filter(ksd -> ksd.getName().equals(ksname)).findAny().orElse(null);
}
public CqlModel(Supplier<List<String>> errorSource) {
@ -112,210 +104,68 @@ public class CqlModel implements CqlModelInfo {
return errors.get();
}
public void newKeyspace() {
keyspace = new CqlKeyspace();
}
public void saveKeyspace(String text, String refddl) {
keyspace.setKeyspaceName(text);
this.keyspaceDefs.put(text, keyspace);
keyspace = null;
}
public void newTable() {
table = new CqlTable();
}
public void saveTable(String keyspace, String text) {
table.setKeyspace(keyspace);
table.setName(text);
this.tableDefs.computeIfAbsent(keyspace, ks -> new LinkedHashMap<>()).put(text, table);
table = null;
}
public void saveColumnDefinition(String colname, String typedef, boolean isPrimaryKey, int position) {
this.table.addcolumnDef(colname, typedef, position);
if (isPrimaryKey) {
this.table.addPartitionKey(colname);
public CqlKeyspaceDef refKeyspace(String ksname) {
CqlKeyspaceDef keyspace = getKeyspace(ksname);
if (getKeyspace(ksname)==null) {
keyspace = new CqlKeyspaceDef(ksname);
keyspaceDefs.add(keyspace);
}
return keyspace;
}
@Override
public Map<String, CqlKeyspace> getKeyspacesByName() {
return keyspaceDefs;
public List<CqlKeyspaceDef> getKeyspaceDefs() {
return this.keyspaceDefs;
}
@Override
public List<CqlKeyspace> getKeyspaceDefs() {
return new ArrayList<>(this.keyspaceDefs.values());
}
@Override
public Map<String, Map<String, CqlTable>> getTableDefsByKeyspaceThenTable() {
return tableDefs;
}
@Override
public List<CqlTable> getTablesForKeyspace(String ksname) {
Map<String, CqlTable> tables = this.tableDefs.get(ksname);
if (tables != null) {
return new ArrayList<>(tables.values());
}
return List.of();
}
@Override
public List<CqlTable> getTableDefs() {
return tableDefs.values().stream().flatMap(m -> m.values().stream()).toList();
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
for (String ks : keyspaceDefs.keySet()) {
CqlKeyspace keyspace = keyspaceDefs.get(ks);
sb.append("keyspace '").append(keyspace.getName()).append("':\n");
sb.append(keyspace).append("\n");
tableDefs.getOrDefault(ks, Map.of()).values().stream()
.forEach(table -> {
sb.append("table '").append(table.getName()).append("':\n");
sb.append(table);
});
}
return sb.toString();
}
/**
* Get all the keyspace names which have been referenced in any way, whether or not
* this was in a keyspace definition or some other DDL like table or udt names.
*
* @return A list of all known keyspace names
*/
@Override
public Set<String> getAllKnownKeyspaceNames() {
Set<String> ksnames = new LinkedHashSet<>();
ksnames.addAll(this.keyspaceDefs.keySet());
ksnames.addAll(this.tableDefs.keySet());
return ksnames;
}
public void addPartitionKey(String partitionKey) {
table.addPartitionKey(partitionKey);
}
public void addClusteringColumn(String ccolumn) {
table.addClusteringColumn(ccolumn);
}
public void newType() {
udt = new CqlType();
}
public void addTypeField(String name, String typedef) {
udt.addField(name, typedef);
}
public void saveType(String keyspace, String name) {
udt.setKeyspace(keyspace);
udt.setName(name);
Map<String, CqlType> ksTypes = this.typeDefs.computeIfAbsent(keyspace, ks -> new LinkedHashMap<>());
ksTypes.put(udt.getName(), udt);
udt = null;
}
@Override
public List<CqlType> getTypeDefs() {
ArrayList<CqlType> list = new ArrayList<>();
for (Map<String, CqlType> cqlTypesByKeyspace : typeDefs.values()) {
for (CqlType cqlType : cqlTypesByKeyspace.values()) {
list.add(cqlType);
}
}
return list;
return this.keyspaceDefs.stream().flatMap(ks -> ks.getTypeDefs().stream()).toList();
}
public void removeKeyspaceDef(String ksname) {
this.keyspaceDefs.remove(ksname);
}
public void removeTablesForKeyspace(String ksname) {
this.tableDefs.remove(ksname);
}
public void removeTypesForKeyspace(String name) {
this.typeDefs.remove(name);
}
@Override
public String getSummaryLine() {
return "keyspaces: " + keyspaceDefs.size() + ", tables: " + getTableDefs().size() +
", columns: " + getTableDefs().stream().mapToInt(t -> t.getColumnDefinitions().size()).sum() +
", columns: " + getTableDefs().stream().mapToInt(t -> t.getColumnDefs().size()).sum() +
", types: " + getTypeDefs().size();
}
public void renamekeyspace(String keyspaceName, String newKeyspaceName) {
if (this.keyspaceDefs.containsKey(keyspaceName)) {
CqlKeyspace keyspace = this.keyspaceDefs.remove(keyspaceName);
keyspace.setKeyspaceName(newKeyspaceName);
this.keyspaceDefs.put(newKeyspaceName, keyspace);
}
if (this.tableDefs.containsKey(keyspaceName)) {
Map<String, CqlTable> tablesForKeyspace = this.tableDefs.remove(keyspaceName);
if (tablesForKeyspace != null) {
for (CqlTable table : tablesForKeyspace.values()) {
table.setKeyspace(newKeyspaceName);
}
}
this.tableDefs.put(newKeyspaceName, tablesForKeyspace);
}
if (this.typeDefs.containsKey(keyspaceName)) {
Map<String, CqlType> typesForKeyspace = this.typeDefs.remove(keyspaceName);
if (typesForKeyspace != null) {
for (CqlType cqltype : typesForKeyspace.values()) {
cqltype.setKeyspace(newKeyspaceName);
}
}
this.typeDefs.put(newKeyspaceName, typesForKeyspace);
}
public List<CqlTable> getTableDefs() {
return this.keyspaceDefs.stream().flatMap(ks -> ks.getTableDefs().stream()).toList();
}
public void renameTable(CqlTable extant, String newTableName) {
Map<String, CqlTable> tablesInKs = tableDefs.get(extant.getKeySpace());
CqlTable table = tablesInKs.get(extant.getName());
table.setName(newTableName);
tablesInKs.put(table.getName(), table);
}
public void renameType(String keyspaceName, String typeName, String newTypeName) {
Map<String, CqlType> typesInKeyspace = typeDefs.get(keyspaceName);
CqlType cqlType = typesInKeyspace.remove(typeName);
cqlType.setName(newTypeName);
typesInKeyspace.put(newTypeName, cqlType);
}
public void setTableCompactStorage(boolean isCompactStorage) {
table.setCompactStorage(isCompactStorage);
}
public void setKeyspaceDurableWrites(String booleanLiteral) {
keyspace.setDurableWrites(Boolean.parseBoolean(booleanLiteral));
}
public void setReplicationData(String repldata) {
keyspace.setReplicationData(repldata);
}
@Override
public Map<String, Map<String, CqlType>> getTypesByKeyspaceThenName() {
return typeDefs;
}
public void addClusteringOrder(String colname, String order) {
table.addTableClusteringOrder(colname, order);
public void renameColumn(CqlColumnBase extant, String newColName) {
extant.setName(newColName);
}
public boolean isEmpty() {
return this.keyspaceDefs.size() == 0 && this.tableDefs.size() == 0 && this.typeDefs.size() == 0;
return this.keyspaceDefs.size() == 0;
}
public List<String> getReferenceErrors() {
List<String> errors = new ArrayList<>();
for (CqlKeyspaceDef keyspace : this.keyspaceDefs) {
keyspace.getReferenceErrors(errors);
}
return errors;
}
public void addKeyspace(CqlKeyspaceDef keyspace) {
this.keyspaceDefs.add(keyspace);
}
public void addType(String ksname, CqlType usertype) {
CqlKeyspaceDef refks = this.refKeyspace(ksname);
usertype.setKeyspace(refks);
refks.addType(usertype);
}
public void addTable(String ksname, CqlTable table) {
CqlKeyspaceDef cqlKeyspaceDef = refKeyspace(ksname);
table.setKeyspace(cqlKeyspaceDef);
cqlKeyspaceDef.addTable(table);
}
}

106
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModelBuilder.java
View File

@ -36,6 +36,13 @@ public class CqlModelBuilder extends CqlParserBaseListener {
private final CqlModel model;
private long counted;
private int colindex;
private CqlKeyspaceDef keyspace;
private CqlType usertype;
transient CqlTable table;
public CqlModelBuilder(CGErrorListener errorListener) {
this.errorListener = errorListener;
@ -62,26 +69,34 @@ public class CqlModelBuilder extends CqlParserBaseListener {
@Override
public void enterCreateKeyspace(CqlParser.CreateKeyspaceContext ctx) {
model.newKeyspace();
this.keyspace=new CqlKeyspaceDef();
}
@Override
public void exitCreateKeyspace(CqlParser.CreateKeyspaceContext ctx) {
model.saveKeyspace(
ctx.keyspace().getText(),
textOf(ctx)
saveKeyspace(
ctx.keyspace().getText()
);
}
public void saveKeyspace(String keyspaceName) {
this.keyspace.setKeyspaceName(keyspaceName);
keyspace.validate();
keyspace.setDefined();
model.addKeyspace(keyspace);
this.keyspace = null;
}
@Override
public void exitReplicationList(CqlParser.ReplicationListContext ctx) {
String repldata = textOf(ctx);
model.setReplicationData(repldata);
keyspace.setReplicationData(repldata);
}
@Override
public void enterCreateTable(CqlParser.CreateTableContext ctx) {
model.newTable();
this.table = new CqlTable();
}
@Override
@ -92,37 +107,40 @@ public class CqlModelBuilder extends CqlParserBaseListener {
@Override
public void exitPrimaryKeyDefinition(CqlParser.PrimaryKeyDefinitionContext ctx) {
if (ctx.singlePrimaryKey() != null) {
model.addPartitionKey(ctx.singlePrimaryKey().column().getText());
addPartitionKey(ctx.singlePrimaryKey().column().getText());
} else if (ctx.compositeKey() != null) {
if (ctx.compositeKey().partitionKeyList() != null) {
for (CqlParser.PartitionKeyContext pkctx : ctx.compositeKey().partitionKeyList().partitionKey()) {
model.addPartitionKey(pkctx.column().getText());
addPartitionKey(pkctx.column().getText());
}
}
if (ctx.compositeKey().clusteringKeyList() != null) {
for (CqlParser.ClusteringKeyContext ccol : ctx.compositeKey().clusteringKeyList().clusteringKey()) {
model.addClusteringColumn(ccol.column().getText());
addClusteringColumn(ccol.column().getText());
}
}
} else if (ctx.compoundKey() != null) {
model.addPartitionKey(ctx.compoundKey().partitionKey().getText());
addPartitionKey(ctx.compoundKey().partitionKey().getText());
for (CqlParser.ClusteringKeyContext ccol : ctx.compoundKey().clusteringKeyList().clusteringKey()) {
model.addClusteringColumn(ccol.column().getText());
addClusteringColumn(ccol.column().getText());
}
}
}
@Override
public void enterCreateType(CqlParser.CreateTypeContext ctx) {
model.newType();
this.usertype = new CqlType();
}
@Override
public void exitCreateType(CqlParser.CreateTypeContext ctx) {
String keyspace = ctx.keyspace().getText();
String name = ctx.type_().getText();
model.saveType(keyspace, name);
usertype.setName(name);
usertype.setDefined();
model.addType(keyspace, usertype);
usertype.validate();
usertype=null;
}
@ -132,10 +150,8 @@ public class CqlModelBuilder extends CqlParserBaseListener {
List<CqlParser.ColumnContext> columns = ctx.column();
List<CqlParser.DataTypeContext> dataTypes = ctx.dataType();
for (int idx = 0; idx < columns.size(); idx++) {
model.addTypeField(
columns.get(idx).getText(),
dataTypes.get(idx).getText()
);
addTypeField(
new CqlTypeColumn(columns.get(idx).getText(),dataTypes.get(idx).getText()));
}
// dataTypes.get(0).dataType().get(0).dataType().get(0)
@ -148,29 +164,32 @@ public class CqlModelBuilder extends CqlParserBaseListener {
@Override
public void exitCreateTable(CqlParser.CreateTableContext ctx) {
model.saveTable(
table.setName(ctx.table().getText());
saveTable(
ctx.keyspace().getText(),
ctx.table().getText()
);
}
private void saveTable(String ksname, String tableName) {
table.setName(tableName);
model.addTable(ksname, table);
table=null;
}
@Override
public void exitOrderDirection(CqlParser.OrderDirectionContext ctx) {
}
@Override
public void exitTableOptionItem(CqlParser.TableOptionItemContext ctx) {
if (ctx.kwCompactStorage()!=null) {
model.setTableCompactStorage(true);
}
super.exitTableOptionItem(ctx);
table.setCompactStorage(ctx.kwCompactStorage()!=null);
}
@Override
public void exitDurableWrites(CqlParser.DurableWritesContext ctx) {
model.setKeyspaceDurableWrites(ctx.booleanLiteral().getText());
keyspace.setDurableWrites(Boolean.parseBoolean(ctx.booleanLiteral().getText()));
}
@Override
@ -187,9 +206,10 @@ public class CqlModelBuilder extends CqlParserBaseListener {
.toList();
IntStream.range(0, columns.size())
.forEach(i -> model.addClusteringOrder(columns.get(i), orders.get(i)));
.forEach(i -> table.addTableClusteringOrder(columns.get(i), orders.get(i)));
}
// @Override
// public void exitColumn(CqlParser.ColumnContext ctx) {
// super.exitColumn(ctx);
@ -214,11 +234,10 @@ public class CqlModelBuilder extends CqlParserBaseListener {
@Override
public void exitColumnDefinition(CqlParser.ColumnDefinitionContext ctx) {
model.saveColumnDefinition(
addColumnDefinition(
ctx.column().getText(),
textOf(ctx.dataType()),
ctx.primaryKeyColumn() != null,
colindex++
ctx.primaryKeyColumn() != null
);
}
@ -236,4 +255,31 @@ public class CqlModelBuilder extends CqlParserBaseListener {
return model.getErrors();
}
private void addColumnDefinition(String colname, String typedef, boolean isPrimaryKey) {
if (table != null) {
table.addcolumnDef(new CqlTableColumn(colname, typedef));
if (isPrimaryKey) {
this.table.addPartitionKey(colname);
}
} else if (usertype != null) {
usertype.addColumn(
new CqlTypeColumn(colname, typedef)
);
}
}
public void addPartitionKey(String partitionKey) {
table.addPartitionKey(partitionKey);
}
public void addClusteringColumn(String ccolumn) {
table.addClusteringColumn(ccolumn);
}
public void addTypeField(CqlTypeColumn coldef) {
this.usertype.addColumn(coldef);
}
}

47
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTable.java
View File

@ -26,13 +26,13 @@ import java.util.function.Function;
import java.util.stream.Collectors;
public class CqlTable implements NBNamedElement, Labeled {
private CqlKeyspaceDef keyspace;
String name = "";
String keyspace = "";
CGTableStats tableAttributes = null;
int[] partitioning = new int[0];
int[] clustering = new int[0];
List<String> clusteringOrders = new ArrayList<>();
List<CqlColumnDef> coldefs = new ArrayList<>();
List<CqlTableColumn> coldefs = new ArrayList<>();
private boolean compactStorage;
private ComputedTableStats computedTableStats;
@ -47,11 +47,11 @@ public class CqlTable implements NBNamedElement, Labeled {
return tableAttributes;
}
public void setTableAttributes(CGTableStats tableAttributes) {
public void setStats(CGTableStats tableAttributes) {
this.tableAttributes = tableAttributes;
}
public void addcolumnDef(CqlColumnDef cqlField) {
public void addcolumnDef(CqlTableColumn cqlField) {
this.coldefs.add(cqlField);
}
@ -59,10 +59,6 @@ public class CqlTable implements NBNamedElement, Labeled {
this.name = tableName;
}
public void addcolumnDef(String colname, String typedef, int position) {
coldefs.add(new CqlColumnDef(this, coldefs.size(), colname, typedef));
}
@Override
public String toString() {
return "cql table: '" + this.name + "':\n"
@ -72,7 +68,7 @@ public class CqlTable implements NBNamedElement, Labeled {
.collect(Collectors.joining("\n"));
}
public List<CqlColumnDef> getColumnDefinitions() {
public List<CqlTableColumn> getColumnDefs() {
return this.coldefs;
}
@ -80,22 +76,14 @@ public class CqlTable implements NBNamedElement, Labeled {
return this.name;
}
public void setKeyspace(String newKsName) {
for (CqlColumnDef coldef : coldefs) {
coldef.setKeyspace(newKsName);
}
this.keyspace = newKsName;
}
public String getKeySpace() {
return this.keyspace;
public void setKeyspace(CqlKeyspaceDef keyspace) {
this.keyspace = keyspace;
}
@Override
public Map<String, String> getLabels() {
return Map.of(
"keyspace", this.keyspace,
"keyspace", this.getName(),
"name", this.name,
"type", "table"
);
@ -142,8 +130,8 @@ public class CqlTable implements NBNamedElement, Labeled {
return Arrays.stream(clustering).mapToObj(i -> this.coldefs.get(i).getName()).toList();
}
public CqlColumnDef getColumnDefForName(String colname) {
Optional<CqlColumnDef> def = coldefs
public CqlTableColumn getColumnDefForName(String colname) {
Optional<CqlTableColumn> def = coldefs
.stream()
.filter(c -> c.getName().equalsIgnoreCase(colname))
.findFirst();
@ -155,15 +143,15 @@ public class CqlTable implements NBNamedElement, Labeled {
}
public void renameColumns(Function<String, String> renamer) {
for (CqlColumnDef coldef : coldefs) {
for (CqlTableColumn coldef : coldefs) {
coldef.setName(renamer.apply(coldef.getName()));
}
}
public List<CqlColumnDef> getNonKeyColumnDefinitions() {
public List<CqlTableColumn> getNonKeyColumnDefinitions() {
int last = partitioning[partitioning.length - 1];
last = (clustering.length > 0 ? clustering[clustering.length - 1] : last);
List<CqlColumnDef> nonkeys = new ArrayList<>();
List<CqlTableColumn> nonkeys = new ArrayList<>();
for (int nonkey = last; nonkey < coldefs.size(); nonkey++) {
nonkeys.add(coldefs.get(nonkey));
}
@ -175,7 +163,7 @@ public class CqlTable implements NBNamedElement, Labeled {
}
public String getFullName() {
return (this.keyspace != null ? this.keyspace + "." : "") + this.name;
return (this.keyspace != null ? this.keyspace.getName() + "." : "") + this.name;
}
public boolean isPartitionKey(int position) {
@ -205,4 +193,11 @@ public class CqlTable implements NBNamedElement, Labeled {
public boolean hasStats() {
return this.computedTableStats!=null;
}
public CqlKeyspaceDef getKeyspace() {
return this.keyspace;
}
public void getReferenceErrors(List<String> errors) {
}
}

38
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTableColumn.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.model;
public class CqlTableColumn extends CqlColumnBase {
private CqlTable table;
public CqlTableColumn(String colname, String typedef) {
super(colname, typedef);
}
@Override
protected String getParentFullName() {
return table.getFullName();
}
public CqlTable getTable() {
return table;
}
public void setTable(CqlTable table) {
this.table = table;
}
}

64
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlType.java
View File

@ -19,27 +19,26 @@ package io.nosqlbench.cqlgen.model;
import io.nosqlbench.api.config.NBNamedElement;
import io.nosqlbench.api.labels.Labeled;
import java.util.LinkedHashMap;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.Objects;
public class CqlType implements NBNamedElement, Labeled {
private String keyspace;
private String name;
private String refddl;
private Map<String,String> fields = new LinkedHashMap<>();
public void setKeyspace(String newksname) {
this.keyspace = newksname;
if (refddl!=null) {
this.refddl = this.refddl.replaceAll(this.keyspace,newksname);
}
private String name;
private CqlKeyspaceDef keyspace;
private List<CqlTypeColumn> columnDefs = new ArrayList<>();
private volatile boolean defined;
public void setKeyspace(CqlKeyspaceDef keyspace) {
this.keyspace = keyspace;
}
public void setName(String name) {
this.name = name;
}
public String getKeyspace() {
public CqlKeyspaceDef getKeyspace() {
return keyspace;
}
@ -47,26 +46,47 @@ public class CqlType implements NBNamedElement, Labeled {
return this.name;
}
public void addField(String name, String typedef) {
this.fields.put(name, typedef);
public void addColumn(CqlTypeColumn def) {
this.columnDefs.add(this.columnDefs.size(),def);
}
public Map<String, String> getFields() {
return fields;
public List<CqlTypeColumn> columns() {
return columnDefs;
}
@Override
public Map<String, String> getLabels() {
return Map.of(
"keyspace", this.keyspace,
"type","udt",
"keyspace", keyspace.getName(),
"type","type",
"name",name
);
}
public void renameColumns(Function<String, String> renamer) {
Map<String,String> newColumns = new LinkedHashMap<>();
fields.forEach((k,v)->newColumns.put(renamer.apply(k),v));
this.fields = newColumns;
public void setColumnDefs(List<CqlTypeColumn> columnDefs) {
this.columnDefs = columnDefs;
}
public List<CqlTypeColumn> getColumnDefs() {
return columnDefs;
}
public String getFullName() {
return keyspace.getName()+"."+getName();
}
public void getReferenceErrors(List<String> errors) {
if (!defined) {
errors.add("type " + this.getName() + " was referenced but not defined.");
}
}
public void validate() {
Objects.requireNonNull(this.name);
Objects.requireNonNull(this.keyspace);
}
public void setDefined() {
this.defined=true;
}
}

38
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTypeColumn.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.model;
public class CqlTypeColumn extends CqlColumnBase {
CqlType type;
public CqlTypeColumn(String colname, String typedef) {
super(colname, typedef);
}
@Override
protected String getParentFullName() {
return type.getFullName();
}
public CqlType getType() {
return type;
}
public void setType(CqlType type) {
this.type = type;
}
}

37
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/FieldPosition.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.model;
public enum FieldPosition {
/**
* This field is used in the partitioning key(s) for a table
*/
Partitioning,
/**
* This field is used in the clustering value(s) for a table
*/
Clustering,
/**
* This field is a non-key field for a table
*/
NonKey,
/**
* This field is used in a type definition
*/
TypeDef
}

53
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGCachingNameRemapper.java
View File

@ -16,19 +16,19 @@
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.api.config.NBNamedElement;
import io.nosqlbench.api.labels.Labeled;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_string.Combinations;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Function;
import java.util.Objects;
import java.util.function.LongFunction;
public class CGCachingNameRemapper {
private LongFunction<String> namefunc;
private final Map<String,String> remapped = new HashMap<>();
private long index=0;
private final Map<String,String> prefixmap = new HashMap<>();
private final Map<String,Long> indexmap = new HashMap<>();
public CGCachingNameRemapper() {
this.namefunc = new Combinations("a-z;a-z;a-z;a-z;a-z;a-z;");
@ -37,28 +37,53 @@ public class CGCachingNameRemapper {
this.namefunc = function;
}
public synchronized String nameForType(String type, String originalName, String prefix) {
String canonical = type+"_"+originalName;
return getOrCreateName(canonical, prefix);
}
public synchronized String nameFor(NBNamedElement element, String prefix) {
String canonical = element.getClass().getSimpleName()+"-"+element.getName();
return getOrCreateName(canonical, prefix);
// public synchronized String nameForType(String type, String originalName, String prefix) {
// String canonical = type+"_"+originalName;
// return getOrCreateName(canonical, prefix);
// }
// public synchronized String nameFor(NBNamedElement element) {
// String prefix = prefixmap.get(element.getClass().getSimpleName());
// String canonical = element.getClass().getSimpleName()+"-"+element.getName();
// return getOrCreateName(canonical, prefix);
// }
private long indexforType(String type) {
long newvalue = indexmap.computeIfAbsent(type, t -> 0L)+1;
indexmap.put(type,newvalue);
return newvalue;
}
private String getOrCreateName(String canonical, String prefix) {
public synchronized String nameFor(Map<String,String> labels) {
String type = labels.get("type");
Objects.requireNonNull(type);
String name = labels.get("name");
Objects.requireNonNull(name);
String canonical = type+"-"+name;
String prefix = prefixmap.getOrDefault(type,"");
if (!remapped.containsKey(canonical)) {
String newname = prefix+namefunc.apply(index++);
long indexForType=indexforType(type);
String newname = (prefix!=null?prefix:"")+namefunc.apply(indexForType);
remapped.put(canonical,newname);
}
return remapped.get(canonical);
}
public Function<String, String> mapperForType(Labeled cqlTable, String prefix) {
return in -> this.nameForType(cqlTable.getClass().getSimpleName(),in, prefix);
public synchronized String nameFor(Labeled element) {
Map<String, String> labels = element.getLabels();
return nameFor(labels);
}
// public Function<String, String> mapperForType(Labeled cqlTable, String prefix) {
// return in -> this.nameForType(cqlTable.getClass().getSimpleName(),in, prefix);
// }
//
public void setNamingFunction(LongFunction<String> namerFunc) {
this.namefunc = namerFunc;
}
public void setTypePrefixes(Map<String,String> prefixesByLabeledType) {
this.prefixmap.clear();
this.prefixmap.putAll(prefixesByLabeledType);
}
}

11
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGGenStatsInjector.java
View File

@ -27,6 +27,7 @@ import java.util.Map;
public class CGGenStatsInjector implements CGModelTransformer, CGTransformerConfigurable {
private CGSchemaStats schemaStats = null;
private String name;
public CGGenStatsInjector() {
}
@ -39,6 +40,11 @@ public class CGGenStatsInjector implements CGModelTransformer, CGTransformerConf
return model;
}
@Override
public void setName(String name) {
this.name = name;
}
@Override
public void accept(Object configObject) {
if (configObject instanceof Map config) {
@ -61,4 +67,9 @@ public class CGGenStatsInjector implements CGModelTransformer, CGTransformerConf
throw new RuntimeException("stats injector requires a map for it's config value");
}
}
@Override
public String getName() {
return this.name;
}
}

14
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeySpaceDDLRemover.java
View File

@ -17,20 +17,30 @@
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.model.CqlKeyspace;
import io.nosqlbench.cqlgen.model.CqlKeyspaceDef;
import io.nosqlbench.cqlgen.model.CqlModel;
import java.util.List;
public class CGKeySpaceDDLRemover implements CGModelTransformer {
private List<String> includes;
private String name;
@Override
public CqlModel apply(CqlModel model) {
for (CqlKeyspace keyspace : model.getKeyspaceDefs()) {
for (CqlKeyspaceDef keyspace : model.getKeyspaceDefs()) {
model.removeKeyspaceDef(keyspace.getName());
}
return model;
}
@Override
public void setName(String name) {
this.name = name;
}
@Override
public String getName() {
return this.name;
}
}

19
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeyspaceFilter.java
View File

@ -18,6 +18,7 @@ package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
import io.nosqlbench.cqlgen.model.CqlKeyspaceDef;
import io.nosqlbench.cqlgen.model.CqlModel;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -25,7 +26,6 @@ import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.regex.Pattern;
@ -33,6 +33,12 @@ public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfig
private final static Logger logger = LogManager.getLogger(CGKeyspaceFilter.class);
private List<TriStateFilter> patterns;
private String name;
@Override
public String getName() {
return this.name;
}
private enum InclExcl {
include,
@ -47,8 +53,8 @@ public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfig
@Override
public CqlModel apply(CqlModel model) {
Set<String> keyspacenames = model.getAllKnownKeyspaceNames();
for (String keyspace : keyspacenames) {
List<String> ksnames = model.getKeyspaceDefs().stream().map(CqlKeyspaceDef::getName).toList();
for (String keyspace : ksnames) {
Action action = Action.inderminate;
for (TriStateFilter pattern : patterns) {
action = pattern.apply(keyspace);
@ -59,8 +65,6 @@ public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfig
case remove:
logger.info("removing all definitions in " + keyspace + " with exclusion pattern " + pattern);
model.removeKeyspaceDef(keyspace);
model.removeTablesForKeyspace(keyspace);
model.removeTypesForKeyspace(keyspace);
case inderminate:
}
}
@ -73,6 +77,11 @@ public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfig
return model;
}
@Override
public void setName(String name) {
this.name = name;
}
private static class TriStateFilter implements Function<String, Action> {
private final InclExcl filterType;
private final Pattern pattern;

17
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGModelTransformers.java
View File

@ -16,6 +16,8 @@
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.api.config.standard.NBConfigurable;
import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
import io.nosqlbench.cqlgen.model.CqlModel;
@ -27,6 +29,7 @@ import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
@ -50,9 +53,11 @@ public class CGModelTransformers implements
// Instantiate Transformer
String classname = cfgmap.get("class").toString();
String name = Optional.ofNullable(cfgmap.get("name")).orElseThrow().toString();
if (!classname.contains(".")) {
String newname = CGNameObfuscator.class.getPackageName() + "." + classname;
logger.info("qualified transformer '" + classname + "' as '" + newname + "'");
logger.debug("qualified transformer '" + classname + "' as '" + newname + "'");
classname = newname;
}
Class<?> txclass = null;
@ -63,6 +68,7 @@ public class CGModelTransformers implements
Object instance = ctor.newInstance();
if (instance instanceof CGModelTransformer t) {
transformer = t;
t.setName(name);
} else {
throw new RuntimeException("Object " + instance.getClass().getName() + " is not a " + CGModelTransformer.class.getName());
}
@ -70,7 +76,16 @@ public class CGModelTransformers implements
throw new RuntimeException(e);
}
if (transformer instanceof NBConfigurable nbc) {
Object cfg = cfgmap.get("config");
if (cfg instanceof Map tcfgmap) {
NBConfiguration configuration = nbc.getConfigModel().apply((Map<String, ?>) cfg);
nbc.applyConfig(configuration);
} else {
throw new RuntimeException("config for " + nbc.getClass().getSimpleName() + " must be map.");
}
}
// Configure Transformer IFF ...
if (transformer instanceof CGTransformerConfigurable configurable) {
Object cfgvalues = cfgmap.get("config");

139
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGNameObfuscator.java
View File

@ -22,62 +22,137 @@
*/
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.api.config.standard.*;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
import io.nosqlbench.cqlgen.model.CqlModel;
import io.nosqlbench.cqlgen.model.CqlTable;
import io.nosqlbench.cqlgen.model.CqlType;
import io.nosqlbench.cqlgen.model.*;
import io.nosqlbench.cqlgen.transformers.namecache.*;
import io.nosqlbench.virtdata.core.bindings.DataMapper;
import io.nosqlbench.virtdata.core.bindings.VirtData;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.nio.file.Path;
import java.util.Map;
import java.util.Optional;
import java.util.function.LongFunction;
public class CGNameObfuscator implements CGModelTransformer, CGTransformerConfigurable {
public class CGNameObfuscator implements CGModelTransformer, NBConfigurable {
private final static Logger logger = LogManager.getLogger(CGNameObfuscator.class);
NameCache cache = new NameCache();
private final CGCachingNameRemapper remapper = new CGCachingNameRemapper();
private String name;
private String mapfile;
@Override
public CqlModel apply(CqlModel model) {
remapper.setTypePrefixes(Map.of("keyspace", "ks_", "type", "typ_", "table", "tb_", "column","col_"));
for (String keyspaceName : model.getAllKnownKeyspaceNames()) {
String newKeyspaceName = remapper.nameForType("keyspace", keyspaceName, "ks_");
model.renamekeyspace(keyspaceName, newKeyspaceName);
if (mapfile != null) {
cache = NameCache.loadOrCreate(Path.of(mapfile));
}
for (CqlTable cqlTable : model.getTableDefs()) {
String tablename = cqlTable.getName();
String newTableName = remapper.nameFor(cqlTable, "tbl_");
model.renameTable(cqlTable, newTableName);
cqlTable.renameColumns(remapper.mapperForType(cqlTable, "col_"));
for (CqlKeyspaceDef keyspaceDef : model.getKeyspaceDefs()) {
NamedKeyspace namedKeyspace = cache.keyspace(keyspaceDef.getName());
String newKeyspaceName = namedKeyspace.computeAlias(keyspaceDef, remapper::nameFor);
keyspaceDef.setKeyspaceName(newKeyspaceName);
for (CqlType typeDef : keyspaceDef.getTypeDefs()) {
NamedType namedType = namedKeyspace.type(typeDef.getName());
String typeDefName = namedType.computeAlias(typeDef,remapper::nameFor);
namedType.setName(typeDefName);
for (CqlTypeColumn columnDef : typeDef.getColumnDefs()) {
NamedColumn namedTypeColumn = namedType.column(columnDef.getName());
String newColumnName = namedTypeColumn.computeAlias(columnDef,remapper::nameFor);
columnDef.setName(newColumnName);
}
}
for (CqlTable table : keyspaceDef.getTableDefs()) {
NamedTable namedTable = namedKeyspace.table(table.getName());
String newTableName = namedTable.computeAlias(table,remapper::nameFor);
table.setName(newTableName);
for (CqlColumnBase columnDef : table.getColumnDefs()) {
NamedColumn namedTableColumn = namedTable.column(columnDef.getName());
String newColumnName = namedTableColumn.computeAlias(columnDef,remapper::nameFor);
columnDef.setName(newColumnName);
}
}
}
for (CqlType type : model.getTypeDefs()) {
String typeName = type.getName();
String newTypeName = remapper.nameFor(type, "typ_");
model.renameType(type.getKeyspace(), typeName, newTypeName);
type.renameColumns(remapper.mapperForType(type, "typ"));
}
// for (String keyspaceName : model.getAllKnownKeyspaceNames()) {
// Map<String, String> labels = Map.of("type", "keyspace", "name", keyspaceName);
// NamedKeyspace cachedKeyspace = cache.keyspace(keyspaceName);
// cachedKeyspace.computeAlias(labels, remapper::nameFor);
//// model.renamekeyspace(keyspaceName, alias);
// }
//
// for (CqlTable cqlTable : model.getTableDefs()) {
// String tablename = cqlTable.getName();
// NamedTable cachedTable = cache.keyspace(cqlTable.getKeyspaceName()).table(tablename);
// String alias = cachedTable.computeAlias(cqlTable, remapper::nameFor);
// model.renameTable(cqlTable, alias);
//
// for (CqlColumnBase coldef : cqlTable.getColumnDefs()) {
// NamedColumn cachedColumn = cache.keyspace(cqlTable.getKeyspaceName()).table(tablename).column(coldef.getName());
// cachedColumn.computeAlias(coldef, remapper::nameFor);
//// model.renameColumn(coldef, colalias);
// }
// }
//
// for (CqlType type : model.getTypeDefs()) {
// String typeName = type.getName();
// NamedType cachedType = cache.keyspace(type.getKeyspace()).type(typeName);
// cachedType.computeAlias(type, remapper::nameFor);
//// model.renameType(type.getKeyspace(), typeName, alias);
//
//// Function<String, String> colmapper = remapper.mapperForType(type, "typ");
// Map<String, String> newdefs = new LinkedHashMap<>();
//
// Set<String> keys = type.getFields().keySet();
// for (String key : keys) {
// NamedColumn cachedColdef = cache.keyspace(type.getKeyspace()).type(typeName).column(key);
// cachedColdef.computeAlias(Map.of("type", "column", "name", key), remapper::nameFor);
//// String def = type.getFields().get(key);
//// newdefs.put(colalias, def);
//// type.setFields(newdefs);
// }
// }
if (mapfile!=null) {
cache.setPath(mapfile);
cache.Save();
}
return model;
}
@Override
public void accept(Object configObject) {
if (configObject instanceof Map cfgmap) {
Object namer = cfgmap.get("namer");
Optional<DataMapper<String>> optionalMapper = VirtData.getOptionalMapper(namer.toString());
LongFunction<String> namerFunc = optionalMapper.orElseThrow(
() -> new RuntimeException("Unable to resolve obfuscator namer '" + namer + "'")
);
remapper.setNamingFunction(namerFunc);
} else {
throw new RuntimeException("name obfuscator requires a map for its configuration value.");
}
public void setName(String name) {
this.name = name;
}
@Override
public void applyConfig(NBConfiguration cfg) {
String namer = cfg.get("namer");
DataMapper<String> namerFunc = VirtData.getMapper(namer);
this.remapper.setNamingFunction(namerFunc);
this.mapfile = cfg.getOptional("mapfile").orElse(null);
}
@Override
public NBConfigModel getConfigModel() {
return ConfigModel.of(CGNameObfuscator.class)
.add(Param.defaultTo("namer", "Combinations('0-9;0-9;0-9;0-9;0-9')"))
.add(Param.optional("mapfile", String.class))
.asReadOnly();
}
@Override
public String getName() {
return this.name;
}
}

11
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioCalculator.java
View File

@ -23,6 +23,8 @@ import io.nosqlbench.cqlgen.model.CqlTable;
public class CGRatioCalculator implements CGModelTransformer {
private String name;
@Override
public CqlModel apply(CqlModel model) {
if (!model.hasStats()) {
@ -77,5 +79,14 @@ public class CGRatioCalculator implements CGModelTransformer {
return model;
}
@Override
public void setName(String name) {
this.name = name;
}
@Override
public String getName() {
return this.name;
}
}

26
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioSuffixer.java
View File

@ -29,6 +29,7 @@ public class CGRatioSuffixer implements CGModelTransformer, NBConfigurable {
private double resolution;
private String format;
private String name;
@Override
public CqlModel apply(CqlModel model) {
@ -39,26 +40,34 @@ public class CGRatioSuffixer implements CGModelTransformer, NBConfigurable {
for (CqlTable tableDef : model.getTableDefs()) {
double opshare = tableDef.getComputedStats().getOpShareOfTotalOps();
String newname = String.format(this.format, tableDef.getName(), opshare);
model.renameTable(tableDef,newname);
double multiplier = Math.pow(10.0, resolution+1);
long value = (long) (opshare*multiplier);
String newname = String.format(this.format, tableDef.getName(), value);
tableDef.setName(newname);
}
return model;
}
@Override
public void setName(String name) {
this.name = name;
}
@Override
public void applyConfig(NBConfiguration cfg) {
this.format = cfg.get("format", String.class);
if (!format.contains("NAME")) {
throw new RuntimeException("format config param for the CGRatioSuffixer must contain 'NAME', but it is '" + format + "'");
if (!format.contains("%1$s")) {
throw new RuntimeException("format config param for the CGRatioSuffixer must contain '%1$s', but it is '" + format + "'");
}
Pattern pattern = Pattern.compile("%(2\\$)?(?<resolution>\\d+)d");
Pattern pattern = Pattern.compile(".*?%2\\$(?<resolution>\\d+)d.*");
Matcher matcher = pattern.matcher(format);
if (!matcher.matches()) {
throw new RuntimeException("Could not find the required decimal format specifier for the format config parameter of " + CGRatioSuffixer.class);
}
this.resolution = Double.parseDouble(matcher.group("resolution"));
this.resolution = Double.parseDouble(matcher.group("resolution"))-1;
this.resolution=Math.max(resolution,2);
}
@ -70,4 +79,9 @@ public class CGRatioSuffixer implements CGModelTransformer, NBConfigurable {
))
.asReadOnly();
}
@Override
public String getName() {
return this.name;
}
}

15
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGReplicationSettingInjector.java
View File

@ -18,22 +18,28 @@ package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
import io.nosqlbench.cqlgen.model.CqlKeyspace;
import io.nosqlbench.cqlgen.model.CqlKeyspaceDef;
import io.nosqlbench.cqlgen.model.CqlModel;
import java.util.Map;
public class CGReplicationSettingInjector implements CGModelTransformer, CGTransformerConfigurable {
private String replicationFields;
private String name;
@Override
public CqlModel apply(CqlModel model) {
for (CqlKeyspace keyspace : model.getKeyspaceDefs()) {
for (CqlKeyspaceDef keyspace : model.getKeyspaceDefs()) {
keyspace.setReplicationData(this.replicationFields);
}
return model;
}
@Override
public void setName(String name) {
this.name = name;
}
@Override
public void accept(Object cfgObject) {
if (cfgObject instanceof Map stringMap) {
@ -44,4 +50,9 @@ public class CGReplicationSettingInjector implements CGModelTransformer, CGTrans
throw new RuntimeException("replication settings injector requires a map for its config value.");
}
}
@Override
public String getName() {
return this.name;
}
}

17
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGUdtReplacer.java
View File

@ -17,7 +17,7 @@
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.model.CqlColumnDef;
import io.nosqlbench.cqlgen.model.CqlColumnBase;
import io.nosqlbench.cqlgen.model.CqlModel;
import io.nosqlbench.cqlgen.model.CqlTable;
@ -25,11 +25,13 @@ import java.util.List;
public class CGUdtReplacer implements CGModelTransformer {
private String name;
@Override
public CqlModel apply(CqlModel model) {
List<String> toReplace = model.getTypeDefs().stream().map(t -> t.getKeyspace() + "." + t.getName()).toList();
List<String> toReplace = model.getTypeDefs().stream().map(t -> t.getKeyspace().getName() + "." + t.getName()).toList();
for (CqlTable table : model.getTableDefs()) {
for (CqlColumnDef coldef : table.getColumnDefinitions()) {
for (CqlColumnBase coldef : table.getColumnDefs()) {
String typedef = coldef.getTrimmedTypedef();
for (String searchFor : toReplace) {
if (typedef.contains(searchFor)) {
@ -42,5 +44,14 @@ public class CGUdtReplacer implements CGModelTransformer {
return model;
}
@Override
public void setName(String name) {
this.name = name;
}
@Override
public String getName() {
return this.name;
}
}

15
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/UnusedTableRemover.java
View File

@ -29,6 +29,7 @@ import java.util.Map;
public class UnusedTableRemover implements CGModelTransformer, CGTransformerConfigurable {
private final static Logger logger = LogManager.getLogger(UnusedTableRemover.class);
private double minimumThreshold = 0.0001;
private String name;
@Override
public CqlModel apply(CqlModel model) {
@ -44,15 +45,20 @@ public class UnusedTableRemover implements CGModelTransformer, CGTransformerConf
double weightedOps = Double.parseDouble(weightedOpsSpec);
if (weightedOps < minimumThreshold) {
logger.info(String.format(
"removing table " + table.getKeySpace() + "." + table.getName() + " with minimum weighted_ops of %1.5f under %1.5f",
"removing table " + table.getKeyspace().getName() + "." + table.getName() + " with minimum weighted_ops of %1.5f under %1.5f",
weightedOps, minimumThreshold)
);
model.getTableDefsByKeyspaceThenTable().get(table.getKeySpace()).remove(table.getName());
table.getKeyspace().removeTable(table);
}
}
return model;
}
@Override
public void setName(String name) {
this.name = name;
}
@Override
public void accept(Object cfgObj) {
if (cfgObj instanceof Map stringMap) {
@ -64,4 +70,9 @@ public class UnusedTableRemover implements CGModelTransformer, CGTransformerConf
throw new RuntimeException("unused table remover requires a Map for its config value.");
}
}
@Override
public String getName() {
return this.name;
}
}

97
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NameCache.java Normal file
View File

@ -0,0 +1,97 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.transformers.namecache;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.cqlgen.model.CqlKeyspaceDef;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.function.Function;
public class NameCache {
private final static Logger logger = LogManager.getLogger(NameCache.class);
private String path;
private final Map<String, NamedKeyspace> keyspaces = new LinkedHashMap();
public NamedKeyspace keyspace(String ksname) {
return keyspaces.computeIfAbsent(ksname, v -> new NamedKeyspace(ksname));
}
public NamedKeyspace computeAlias(CqlKeyspaceDef labeledKs, Function<CqlKeyspaceDef, String> ksfunc) {
return keyspaces.computeIfAbsent(
labeledKs.getName(),
ksname -> new NamedKeyspace(ksname)
.alias(ksfunc.apply(labeledKs)
));
}
public static NameCache loadOrCreate(Path path) {
Gson gson = new GsonBuilder().setPrettyPrinting().create();
if (Files.exists(path)) {
BufferedReader reader = null;
try {
reader = Files.newBufferedReader(path);
} catch (IOException e) {
throw new RuntimeException(e);
}
NameCache nameCache = gson.fromJson(reader, NameCache.class);
nameCache.setPath(path.toString());
return nameCache;
} else {
return new NameCache().setPath(path.toString());
}
}
public NameCache setPath(String path) {
if (this.path!=null) {
if (this.path.equals(path)) {
logger.debug("mapfile unchanged '" + path + "'");
} else {
logger.info("mapfile changed from '" + this.path + "' to '" + path + "'");
this.path = path;
}
}
return this;
}
public NameCache Save() {
Gson gson = new GsonBuilder().setPrettyPrinting().create();
String json = gson.toJson(this);
Path saveto = Path.of(this.path);
try {
Files.writeString(saveto, json);
} catch (IOException e) {
throw new RuntimeException(e);
}
return this;
}
public Collection<NamedKeyspace> keyspaces() {
return this.keyspaces.values();
}
}

50
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedColumn.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.transformers.namecache;
import io.nosqlbench.api.labels.Labeled;
import java.util.Map;
import java.util.function.Function;
public class NamedColumn{
private final String name;
private String alias;
public NamedColumn(String name) {
this.name = name;
}
public void alias(String alias) {
this.alias = alias;
}
public String computeAlias(Labeled labeled, Function<Labeled, String> namer) {
if (this.alias==null) {
this.alias = namer.apply(labeled);
}
return this.alias;
}
public String computeAlias(Map<String,String> labels, Function<Map<String,String>,String> namer) {
if (this.alias==null) {
this.alias= namer.apply(labels);
}
return this.alias;
}
}

63
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedKeyspace.java Normal file
View File

@ -0,0 +1,63 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.transformers.namecache;
import io.nosqlbench.api.labels.Labeled;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.function.Function;
public class NamedKeyspace {
private final String ksname;
private final Map<String, NamedTable> tables = new LinkedHashMap<>();
private final Map<String, NamedType> types = new LinkedHashMap<>();
private String alias;
public NamedKeyspace(String ksname) {
this.ksname = ksname;
}
public NamedType type(String typename) {
return types.computeIfAbsent(typename, NamedType::new);
}
public NamedTable table(String tablename) {
return tables.computeIfAbsent(tablename, NamedTable::new);
}
public NamedKeyspace alias(String alias) {
this.alias = alias;
return this;
}
public String computeAlias(Labeled labeled, Function<Labeled,String> namer) {
if (this.alias==null) {
this.alias = namer.apply(labeled);
}
return this.alias;
}
public Collection<NamedTable> tables() {
return tables.values();
}
public Collection<NamedType> types() {
return types.values();
}
}

58
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedTable.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.transformers.namecache;
import io.nosqlbench.api.labels.Labeled;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.function.Function;
public class NamedTable {
private final String tablename;
private final Map<String, NamedColumn> columns = new LinkedHashMap<>();
private String alias;
public NamedTable(String tablename) {
this.tablename = tablename;
}
public NamedColumn column(String name) {
return this.columns.computeIfAbsent(name, NamedColumn::new);
}
public NamedTable alias(String alias) {
this.alias = alias;
return this;
}
public String computeAlias(Labeled labeled, Function<Labeled,String> namer) {
if (this.alias==null) {
this.alias = namer.apply(labeled);
}
return this.alias;
}
public String getAlias() {
return this.alias;
}
public Collection<NamedColumn> columns() {
return columns.values();
}
}

57
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedType.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.transformers.namecache;
import io.nosqlbench.api.labels.Labeled;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
public class NamedType {
private String name;
private String alias;
private final Map<String,NamedColumn> columns = new LinkedHashMap<>();
public NamedType(String typename) {
this.name = typename;
}
public void alias(String alias) {
this.alias = alias;
}
public NamedColumn column(String key) {
return this.columns.computeIfAbsent(key, NamedColumn::new);
}
public List<NamedColumn> getColumnDefs() {
return new ArrayList<>(columns.values());
}
public String computeAlias(Labeled labeled, Function<Labeled, String> namer) {
if (this.alias==null) {
this.alias = namer.apply(labeled);
}
return this.alias;
}
public void setName(String name) {
this.name = name;
}
}

10
adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/udts/ToUdt.java
View File

@ -43,11 +43,11 @@ public class ToUdt implements LongFunction<UdtValue> {
public ToUdt(String spec) {
this.spec=spec;
typeinfo = CqlModelParser.parseCqlType(spec);
UserDefinedTypeBuilder builder = new UserDefinedTypeBuilder(typeinfo.getKeyspace(), typeinfo.getName());
typeinfo.getFields().forEach((name,typedef) -> {
DataType dataType = resolveDataType(typedef);
builder.withField(name,dataType);
});
UserDefinedTypeBuilder builder = new UserDefinedTypeBuilder(typeinfo.getKeyspace().getName(), typeinfo.getName());
// typeinfo.getFields().forEach((name,typedef) -> {
// DataType dataType = resolveDataType(typedef);
// builder.withField(name,dataType);
// });
this.udt = builder.build();
{

31
adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf
View File

@ -19,7 +19,8 @@
model_transformers:
# filters in or out keyspaces
- class: CGKeyspaceFilter
- name: keyspace_filter
class: CGKeyspaceFilter
config:
- exclude: system
- exclude: system_.*
@ -32,8 +33,9 @@ model_transformers:
- include: .*
# replaces the replication settings with the provided values here,
# specifed as a text block to be put inside the curly braces
- class: CGReplicationSettingInjector
# specified as a text block to be put inside the curly braces
- name: replication
class: CGReplicationSettingInjector
config:
replication_fields: |
'class': 'SimpleStrategy',
@ -45,12 +47,14 @@ model_transformers:
# - class: CGKeySpaceDDLRemover
# Replaces UDTs with blobs until we have full UDT generation capability
- class: CGUdtReplacer
- name: udtskipper
class: CGUdtReplacer
# Reads a configured file path containing nodetool histogram stats output
# If no histostats file is provided, then this is skipped, including
# any downstream usage of this data
- class: CGGenStatsInjector
- name: tablestats
class: CGGenStatsInjector
config:
path: tablestats
@ -58,7 +62,8 @@ model_transformers:
# This depends on data from the stats injector above. If not provided,
# this skips modifying ratios gracefully and they re all just set to 1
# as usual.
- class: CGRatioCalculator
- name: ratios
class: CGRatioCalculator
# # if this is set, and the fractional rate of operations against a table
# # counting reads and writes is less than the percent threshold, then
@ -69,10 +74,21 @@ model_transformers:
# minimum_threshold: 0.001
# replaces names of keyspaces, tables, and columns with generated values
- class: CGNameObfuscator
- name: obfuscator
class: CGNameObfuscator
config:
namer: Combinations('0-9;0-9;0-9;0-9;0-9');
# mapfile: mapfile.json
# format the table names based on their total share of all ops
# The format is a String.format specifier.
# Arg 1 "%1$" is the original name.
# Arg 2 "%2$" is a double value.
# The resolution is specified by the decimal field width.
- name: ratio_suffix
class: CGRatioSuffixer
config:
format: "%1$s_%2$03d"
# This controls how the elements in the schema are named in the yaml.
# This affects block names, op template names and so on, and also how
# op templates will be named in all logs and metric views.
@ -108,7 +124,6 @@ timeouts:
delete: 10.0
# future use, not active right now
blockplan:
# not needed when tags=block:'schema.*'
# schema: schema-keyspaces, schema-tables, schema-types

2
adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpSpace.java
View File

@ -71,7 +71,7 @@ public class HttpSpace implements NBNamedElement {
public synchronized void applyConfig(NBConfiguration cfg) {
this.followRedirects =
HttpClient.Redirect.valueOf(
cfg.get("follow_redirects").toUpperCase(Locale.ROOT)
cfg.get("follow_redirects", String.class).toUpperCase(Locale.ROOT)
);
this.timeout = Duration.ofMillis(cfg.get("timeout", long.class));
this.timeoutMillis = cfg.get("timeout", long.class);

4
driver-kafka/pom.xml
View File

@ -20,7 +20,7 @@
<parent>
<artifactId>mvn-defaults</artifactId>
<groupId>io.nosqlbench</groupId>
<version>4.17.20-SNAPSHOT</version>
<version>4.17.21-SNAPSHOT</version>
<relativePath>../mvn-defaults</relativePath>
</parent>
@ -60,7 +60,7 @@
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>engine-api</artifactId>
<version>4.17.20-SNAPSHOT</version>
<version>4.17.21-SNAPSHOT</version>
</dependency>
</dependencies>