diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/BindingsLibrary.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/BindingsLibrary.java index 6f2d25c10..2516643ee 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/BindingsLibrary.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/BindingsLibrary.java @@ -17,7 +17,7 @@ package io.nosqlbench.cqlgen.api; import io.nosqlbench.cqlgen.binders.Binding; -import io.nosqlbench.cqlgen.model.CqlColumnDef; +import io.nosqlbench.cqlgen.model.CqlColumnBase; import java.util.Optional; @@ -26,5 +26,5 @@ import java.util.Optional; * to map a column definition to a binding function. */ public interface BindingsLibrary { - Optional resolveBindingsFor(CqlColumnDef def); + Optional resolveBindingsFor(CqlColumnBase def); } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGModelTransformer.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGModelTransformer.java index 753bc39e4..1ee4e9945 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGModelTransformer.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGModelTransformer.java @@ -16,6 +16,7 @@ package io.nosqlbench.cqlgen.api; +import io.nosqlbench.api.config.NBNamedElement; import io.nosqlbench.cqlgen.model.CqlModel; import java.util.function.Function; @@ -25,7 +26,8 @@ import java.util.function.Function; * The type and order of transformers is important, as one transformer may be responsible * for preparing the model for one or more downstream transformers. */ -public interface CGModelTransformer extends Function { +public interface CGModelTransformer extends Function, NBNamedElement { @Override CqlModel apply(CqlModel model); + void setName(String name); } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CqlModelInfo.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CqlModelInfo.java deleted file mode 100644 index dbc172b6f..000000000 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CqlModelInfo.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.cqlgen.api; - -import io.nosqlbench.cqlgen.core.CGSchemaStats; -import io.nosqlbench.cqlgen.model.CqlKeyspace; -import io.nosqlbench.cqlgen.model.CqlTable; -import io.nosqlbench.cqlgen.model.CqlType; - -import java.util.List; -import java.util.Map; -import java.util.Set; - -public interface CqlModelInfo { - - CGSchemaStats getStats(); - - boolean hasStats(); - - Map getKeyspacesByName(); - - List getKeyspaceDefs(); - - Map> getTableDefsByKeyspaceThenTable(); - - List getTablesForKeyspace(String ksname); - - List getTableDefs(); - - Set getAllKnownKeyspaceNames(); - - List getTypeDefs(); - - String getSummaryLine(); - - Map> getTypesByKeyspaceThenName(); -} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/BindingsAccumulator.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/BindingsAccumulator.java index d02ae1f25..a8b794e55 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/BindingsAccumulator.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/BindingsAccumulator.java @@ -17,7 +17,7 @@ package io.nosqlbench.cqlgen.binders; import io.nosqlbench.cqlgen.api.BindingsLibrary; -import io.nosqlbench.cqlgen.model.CqlColumnDef; +import io.nosqlbench.cqlgen.model.CqlColumnBase; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -42,10 +42,10 @@ public class BindingsAccumulator { this.libraries = libraries; } - public Binding forColumn(CqlColumnDef def, String... prefixes) { + public Binding forColumn(CqlColumnBase def, String... prefixes) { return forColumn(def,Map.of(), prefixes); } - public Binding forColumn(CqlColumnDef def, Map extra, String... prefixes) { + public Binding forColumn(CqlColumnBase def, Map extra, String... prefixes) { for (BindingsLibrary library : libraries) { Optional optionalBinding = switch (namingStyle) { case FullyQualified -> this.resolveFullyQualifiedBinding(def, extra); @@ -72,11 +72,11 @@ public class BindingsAccumulator { } - private Optional resolvedCondensedBinding(CqlColumnDef def, Map extra) { + private Optional resolvedCondensedBinding(CqlColumnBase def, Map extra) { throw new RuntimeException("Implement me!"); } - private Optional resolveSymbolicBinding(CqlColumnDef def, Map extra) { + private Optional resolveSymbolicBinding(CqlColumnBase def, Map extra) { for (BindingsLibrary library : libraries) { Optional binding = library.resolveBindingsFor(def); if (binding.isPresent()) { @@ -90,7 +90,7 @@ public class BindingsAccumulator { } - private Optional resolveFullyQualifiedBinding(CqlColumnDef def, Map extra) { + private Optional resolveFullyQualifiedBinding(CqlColumnBase def, Map extra) { for (BindingsLibrary library : libraries) { Optional bindingRecipe = library.resolveBindingsFor(def); if (bindingRecipe.isPresent()) { diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/NamingFolio.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/NamingFolio.java index 3a10615be..460dfe555 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/NamingFolio.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/NamingFolio.java @@ -16,7 +16,7 @@ package io.nosqlbench.cqlgen.binders; -import io.nosqlbench.cqlgen.model.CqlColumnDef; +import io.nosqlbench.cqlgen.model.CqlColumnBase; import io.nosqlbench.cqlgen.model.CqlModel; import io.nosqlbench.cqlgen.model.CqlTable; import io.nosqlbench.cqlgen.core.CGElementNamer; @@ -84,7 +84,7 @@ public class NamingFolio { public void informNamerOfAllKnownNames(CqlModel model) { for (CqlTable table : model.getTableDefs()) { - for (CqlColumnDef coldef : table.getColumnDefinitions()) { + for (CqlColumnBase coldef : table.getColumnDefs()) { addFieldRef(coldef.getLabels()); } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/UnresolvedBindingException.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/UnresolvedBindingException.java index 360d02035..9c9dff079 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/UnresolvedBindingException.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/UnresolvedBindingException.java @@ -16,12 +16,12 @@ package io.nosqlbench.cqlgen.binders; -import io.nosqlbench.cqlgen.model.CqlColumnDef; +import io.nosqlbench.cqlgen.model.CqlColumnBase; public class UnresolvedBindingException extends RuntimeException { - private final CqlColumnDef def; + private final CqlColumnBase def; - public UnresolvedBindingException(CqlColumnDef def) { + public UnresolvedBindingException(CqlColumnBase def) { this.def = def; } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGColumnRebinder.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGColumnRebinder.java index facc133e0..b8a9c268e 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGColumnRebinder.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGColumnRebinder.java @@ -16,9 +16,10 @@ package io.nosqlbench.cqlgen.core; -import io.nosqlbench.cqlgen.model.CqlColumnDef; import io.nosqlbench.cqlgen.binders.Binding; import io.nosqlbench.cqlgen.binders.BindingsAccumulator; +import io.nosqlbench.cqlgen.model.CqlTableColumn; +import io.nosqlbench.cqlgen.model.FieldPosition; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -35,16 +36,15 @@ public class CGColumnRebinder { this.quantizerDigits = quantizerDigits; } - public Binding forColumn(CqlColumnDef cdef) { - if (cdef.isPartitionKey()) { + public Binding forColumn(CqlTableColumn cdef) { + if (cdef.getPosition()== FieldPosition.Partitioning) { return dividedBinding(cdef); } else { return accumulator.forColumn(cdef); } } - - private Binding dividedBinding(CqlColumnDef column) { + private Binding dividedBinding(CqlTableColumn column) { CGTableStats stats = column.getTable().getTableAttributes(); if (stats == null) { return accumulator.forColumn(column); diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java index 705852c07..01329db9d 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java @@ -16,7 +16,7 @@ package io.nosqlbench.cqlgen.core; -import io.nosqlbench.cqlgen.model.CqlColumnDef; +import io.nosqlbench.cqlgen.model.CqlColumnBase; import io.nosqlbench.cqlgen.binders.Binding; import io.nosqlbench.cqlgen.api.BindingsLibrary; import io.nosqlbench.engine.api.activityconfig.StatementsLoader; @@ -58,7 +58,7 @@ public class CGDefaultCqlBindings implements BindingsLibrary { } @Override - public Optional resolveBindingsFor(CqlColumnDef def) { + public Optional resolveBindingsFor(CqlColumnBase def) { String typedef = def.getTrimmedTypedef(); String recipe = bindings.get(def.getTrimmedTypedef()); Binding optionalBinding = new Binding(typedef, recipe); diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGLiteralFormat.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGLiteralFormat.java index 2cdb54d16..7c8e6e1dd 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGLiteralFormat.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGLiteralFormat.java @@ -16,7 +16,7 @@ package io.nosqlbench.cqlgen.core; -import io.nosqlbench.cqlgen.model.CqlColumnDef; +import io.nosqlbench.cqlgen.model.CqlColumnBase; import java.util.Locale; import java.util.Optional; @@ -49,7 +49,7 @@ public enum CGLiteralFormat { this.literalFormat = modifier; } - public static String formatBindType(CqlColumnDef cd, String valueref) { + public static String formatBindType(CqlColumnBase cd, String valueref) { return CGLiteralFormat.valueOfCqlType(cd.getTrimmedTypedef()).orElse(CGLiteralFormat.UNKNOWN).format(valueref); } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java index 42506d149..4b5497032 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java @@ -126,7 +126,7 @@ public class CGWorkloadExporter implements BundledApp { Yaml yaml = new Yaml(); CGWorkloadExporter exporter; - Content cqlgencfg = NBIO.local().prefix("cqlgen").name("cqlgen").extension("conf").one(); + Content cqlgencfg = NBIO.local().prefix("cqlgen").name("cqlgen").extension("conf").first().orElseThrow(); if (cqlgencfg == null) { throw new RuntimeException("Unable to load cqlgen.conf"); } @@ -161,13 +161,18 @@ public class CGWorkloadExporter implements BundledApp { String partition_multipler = cfgmap.get("partition_multiplier").toString(); setPartitionMultiplier(Double.parseDouble(partition_multipler)); - configureTimeouts(cfgmap.get("timeouts")); - configureBlocks(cfgmap.get("blockplan")); configureQuantizerDigits(cfgmap.get("quantizer_digits")); this.model = CqlModelParser.parse(ddl, srcpath); + List errorlist = model.getReferenceErrors(); + if (errorlist.size()>0) { + for (String error : errorlist) { + logger.error(error); + } + throw new RuntimeException("there were " + errorlist.size() + " reference errors in the model."); + } this.model = modelTransformers.apply(this.model); String workload = getWorkloadAsYaml(); @@ -360,7 +365,7 @@ public class CGWorkloadExporter implements BundledApp { where PREDICATE LIMIT; """ - .replace("KEYSPACE", table.getKeySpace()) + .replace("KEYSPACE", table.getKeyspace().getName()) .replace("TABLE", table.getName()) .replace("PREDICATE", genPredicateTemplate(table, -1)) .replace("LIMIT", genLimitSyntax(table)); @@ -390,7 +395,7 @@ public class CGWorkloadExporter implements BundledApp { where PREDICATE LIMIT; """ - .replace("KEYSPACE", table.getKeySpace()) + .replace("KEYSPACE", table.getKeyspace().getName()) .replace("TABLE", table.getName()) .replace("PREDICATE", genPredicateTemplate(table, 0)) .replace("LIMIT", genLimitSyntax(table)); @@ -430,15 +435,15 @@ public class CGWorkloadExporter implements BundledApp { VALUES ( BINDINGS ); """ - .replace("KEYSPACE", table.getKeySpace()) + .replace("KEYSPACE", table.getKeyspace().getName()) .replace("TABLE", table.getName()) .replace("FIELDNAMES", String.join(", ", - table.getColumnDefinitions().stream() - .map(CqlColumnDef::getName).toList())) + table.getColumnDefs().stream() + .map(CqlTableColumn::getName).toList())) .replaceAll("BINDINGS", String.join(", ", - table.getColumnDefinitions().stream() + table.getColumnDefs().stream() .map(c -> binder.forColumn(c)) .map(c -> "{" + c.getName() + "}").toList())); } @@ -463,7 +468,7 @@ public class CGWorkloadExporter implements BundledApp { private boolean isCounterTable(CqlTable table) { - return table.getColumnDefinitions().stream() + return table.getColumnDefs().stream() .anyMatch(cd -> cd.getTrimmedTypedef().equalsIgnoreCase("counter")); } @@ -505,13 +510,13 @@ public class CGWorkloadExporter implements BundledApp { private String genPredicateTemplate(CqlTable table, int keycount) { StringBuilder sb = new StringBuilder(); - LinkedList pkeys = new LinkedList<>(); + LinkedList pkeys = new LinkedList<>(); for (String pkey : table.getPartitionKeys()) { - CqlColumnDef coldef = table.getColumnDefForName(pkey); + CqlTableColumn coldef = table.getColumnDefForName(pkey); pkeys.push(coldef); } for (String ccol : table.getClusteringColumns()) { - CqlColumnDef coldef = table.getColumnDefForName(ccol); + CqlTableColumn coldef = table.getColumnDefForName(ccol); pkeys.push(coldef); } @@ -541,7 +546,7 @@ public class CGWorkloadExporter implements BundledApp { return sb.toString(); } - private String genPredicatePart(CqlColumnDef def) { + private String genPredicatePart(CqlTableColumn def) { String typeName = def.getTrimmedTypedef(); Binding binding = binder.forColumn(def); return def.getName() + "={" + binding.getName() + "}"; @@ -553,7 +558,7 @@ public class CGWorkloadExporter implements BundledApp { set ASSIGNMENTS where PREDICATES; """ - .replaceAll("KEYSPACE", table.getKeySpace()) + .replaceAll("KEYSPACE", table.getKeyspace().getName()) .replaceAll("TABLE", table.getName()) .replaceAll("PREDICATES", genPredicateTemplate(table, 0)) .replaceAll("ASSIGNMENTS", genAssignments(table)); @@ -561,7 +566,7 @@ public class CGWorkloadExporter implements BundledApp { private String genAssignments(CqlTable table) { StringBuilder sb = new StringBuilder(); - for (CqlColumnDef coldef : table.getNonKeyColumnDefinitions()) { + for (CqlTableColumn coldef : table.getNonKeyColumnDefinitions()) { if (coldef.isCounter()) { sb.append(coldef.getName()).append("=") .append(coldef.getName()).append("+").append("{").append(binder.forColumn(coldef).getName()).append("}") @@ -693,7 +698,7 @@ public class CGWorkloadExporter implements BundledApp { Map schemablock = new LinkedHashMap<>(); Map ops = new LinkedHashMap<>(); - for (CqlKeyspace ks : model.getKeyspacesByName().values()) { + for (CqlKeyspaceDef ks : model.getKeyspaceDefs()) { ops.put( namer.nameFor(ks, "optype", "create", "blockname", blockname), Map.of( @@ -712,22 +717,21 @@ public class CGWorkloadExporter implements BundledApp { Map ops = new LinkedHashMap<>(); blockdata.put("ops", ops); - for (String keyspace : model.getTypesByKeyspaceThenName().keySet()) { - for (CqlType type : model.getTypesByKeyspaceThenName().get(keyspace).values()) { - ops.put( - namer.nameFor(type, "optype", "create", "blockname", blockname), - Map.of( - "simple", genTypeDDL(type), - "timeout", timeouts.get("create") - ) - ); - } - } + model.getTypeDefs().forEach(type -> { + ops.put( + namer.nameFor(type,"optype","create","blockname",blockname), + Map.of( + "simple",genTypeDDL(type), + "timeout",timeouts.get("create") + ) + ); + }); + return blockdata; } - private String genKeyspaceDDL(CqlKeyspace keyspace) { + private String genKeyspaceDDL(CqlKeyspaceDef keyspace) { return """ create keyspace KEYSPACE with replication = {REPLICATION}DURABLEWRITES?; @@ -742,17 +746,15 @@ public class CGWorkloadExporter implements BundledApp { Map schemablock = new LinkedHashMap<>(); Map ops = new LinkedHashMap<>(); - for (String ksname : model.getTableDefsByKeyspaceThenTable().keySet()) { - for (CqlTable cqltable : model.getTableDefsByKeyspaceThenTable().get(ksname).values()) { - ops.put( - namer.nameFor(cqltable, "optype", "create", "blockname", blockname), - Map.of( - "simple", genTableDDL(cqltable), - "timeout", timeouts.get("create") - ) - ); - } - } + model.getTableDefs().forEach(table -> { + ops.put( + namer.nameFor(table, "optype","create","blockname",blockname), + Map.of( + "simple",genTableDDL(table), + "timeout",timeouts.get("create") + ) + ); + }); schemablock.put("ops", ops); return schemablock; @@ -765,10 +767,10 @@ public class CGWorkloadExporter implements BundledApp { TYPEDEF ); """ - .replace("KEYSPACE", type.getKeyspace()) + .replace("KEYSPACE", type.getKeyspace().getName()) .replace("TYPENAME", type.getName()) - .replace("TYPEDEF", type.getFields().entrySet().stream() - .map(entry -> entry.getKey() + " " + entry.getValue()).collect(Collectors.joining(",\n"))); + .replace("TYPEDEF", type.getColumnDefs().stream() + .map(def -> def.getName() + " " + def.getTypedef()).collect(Collectors.joining(",\n"))); } private Object genTableDDL(CqlTable cqltable) { @@ -782,7 +784,7 @@ public class CGWorkloadExporter implements BundledApp { primary key (PRIMARYKEY) )CLUSTERING; """ - .replace("KEYSPACE", cqltable.getKeySpace()) + .replace("KEYSPACE", cqltable.getKeyspace().getName()) .replace("TABLE", cqltable.getName()) .replace("COLUMN_DEFS", genTableColumnDDL(cqltable)) .replace("PRIMARYKEY", genPrimaryKeyDDL(cqltable)) @@ -823,7 +825,7 @@ public class CGWorkloadExporter implements BundledApp { } private String genTableColumnDDL(CqlTable cqltable) { - return cqltable.getColumnDefinitions().stream() + return cqltable.getColumnDefs().stream() .map(cd -> cd.getName() + " " + cd.getTrimmedTypedef()) .collect(Collectors.joining(",\n")); } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlColumnDef.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlColumnBase.java similarity index 53% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlColumnDef.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlColumnBase.java index 5222894a9..9198f1a16 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlColumnDef.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlColumnBase.java @@ -21,42 +21,42 @@ import io.nosqlbench.api.labels.Labeled; import java.util.Map; -public class CqlColumnDef implements NBNamedElement, Labeled { - private CqlTable table; - private String keyspace; - private String name; - private String type; - private final int position; +/** + * Not anchored to a parent, as it could be a table or a type. + * All access to these must be through their parent element. + */ +public abstract class CqlColumnBase implements NBNamedElement, Labeled { - public CqlColumnDef(CqlTable table, int position, String colname, String typedef) { - this.table = table; - this.position = position; - this.type = typedef; + private String name; + private String typedef; + private FieldPosition position; + + public CqlColumnBase(String colname, String typedef) { + this.typedef = typedef; this.name = colname; } + public void setPosition(FieldPosition position) { + this.position = position; + } + + public FieldPosition getPosition() { + return this.position; + } public void setTypeDef(String type) { - this.type = type; + this.typedef = type; } public String getName() { return name; } - public String getType() { - return type; + public String getTypedef() { + return typedef; } public String getTrimmedTypedef() { - return type.replaceAll(" ", ""); - } - - public String getTableName() { - return table.getName(); - } - - public String getKeyspace() { - return keyspace; + return typedef.replaceAll(" ", ""); } @Override @@ -68,21 +68,11 @@ public class CqlColumnDef implements NBNamedElement, Labeled { public Map getLabels() { return Map.of( "name", name, - "typedef", type, - "table", table.getName(), - "keyspace", keyspace, + "typedef", typedef, "type", "column" ); } - public void setKeyspace(String keyspace) { - this.keyspace = keyspace; - } - - public void setTable(CqlTable table) { - this.table = table; - } - public boolean isCounter() { return getTrimmedTypedef().equalsIgnoreCase("counter"); } @@ -95,27 +85,10 @@ public class CqlColumnDef implements NBNamedElement, Labeled { return getName() + " " + getTrimmedTypedef(); } - public boolean isPartitionKey() { - return table.isPartitionKey(position); - } - - public boolean isLastPartitionKey() { - return table.isLastPartitionKey(position); - } - - public boolean isClusteringColumn() { - return table.isClusteringColumn(position); - } - - public boolean isLastClusteringColumn() { - return table.isLastClusteringColumn(position); - } - - public CqlTable getTable() { - return this.table; - } - public String getFullName() { - return getKeyspace() + "." + getTable().getName() + "." + getName() + "(column)"; + return getParentFullName() + "." + getName(); } + + protected abstract String getParentFullName(); + } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlKeyspace.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlKeyspaceDef.java similarity index 52% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlKeyspace.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlKeyspaceDef.java index 6a70af512..9069d2629 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlKeyspace.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlKeyspaceDef.java @@ -16,19 +16,31 @@ package io.nosqlbench.cqlgen.model; +import com.datastax.oss.driver.internal.core.util.Strings; import io.nosqlbench.api.config.NBNamedElement; import io.nosqlbench.api.labels.Labeled; import io.nosqlbench.cqlgen.core.CGKeyspaceStats; -import java.util.Map; +import java.util.*; -public class CqlKeyspace implements NBNamedElement, Labeled { +public class CqlKeyspaceDef implements NBNamedElement, Labeled { String keyspaceName= ""; CGKeyspaceStats stats; private boolean isDurableWrites; private String replicationData; + private final List tableDefs = new ArrayList<>(); + private final List typeDefs = new ArrayList<>(); + /** + * Has this been populated by keyspace definition? If false, it is only + * here because it was vivified by a reference. + */ + private transient boolean defined; - public CqlKeyspace() { + public CqlKeyspaceDef() { + } + + public CqlKeyspaceDef(String ksname) { + setKeyspaceName(ksname); } public void setKeyspaceName(String newname) { @@ -39,7 +51,6 @@ public class CqlKeyspace implements NBNamedElement, Labeled { return this.keyspaceName; } - @Override public String toString() { return "CqlKeyspace{" + @@ -77,4 +88,53 @@ public class CqlKeyspace implements NBNamedElement, Labeled { public String getReplicationData() { return this.replicationData; } + + public CqlTable getTable(String table) { + return this.tableDefs.stream().filter(t -> t.getName().equals(table)).findAny().orElse(null); + } + + public void addTable(CqlTable table) { + table.setKeyspace(this); + this.tableDefs.add(table); + } + + public List getTypeDefs() { + return this.typeDefs; + } + + public List getTableDefs() { + return this.tableDefs; + + } + + public void removeTable(CqlTable table) { + this.tableDefs.remove(table.getName()); + } + + public void getReferenceErrors(List errors) { + if (!defined) { + errors.add("keyspace " + this.getName() + " was referenced but not defined."); + } + for (CqlType typedef : typeDefs) { + typedef.getReferenceErrors(errors); + } + for (CqlTable value : tableDefs) { + value.getReferenceErrors(errors); + } + } + + public void setDefined() { + if (this.keyspaceName==null) { + throw new RuntimeException("nuh uh"); + } + this.defined=true; + } + + public void validate() { + Strings.requireNotEmpty(this.keyspaceName, "keyspace name"); + } + + public void addType(CqlType usertype) { + this.typeDefs.add(usertype); + } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModel.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModel.java index 3f62f509f..dba604051 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModel.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModel.java @@ -16,14 +16,14 @@ package io.nosqlbench.cqlgen.model; -import io.nosqlbench.cqlgen.api.CqlModelInfo; import io.nosqlbench.cqlgen.core.CGKeyspaceStats; import io.nosqlbench.cqlgen.core.CGSchemaStats; -import io.nosqlbench.cqlgen.core.CGTableStats; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import java.util.function.Supplier; /** @@ -45,25 +45,22 @@ import java.util.function.Supplier; * name without requiring a fully-interconnected keyspace->table->column object graph. *

*/ -public class CqlModel implements CqlModelInfo { +public class CqlModel { private final static Logger logger = LogManager.getLogger(CqlModel.class); private final Supplier> errors; - private final Map keyspaceDefs = new LinkedHashMap<>(); - private final Map> tableDefs = new LinkedHashMap<>(); - private final Map> typeDefs = new LinkedHashMap<>(); + private final List keyspaceDefs = new ArrayList(); private CGSchemaStats schemaStats = null; private ComputedSchemaStats computedSchemaStats; + private Map ksNameCache; - @Override public CGSchemaStats getStats() { return schemaStats; } - @Override public boolean hasStats() { - return schemaStats!=null; + return schemaStats != null; } public ComputedSchemaStats getComputedStats() { @@ -74,34 +71,29 @@ public class CqlModel implements CqlModelInfo { this.schemaStats = schemaStats; for (String statsKeyspacename : schemaStats.getKeyspaces().keySet()) { CGKeyspaceStats keyspaceStats = schemaStats.getKeyspace(statsKeyspacename); - if (keyspaceDefs.containsKey(statsKeyspacename)) { + + CqlKeyspaceDef ksdef = getKeyspace(statsKeyspacename); + if (ksdef !=null) { logger.debug("setting keyspace stats for '" + statsKeyspacename + "'"); - keyspaceDefs.get(statsKeyspacename).setStats(keyspaceStats); + ksdef.setStats(keyspaceStats); + keyspaceStats.getKeyspaceTables().forEach((tbname, tbstats) -> { + CqlTable table = ksdef.getTable(tbname); + if (table != null) { + table.setStats(tbstats); + } else { + logger.debug(" skipping table '" + statsKeyspacename + "." + tbname + ", since it was not found in the model."); + } + }); } else { logger.debug(" skipping keyspace stats for '" + statsKeyspacename + "'"); } - for (String statsTableName : keyspaceStats.getKeyspaceTables().keySet()) { - CGTableStats tableStats = keyspaceStats.getKeyspaceTables().get(statsTableName); - Map modelTables = tableDefs.get(statsKeyspacename); - if (modelTables != null) { - CqlTable modelTable = modelTables.get(statsTableName); - if (modelTable != null) { - logger.debug("setting table stats for '" + statsKeyspacename + "." + statsTableName + "'"); - modelTable.setTableAttributes(tableStats); - } else { - logger.debug(" skipping table stats for '" + statsKeyspacename + "." + statsTableName + "'"); - } - } else { - logger.debug(" SKIPPING stats for all tables in keyspace '" + statsKeyspacename + "'"); - } - } } } - transient CqlKeyspace keyspace = null; - transient CqlTable table; - transient CqlType udt; + private CqlKeyspaceDef getKeyspace(String ksname) { + return this.keyspaceDefs.stream().filter(ksd -> ksd.getName().equals(ksname)).findAny().orElse(null); + } public CqlModel(Supplier> errorSource) { @@ -112,210 +104,68 @@ public class CqlModel implements CqlModelInfo { return errors.get(); } - public void newKeyspace() { - keyspace = new CqlKeyspace(); - } - - public void saveKeyspace(String text, String refddl) { - keyspace.setKeyspaceName(text); - this.keyspaceDefs.put(text, keyspace); - keyspace = null; - } - - public void newTable() { - table = new CqlTable(); - } - - public void saveTable(String keyspace, String text) { - table.setKeyspace(keyspace); - table.setName(text); - this.tableDefs.computeIfAbsent(keyspace, ks -> new LinkedHashMap<>()).put(text, table); - table = null; - } - - public void saveColumnDefinition(String colname, String typedef, boolean isPrimaryKey, int position) { - this.table.addcolumnDef(colname, typedef, position); - if (isPrimaryKey) { - this.table.addPartitionKey(colname); + public CqlKeyspaceDef refKeyspace(String ksname) { + CqlKeyspaceDef keyspace = getKeyspace(ksname); + if (getKeyspace(ksname)==null) { + keyspace = new CqlKeyspaceDef(ksname); + keyspaceDefs.add(keyspace); } + return keyspace; } - @Override - public Map getKeyspacesByName() { - return keyspaceDefs; + + public List getKeyspaceDefs() { + return this.keyspaceDefs; } - @Override - public List getKeyspaceDefs() { - return new ArrayList<>(this.keyspaceDefs.values()); - } - - @Override - public Map> getTableDefsByKeyspaceThenTable() { - return tableDefs; - } - - @Override - public List getTablesForKeyspace(String ksname) { - Map tables = this.tableDefs.get(ksname); - if (tables != null) { - return new ArrayList<>(tables.values()); - } - return List.of(); - } - - @Override - public List getTableDefs() { - return tableDefs.values().stream().flatMap(m -> m.values().stream()).toList(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - for (String ks : keyspaceDefs.keySet()) { - CqlKeyspace keyspace = keyspaceDefs.get(ks); - sb.append("keyspace '").append(keyspace.getName()).append("':\n"); - sb.append(keyspace).append("\n"); - - tableDefs.getOrDefault(ks, Map.of()).values().stream() - .forEach(table -> { - sb.append("table '").append(table.getName()).append("':\n"); - sb.append(table); - }); - } - return sb.toString(); - } - - /** - * Get all the keyspace names which have been referenced in any way, whether or not - * this was in a keyspace definition or some other DDL like table or udt names. - * - * @return A list of all known keyspace names - */ - @Override - public Set getAllKnownKeyspaceNames() { - Set ksnames = new LinkedHashSet<>(); - ksnames.addAll(this.keyspaceDefs.keySet()); - ksnames.addAll(this.tableDefs.keySet()); - return ksnames; - } - - public void addPartitionKey(String partitionKey) { - table.addPartitionKey(partitionKey); - } - - public void addClusteringColumn(String ccolumn) { - table.addClusteringColumn(ccolumn); - } - - public void newType() { - udt = new CqlType(); - } - - public void addTypeField(String name, String typedef) { - udt.addField(name, typedef); - } - - public void saveType(String keyspace, String name) { - udt.setKeyspace(keyspace); - udt.setName(name); - Map ksTypes = this.typeDefs.computeIfAbsent(keyspace, ks -> new LinkedHashMap<>()); - ksTypes.put(udt.getName(), udt); - udt = null; - } - - @Override public List getTypeDefs() { - ArrayList list = new ArrayList<>(); - for (Map cqlTypesByKeyspace : typeDefs.values()) { - for (CqlType cqlType : cqlTypesByKeyspace.values()) { - list.add(cqlType); - } - } - return list; + return this.keyspaceDefs.stream().flatMap(ks -> ks.getTypeDefs().stream()).toList(); } public void removeKeyspaceDef(String ksname) { this.keyspaceDefs.remove(ksname); } - public void removeTablesForKeyspace(String ksname) { - this.tableDefs.remove(ksname); - } - - public void removeTypesForKeyspace(String name) { - this.typeDefs.remove(name); - } - - @Override public String getSummaryLine() { return "keyspaces: " + keyspaceDefs.size() + ", tables: " + getTableDefs().size() + - ", columns: " + getTableDefs().stream().mapToInt(t -> t.getColumnDefinitions().size()).sum() + + ", columns: " + getTableDefs().stream().mapToInt(t -> t.getColumnDefs().size()).sum() + ", types: " + getTypeDefs().size(); } - public void renamekeyspace(String keyspaceName, String newKeyspaceName) { - if (this.keyspaceDefs.containsKey(keyspaceName)) { - CqlKeyspace keyspace = this.keyspaceDefs.remove(keyspaceName); - keyspace.setKeyspaceName(newKeyspaceName); - this.keyspaceDefs.put(newKeyspaceName, keyspace); - } - if (this.tableDefs.containsKey(keyspaceName)) { - Map tablesForKeyspace = this.tableDefs.remove(keyspaceName); - if (tablesForKeyspace != null) { - for (CqlTable table : tablesForKeyspace.values()) { - table.setKeyspace(newKeyspaceName); - } - } - this.tableDefs.put(newKeyspaceName, tablesForKeyspace); - } - if (this.typeDefs.containsKey(keyspaceName)) { - Map typesForKeyspace = this.typeDefs.remove(keyspaceName); - if (typesForKeyspace != null) { - for (CqlType cqltype : typesForKeyspace.values()) { - cqltype.setKeyspace(newKeyspaceName); - } - } - this.typeDefs.put(newKeyspaceName, typesForKeyspace); - } + public List getTableDefs() { + return this.keyspaceDefs.stream().flatMap(ks -> ks.getTableDefs().stream()).toList(); } - public void renameTable(CqlTable extant, String newTableName) { - Map tablesInKs = tableDefs.get(extant.getKeySpace()); - CqlTable table = tablesInKs.get(extant.getName()); - table.setName(newTableName); - tablesInKs.put(table.getName(), table); - } - - public void renameType(String keyspaceName, String typeName, String newTypeName) { - Map typesInKeyspace = typeDefs.get(keyspaceName); - CqlType cqlType = typesInKeyspace.remove(typeName); - cqlType.setName(newTypeName); - typesInKeyspace.put(newTypeName, cqlType); - } - - public void setTableCompactStorage(boolean isCompactStorage) { - table.setCompactStorage(isCompactStorage); - } - - public void setKeyspaceDurableWrites(String booleanLiteral) { - keyspace.setDurableWrites(Boolean.parseBoolean(booleanLiteral)); - } - - public void setReplicationData(String repldata) { - keyspace.setReplicationData(repldata); - } - - @Override - public Map> getTypesByKeyspaceThenName() { - return typeDefs; - } - - public void addClusteringOrder(String colname, String order) { - table.addTableClusteringOrder(colname, order); + public void renameColumn(CqlColumnBase extant, String newColName) { + extant.setName(newColName); } public boolean isEmpty() { - return this.keyspaceDefs.size() == 0 && this.tableDefs.size() == 0 && this.typeDefs.size() == 0; + return this.keyspaceDefs.size() == 0; + } + + public List getReferenceErrors() { + List errors = new ArrayList<>(); + for (CqlKeyspaceDef keyspace : this.keyspaceDefs) { + keyspace.getReferenceErrors(errors); + } + + return errors; + } + + public void addKeyspace(CqlKeyspaceDef keyspace) { + this.keyspaceDefs.add(keyspace); + } + + public void addType(String ksname, CqlType usertype) { + CqlKeyspaceDef refks = this.refKeyspace(ksname); + usertype.setKeyspace(refks); + refks.addType(usertype); + } + + public void addTable(String ksname, CqlTable table) { + CqlKeyspaceDef cqlKeyspaceDef = refKeyspace(ksname); + table.setKeyspace(cqlKeyspaceDef); + cqlKeyspaceDef.addTable(table); } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModelBuilder.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModelBuilder.java index ae78e6d03..57cae834c 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModelBuilder.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModelBuilder.java @@ -36,6 +36,13 @@ public class CqlModelBuilder extends CqlParserBaseListener { private final CqlModel model; private long counted; private int colindex; + private CqlKeyspaceDef keyspace; + private CqlType usertype; + transient CqlTable table; + + + + public CqlModelBuilder(CGErrorListener errorListener) { this.errorListener = errorListener; @@ -62,26 +69,34 @@ public class CqlModelBuilder extends CqlParserBaseListener { @Override public void enterCreateKeyspace(CqlParser.CreateKeyspaceContext ctx) { - model.newKeyspace(); + this.keyspace=new CqlKeyspaceDef(); } @Override public void exitCreateKeyspace(CqlParser.CreateKeyspaceContext ctx) { - model.saveKeyspace( - ctx.keyspace().getText(), - textOf(ctx) + saveKeyspace( + ctx.keyspace().getText() ); } + public void saveKeyspace(String keyspaceName) { + this.keyspace.setKeyspaceName(keyspaceName); + keyspace.validate(); + keyspace.setDefined(); + model.addKeyspace(keyspace); + this.keyspace = null; + } + + @Override public void exitReplicationList(CqlParser.ReplicationListContext ctx) { String repldata = textOf(ctx); - model.setReplicationData(repldata); + keyspace.setReplicationData(repldata); } @Override public void enterCreateTable(CqlParser.CreateTableContext ctx) { - model.newTable(); + this.table = new CqlTable(); } @Override @@ -92,37 +107,40 @@ public class CqlModelBuilder extends CqlParserBaseListener { @Override public void exitPrimaryKeyDefinition(CqlParser.PrimaryKeyDefinitionContext ctx) { if (ctx.singlePrimaryKey() != null) { - model.addPartitionKey(ctx.singlePrimaryKey().column().getText()); + addPartitionKey(ctx.singlePrimaryKey().column().getText()); } else if (ctx.compositeKey() != null) { if (ctx.compositeKey().partitionKeyList() != null) { for (CqlParser.PartitionKeyContext pkctx : ctx.compositeKey().partitionKeyList().partitionKey()) { - model.addPartitionKey(pkctx.column().getText()); + addPartitionKey(pkctx.column().getText()); } } if (ctx.compositeKey().clusteringKeyList() != null) { for (CqlParser.ClusteringKeyContext ccol : ctx.compositeKey().clusteringKeyList().clusteringKey()) { - model.addClusteringColumn(ccol.column().getText()); + addClusteringColumn(ccol.column().getText()); } } } else if (ctx.compoundKey() != null) { - model.addPartitionKey(ctx.compoundKey().partitionKey().getText()); + addPartitionKey(ctx.compoundKey().partitionKey().getText()); for (CqlParser.ClusteringKeyContext ccol : ctx.compoundKey().clusteringKeyList().clusteringKey()) { - model.addClusteringColumn(ccol.column().getText()); + addClusteringColumn(ccol.column().getText()); } } } - @Override public void enterCreateType(CqlParser.CreateTypeContext ctx) { - model.newType(); + this.usertype = new CqlType(); } @Override public void exitCreateType(CqlParser.CreateTypeContext ctx) { String keyspace = ctx.keyspace().getText(); String name = ctx.type_().getText(); - model.saveType(keyspace, name); + usertype.setName(name); + usertype.setDefined(); + model.addType(keyspace, usertype); + usertype.validate(); + usertype=null; } @@ -132,10 +150,8 @@ public class CqlModelBuilder extends CqlParserBaseListener { List columns = ctx.column(); List dataTypes = ctx.dataType(); for (int idx = 0; idx < columns.size(); idx++) { - model.addTypeField( - columns.get(idx).getText(), - dataTypes.get(idx).getText() - ); + addTypeField( + new CqlTypeColumn(columns.get(idx).getText(),dataTypes.get(idx).getText())); } // dataTypes.get(0).dataType().get(0).dataType().get(0) @@ -148,29 +164,32 @@ public class CqlModelBuilder extends CqlParserBaseListener { @Override public void exitCreateTable(CqlParser.CreateTableContext ctx) { - model.saveTable( + table.setName(ctx.table().getText()); + saveTable( ctx.keyspace().getText(), ctx.table().getText() ); } + private void saveTable(String ksname, String tableName) { + table.setName(tableName); + model.addTable(ksname, table); + table=null; + } + + @Override public void exitOrderDirection(CqlParser.OrderDirectionContext ctx) { } @Override public void exitTableOptionItem(CqlParser.TableOptionItemContext ctx) { - if (ctx.kwCompactStorage()!=null) { - model.setTableCompactStorage(true); - } - super.exitTableOptionItem(ctx); + table.setCompactStorage(ctx.kwCompactStorage()!=null); } @Override public void exitDurableWrites(CqlParser.DurableWritesContext ctx) { - model.setKeyspaceDurableWrites(ctx.booleanLiteral().getText()); - - + keyspace.setDurableWrites(Boolean.parseBoolean(ctx.booleanLiteral().getText())); } @Override @@ -187,9 +206,10 @@ public class CqlModelBuilder extends CqlParserBaseListener { .toList(); IntStream.range(0, columns.size()) - .forEach(i -> model.addClusteringOrder(columns.get(i), orders.get(i))); + .forEach(i -> table.addTableClusteringOrder(columns.get(i), orders.get(i))); } + // @Override // public void exitColumn(CqlParser.ColumnContext ctx) { // super.exitColumn(ctx); @@ -214,11 +234,10 @@ public class CqlModelBuilder extends CqlParserBaseListener { @Override public void exitColumnDefinition(CqlParser.ColumnDefinitionContext ctx) { - model.saveColumnDefinition( + addColumnDefinition( ctx.column().getText(), textOf(ctx.dataType()), - ctx.primaryKeyColumn() != null, - colindex++ + ctx.primaryKeyColumn() != null ); } @@ -236,4 +255,31 @@ public class CqlModelBuilder extends CqlParserBaseListener { return model.getErrors(); } + private void addColumnDefinition(String colname, String typedef, boolean isPrimaryKey) { + if (table != null) { + table.addcolumnDef(new CqlTableColumn(colname, typedef)); + if (isPrimaryKey) { + this.table.addPartitionKey(colname); + } + } else if (usertype != null) { + usertype.addColumn( + new CqlTypeColumn(colname, typedef) + ); + } + } + + public void addPartitionKey(String partitionKey) { + table.addPartitionKey(partitionKey); + } + + public void addClusteringColumn(String ccolumn) { + table.addClusteringColumn(ccolumn); + } + + public void addTypeField(CqlTypeColumn coldef) { + this.usertype.addColumn(coldef); + } + + + } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTable.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTable.java index 6581fc34e..88ec54a91 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTable.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTable.java @@ -26,13 +26,13 @@ import java.util.function.Function; import java.util.stream.Collectors; public class CqlTable implements NBNamedElement, Labeled { + private CqlKeyspaceDef keyspace; String name = ""; - String keyspace = ""; CGTableStats tableAttributes = null; int[] partitioning = new int[0]; int[] clustering = new int[0]; List clusteringOrders = new ArrayList<>(); - List coldefs = new ArrayList<>(); + List coldefs = new ArrayList<>(); private boolean compactStorage; private ComputedTableStats computedTableStats; @@ -47,11 +47,11 @@ public class CqlTable implements NBNamedElement, Labeled { return tableAttributes; } - public void setTableAttributes(CGTableStats tableAttributes) { + public void setStats(CGTableStats tableAttributes) { this.tableAttributes = tableAttributes; } - public void addcolumnDef(CqlColumnDef cqlField) { + public void addcolumnDef(CqlTableColumn cqlField) { this.coldefs.add(cqlField); } @@ -59,10 +59,6 @@ public class CqlTable implements NBNamedElement, Labeled { this.name = tableName; } - public void addcolumnDef(String colname, String typedef, int position) { - coldefs.add(new CqlColumnDef(this, coldefs.size(), colname, typedef)); - } - @Override public String toString() { return "cql table: '" + this.name + "':\n" @@ -72,7 +68,7 @@ public class CqlTable implements NBNamedElement, Labeled { .collect(Collectors.joining("\n")); } - public List getColumnDefinitions() { + public List getColumnDefs() { return this.coldefs; } @@ -80,22 +76,14 @@ public class CqlTable implements NBNamedElement, Labeled { return this.name; } - public void setKeyspace(String newKsName) { - for (CqlColumnDef coldef : coldefs) { - coldef.setKeyspace(newKsName); - } - this.keyspace = newKsName; - - } - - public String getKeySpace() { - return this.keyspace; + public void setKeyspace(CqlKeyspaceDef keyspace) { + this.keyspace = keyspace; } @Override public Map getLabels() { return Map.of( - "keyspace", this.keyspace, + "keyspace", this.getName(), "name", this.name, "type", "table" ); @@ -142,8 +130,8 @@ public class CqlTable implements NBNamedElement, Labeled { return Arrays.stream(clustering).mapToObj(i -> this.coldefs.get(i).getName()).toList(); } - public CqlColumnDef getColumnDefForName(String colname) { - Optional def = coldefs + public CqlTableColumn getColumnDefForName(String colname) { + Optional def = coldefs .stream() .filter(c -> c.getName().equalsIgnoreCase(colname)) .findFirst(); @@ -155,15 +143,15 @@ public class CqlTable implements NBNamedElement, Labeled { } public void renameColumns(Function renamer) { - for (CqlColumnDef coldef : coldefs) { + for (CqlTableColumn coldef : coldefs) { coldef.setName(renamer.apply(coldef.getName())); } } - public List getNonKeyColumnDefinitions() { + public List getNonKeyColumnDefinitions() { int last = partitioning[partitioning.length - 1]; last = (clustering.length > 0 ? clustering[clustering.length - 1] : last); - List nonkeys = new ArrayList<>(); + List nonkeys = new ArrayList<>(); for (int nonkey = last; nonkey < coldefs.size(); nonkey++) { nonkeys.add(coldefs.get(nonkey)); } @@ -175,7 +163,7 @@ public class CqlTable implements NBNamedElement, Labeled { } public String getFullName() { - return (this.keyspace != null ? this.keyspace + "." : "") + this.name; + return (this.keyspace != null ? this.keyspace.getName() + "." : "") + this.name; } public boolean isPartitionKey(int position) { @@ -205,4 +193,11 @@ public class CqlTable implements NBNamedElement, Labeled { public boolean hasStats() { return this.computedTableStats!=null; } + + public CqlKeyspaceDef getKeyspace() { + return this.keyspace; + } + + public void getReferenceErrors(List errors) { + } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTableColumn.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTableColumn.java new file mode 100644 index 000000000..d8dbee822 --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTableColumn.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.cqlgen.model; + +public class CqlTableColumn extends CqlColumnBase { + + private CqlTable table; + public CqlTableColumn(String colname, String typedef) { + super(colname, typedef); + } + + @Override + protected String getParentFullName() { + return table.getFullName(); + } + + public CqlTable getTable() { + return table; + } + + public void setTable(CqlTable table) { + this.table = table; + } +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlType.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlType.java index d6fe0a117..a9d30fbf9 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlType.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlType.java @@ -19,27 +19,26 @@ package io.nosqlbench.cqlgen.model; import io.nosqlbench.api.config.NBNamedElement; import io.nosqlbench.api.labels.Labeled; -import java.util.LinkedHashMap; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -import java.util.function.Function; +import java.util.Objects; public class CqlType implements NBNamedElement, Labeled { - private String keyspace; - private String name; - private String refddl; - private Map fields = new LinkedHashMap<>(); - public void setKeyspace(String newksname) { - this.keyspace = newksname; - if (refddl!=null) { - this.refddl = this.refddl.replaceAll(this.keyspace,newksname); - } + private String name; + private CqlKeyspaceDef keyspace; + private List columnDefs = new ArrayList<>(); + private volatile boolean defined; + + public void setKeyspace(CqlKeyspaceDef keyspace) { + this.keyspace = keyspace; } public void setName(String name) { this.name = name; } - public String getKeyspace() { + public CqlKeyspaceDef getKeyspace() { return keyspace; } @@ -47,26 +46,47 @@ public class CqlType implements NBNamedElement, Labeled { return this.name; } - public void addField(String name, String typedef) { - this.fields.put(name, typedef); + public void addColumn(CqlTypeColumn def) { + this.columnDefs.add(this.columnDefs.size(),def); } - public Map getFields() { - return fields; + public List columns() { + return columnDefs; } @Override public Map getLabels() { return Map.of( - "keyspace", this.keyspace, - "type","udt", + "keyspace", keyspace.getName(), + "type","type", "name",name ); } - public void renameColumns(Function renamer) { - Map newColumns = new LinkedHashMap<>(); - fields.forEach((k,v)->newColumns.put(renamer.apply(k),v)); - this.fields = newColumns; + public void setColumnDefs(List columnDefs) { + this.columnDefs = columnDefs; + } + + public List getColumnDefs() { + return columnDefs; + } + + public String getFullName() { + return keyspace.getName()+"."+getName(); + } + + public void getReferenceErrors(List errors) { + if (!defined) { + errors.add("type " + this.getName() + " was referenced but not defined."); + } + } + + public void validate() { + Objects.requireNonNull(this.name); + Objects.requireNonNull(this.keyspace); + } + + public void setDefined() { + this.defined=true; } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTypeColumn.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTypeColumn.java new file mode 100644 index 000000000..4d4326db9 --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTypeColumn.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.cqlgen.model; + +public class CqlTypeColumn extends CqlColumnBase { + CqlType type; + + public CqlTypeColumn(String colname, String typedef) { + super(colname, typedef); + } + + @Override + protected String getParentFullName() { + return type.getFullName(); + } + + public CqlType getType() { + return type; + } + + public void setType(CqlType type) { + this.type = type; + } +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/FieldPosition.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/FieldPosition.java new file mode 100644 index 000000000..4d60c0b33 --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/FieldPosition.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.cqlgen.model; + +public enum FieldPosition { + /** + * This field is used in the partitioning key(s) for a table + */ + Partitioning, + /** + * This field is used in the clustering value(s) for a table + */ + Clustering, + /** + * This field is a non-key field for a table + */ + NonKey, + + /** + * This field is used in a type definition + */ + TypeDef +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGCachingNameRemapper.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGCachingNameRemapper.java index 467292a11..c4404cfd5 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGCachingNameRemapper.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGCachingNameRemapper.java @@ -16,19 +16,19 @@ package io.nosqlbench.cqlgen.transformers; -import io.nosqlbench.api.config.NBNamedElement; import io.nosqlbench.api.labels.Labeled; import io.nosqlbench.virtdata.library.basics.shared.from_long.to_string.Combinations; import java.util.HashMap; import java.util.Map; -import java.util.function.Function; +import java.util.Objects; import java.util.function.LongFunction; public class CGCachingNameRemapper { private LongFunction namefunc; private final Map remapped = new HashMap<>(); - private long index=0; + private final Map prefixmap = new HashMap<>(); + private final Map indexmap = new HashMap<>(); public CGCachingNameRemapper() { this.namefunc = new Combinations("a-z;a-z;a-z;a-z;a-z;a-z;"); @@ -37,28 +37,53 @@ public class CGCachingNameRemapper { this.namefunc = function; } - public synchronized String nameForType(String type, String originalName, String prefix) { - String canonical = type+"_"+originalName; - return getOrCreateName(canonical, prefix); - } - public synchronized String nameFor(NBNamedElement element, String prefix) { - String canonical = element.getClass().getSimpleName()+"-"+element.getName(); - return getOrCreateName(canonical, prefix); +// public synchronized String nameForType(String type, String originalName, String prefix) { +// String canonical = type+"_"+originalName; +// return getOrCreateName(canonical, prefix); +// } + +// public synchronized String nameFor(NBNamedElement element) { +// String prefix = prefixmap.get(element.getClass().getSimpleName()); +// String canonical = element.getClass().getSimpleName()+"-"+element.getName(); +// return getOrCreateName(canonical, prefix); +// } + + + private long indexforType(String type) { + long newvalue = indexmap.computeIfAbsent(type, t -> 0L)+1; + indexmap.put(type,newvalue); + return newvalue; } - private String getOrCreateName(String canonical, String prefix) { + public synchronized String nameFor(Map labels) { + String type = labels.get("type"); + Objects.requireNonNull(type); + String name = labels.get("name"); + Objects.requireNonNull(name); + String canonical = type+"-"+name; + String prefix = prefixmap.getOrDefault(type,""); if (!remapped.containsKey(canonical)) { - String newname = prefix+namefunc.apply(index++); + long indexForType=indexforType(type); + String newname = (prefix!=null?prefix:"")+namefunc.apply(indexForType); remapped.put(canonical,newname); } return remapped.get(canonical); } - public Function mapperForType(Labeled cqlTable, String prefix) { - return in -> this.nameForType(cqlTable.getClass().getSimpleName(),in, prefix); + public synchronized String nameFor(Labeled element) { + Map labels = element.getLabels(); + return nameFor(labels); } + // public Function mapperForType(Labeled cqlTable, String prefix) { +// return in -> this.nameForType(cqlTable.getClass().getSimpleName(),in, prefix); +// } +// public void setNamingFunction(LongFunction namerFunc) { this.namefunc = namerFunc; } + public void setTypePrefixes(Map prefixesByLabeledType) { + this.prefixmap.clear(); + this.prefixmap.putAll(prefixesByLabeledType); + } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGGenStatsInjector.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGGenStatsInjector.java index e81a37503..671d01abd 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGGenStatsInjector.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGGenStatsInjector.java @@ -27,6 +27,7 @@ import java.util.Map; public class CGGenStatsInjector implements CGModelTransformer, CGTransformerConfigurable { private CGSchemaStats schemaStats = null; + private String name; public CGGenStatsInjector() { } @@ -39,6 +40,11 @@ public class CGGenStatsInjector implements CGModelTransformer, CGTransformerConf return model; } + @Override + public void setName(String name) { + this.name = name; + } + @Override public void accept(Object configObject) { if (configObject instanceof Map config) { @@ -61,4 +67,9 @@ public class CGGenStatsInjector implements CGModelTransformer, CGTransformerConf throw new RuntimeException("stats injector requires a map for it's config value"); } } + + @Override + public String getName() { + return this.name; + } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeySpaceDDLRemover.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeySpaceDDLRemover.java index f600f608d..4f67e5606 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeySpaceDDLRemover.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeySpaceDDLRemover.java @@ -17,20 +17,30 @@ package io.nosqlbench.cqlgen.transformers; import io.nosqlbench.cqlgen.api.CGModelTransformer; -import io.nosqlbench.cqlgen.model.CqlKeyspace; +import io.nosqlbench.cqlgen.model.CqlKeyspaceDef; import io.nosqlbench.cqlgen.model.CqlModel; import java.util.List; public class CGKeySpaceDDLRemover implements CGModelTransformer { private List includes; + private String name; @Override public CqlModel apply(CqlModel model) { - for (CqlKeyspace keyspace : model.getKeyspaceDefs()) { + for (CqlKeyspaceDef keyspace : model.getKeyspaceDefs()) { model.removeKeyspaceDef(keyspace.getName()); } return model; } + @Override + public void setName(String name) { + this.name = name; + } + + @Override + public String getName() { + return this.name; + } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeyspaceFilter.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeyspaceFilter.java index 0f98510b8..c06db2c05 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeyspaceFilter.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeyspaceFilter.java @@ -18,6 +18,7 @@ package io.nosqlbench.cqlgen.transformers; import io.nosqlbench.cqlgen.api.CGModelTransformer; import io.nosqlbench.cqlgen.api.CGTransformerConfigurable; +import io.nosqlbench.cqlgen.model.CqlKeyspaceDef; import io.nosqlbench.cqlgen.model.CqlModel; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -25,7 +26,6 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.function.Function; import java.util.regex.Pattern; @@ -33,6 +33,12 @@ public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfig private final static Logger logger = LogManager.getLogger(CGKeyspaceFilter.class); private List patterns; + private String name; + + @Override + public String getName() { + return this.name; + } private enum InclExcl { include, @@ -47,8 +53,8 @@ public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfig @Override public CqlModel apply(CqlModel model) { - Set keyspacenames = model.getAllKnownKeyspaceNames(); - for (String keyspace : keyspacenames) { + List ksnames = model.getKeyspaceDefs().stream().map(CqlKeyspaceDef::getName).toList(); + for (String keyspace : ksnames) { Action action = Action.inderminate; for (TriStateFilter pattern : patterns) { action = pattern.apply(keyspace); @@ -59,8 +65,6 @@ public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfig case remove: logger.info("removing all definitions in " + keyspace + " with exclusion pattern " + pattern); model.removeKeyspaceDef(keyspace); - model.removeTablesForKeyspace(keyspace); - model.removeTypesForKeyspace(keyspace); case inderminate: } } @@ -73,6 +77,11 @@ public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfig return model; } + @Override + public void setName(String name) { + this.name = name; + } + private static class TriStateFilter implements Function { private final InclExcl filterType; private final Pattern pattern; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGModelTransformers.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGModelTransformers.java index 20ea43f73..935ae1475 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGModelTransformers.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGModelTransformers.java @@ -16,6 +16,8 @@ package io.nosqlbench.cqlgen.transformers; +import io.nosqlbench.api.config.standard.NBConfigurable; +import io.nosqlbench.api.config.standard.NBConfiguration; import io.nosqlbench.cqlgen.api.CGModelTransformer; import io.nosqlbench.cqlgen.api.CGTransformerConfigurable; import io.nosqlbench.cqlgen.model.CqlModel; @@ -27,6 +29,7 @@ import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; @@ -50,9 +53,11 @@ public class CGModelTransformers implements // Instantiate Transformer String classname = cfgmap.get("class").toString(); + String name = Optional.ofNullable(cfgmap.get("name")).orElseThrow().toString(); + if (!classname.contains(".")) { String newname = CGNameObfuscator.class.getPackageName() + "." + classname; - logger.info("qualified transformer '" + classname + "' as '" + newname + "'"); + logger.debug("qualified transformer '" + classname + "' as '" + newname + "'"); classname = newname; } Class txclass = null; @@ -63,6 +68,7 @@ public class CGModelTransformers implements Object instance = ctor.newInstance(); if (instance instanceof CGModelTransformer t) { transformer = t; + t.setName(name); } else { throw new RuntimeException("Object " + instance.getClass().getName() + " is not a " + CGModelTransformer.class.getName()); } @@ -70,7 +76,16 @@ public class CGModelTransformers implements throw new RuntimeException(e); } + if (transformer instanceof NBConfigurable nbc) { + Object cfg = cfgmap.get("config"); + if (cfg instanceof Map tcfgmap) { + NBConfiguration configuration = nbc.getConfigModel().apply((Map) cfg); + nbc.applyConfig(configuration); + } else { + throw new RuntimeException("config for " + nbc.getClass().getSimpleName() + " must be map."); + } + } // Configure Transformer IFF ... if (transformer instanceof CGTransformerConfigurable configurable) { Object cfgvalues = cfgmap.get("config"); diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGNameObfuscator.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGNameObfuscator.java index cfe7e5648..96c6cd332 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGNameObfuscator.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGNameObfuscator.java @@ -22,62 +22,137 @@ */ package io.nosqlbench.cqlgen.transformers; +import io.nosqlbench.api.config.standard.*; import io.nosqlbench.cqlgen.api.CGModelTransformer; -import io.nosqlbench.cqlgen.api.CGTransformerConfigurable; -import io.nosqlbench.cqlgen.model.CqlModel; -import io.nosqlbench.cqlgen.model.CqlTable; -import io.nosqlbench.cqlgen.model.CqlType; +import io.nosqlbench.cqlgen.model.*; +import io.nosqlbench.cqlgen.transformers.namecache.*; import io.nosqlbench.virtdata.core.bindings.DataMapper; import io.nosqlbench.virtdata.core.bindings.VirtData; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.nio.file.Path; import java.util.Map; -import java.util.Optional; -import java.util.function.LongFunction; -public class CGNameObfuscator implements CGModelTransformer, CGTransformerConfigurable { +public class CGNameObfuscator implements CGModelTransformer, NBConfigurable { private final static Logger logger = LogManager.getLogger(CGNameObfuscator.class); + NameCache cache = new NameCache(); private final CGCachingNameRemapper remapper = new CGCachingNameRemapper(); + private String name; + private String mapfile; @Override public CqlModel apply(CqlModel model) { + remapper.setTypePrefixes(Map.of("keyspace", "ks_", "type", "typ_", "table", "tb_", "column","col_")); - for (String keyspaceName : model.getAllKnownKeyspaceNames()) { - String newKeyspaceName = remapper.nameForType("keyspace", keyspaceName, "ks_"); - model.renamekeyspace(keyspaceName, newKeyspaceName); + if (mapfile != null) { + cache = NameCache.loadOrCreate(Path.of(mapfile)); } - for (CqlTable cqlTable : model.getTableDefs()) { - String tablename = cqlTable.getName(); - String newTableName = remapper.nameFor(cqlTable, "tbl_"); - model.renameTable(cqlTable, newTableName); - cqlTable.renameColumns(remapper.mapperForType(cqlTable, "col_")); + for (CqlKeyspaceDef keyspaceDef : model.getKeyspaceDefs()) { + NamedKeyspace namedKeyspace = cache.keyspace(keyspaceDef.getName()); + String newKeyspaceName = namedKeyspace.computeAlias(keyspaceDef, remapper::nameFor); + keyspaceDef.setKeyspaceName(newKeyspaceName); + + for (CqlType typeDef : keyspaceDef.getTypeDefs()) { + NamedType namedType = namedKeyspace.type(typeDef.getName()); + String typeDefName = namedType.computeAlias(typeDef,remapper::nameFor); + namedType.setName(typeDefName); + + for (CqlTypeColumn columnDef : typeDef.getColumnDefs()) { + NamedColumn namedTypeColumn = namedType.column(columnDef.getName()); + String newColumnName = namedTypeColumn.computeAlias(columnDef,remapper::nameFor); + columnDef.setName(newColumnName); + } + } + + for (CqlTable table : keyspaceDef.getTableDefs()) { + + NamedTable namedTable = namedKeyspace.table(table.getName()); + String newTableName = namedTable.computeAlias(table,remapper::nameFor); + table.setName(newTableName); + + for (CqlColumnBase columnDef : table.getColumnDefs()) { + NamedColumn namedTableColumn = namedTable.column(columnDef.getName()); + String newColumnName = namedTableColumn.computeAlias(columnDef,remapper::nameFor); + columnDef.setName(newColumnName); + } + + } + } - for (CqlType type : model.getTypeDefs()) { - String typeName = type.getName(); - String newTypeName = remapper.nameFor(type, "typ_"); - model.renameType(type.getKeyspace(), typeName, newTypeName); - type.renameColumns(remapper.mapperForType(type, "typ")); - } +// for (String keyspaceName : model.getAllKnownKeyspaceNames()) { +// Map labels = Map.of("type", "keyspace", "name", keyspaceName); +// NamedKeyspace cachedKeyspace = cache.keyspace(keyspaceName); +// cachedKeyspace.computeAlias(labels, remapper::nameFor); +//// model.renamekeyspace(keyspaceName, alias); +// } +// +// for (CqlTable cqlTable : model.getTableDefs()) { +// String tablename = cqlTable.getName(); +// NamedTable cachedTable = cache.keyspace(cqlTable.getKeyspaceName()).table(tablename); +// String alias = cachedTable.computeAlias(cqlTable, remapper::nameFor); +// model.renameTable(cqlTable, alias); +// +// for (CqlColumnBase coldef : cqlTable.getColumnDefs()) { +// NamedColumn cachedColumn = cache.keyspace(cqlTable.getKeyspaceName()).table(tablename).column(coldef.getName()); +// cachedColumn.computeAlias(coldef, remapper::nameFor); +//// model.renameColumn(coldef, colalias); +// } +// } +// +// for (CqlType type : model.getTypeDefs()) { +// String typeName = type.getName(); +// NamedType cachedType = cache.keyspace(type.getKeyspace()).type(typeName); +// cachedType.computeAlias(type, remapper::nameFor); +//// model.renameType(type.getKeyspace(), typeName, alias); +// +//// Function colmapper = remapper.mapperForType(type, "typ"); +// Map newdefs = new LinkedHashMap<>(); +// +// Set keys = type.getFields().keySet(); +// for (String key : keys) { +// NamedColumn cachedColdef = cache.keyspace(type.getKeyspace()).type(typeName).column(key); +// cachedColdef.computeAlias(Map.of("type", "column", "name", key), remapper::nameFor); +//// String def = type.getFields().get(key); +//// newdefs.put(colalias, def); +//// type.setFields(newdefs); +// } +// } + if (mapfile!=null) { + cache.setPath(mapfile); + cache.Save(); + } return model; } @Override - public void accept(Object configObject) { - if (configObject instanceof Map cfgmap) { - Object namer = cfgmap.get("namer"); - Optional> optionalMapper = VirtData.getOptionalMapper(namer.toString()); - LongFunction namerFunc = optionalMapper.orElseThrow( - () -> new RuntimeException("Unable to resolve obfuscator namer '" + namer + "'") - ); - remapper.setNamingFunction(namerFunc); - } else { - throw new RuntimeException("name obfuscator requires a map for its configuration value."); - } + public void setName(String name) { + this.name = name; + } + + @Override + public void applyConfig(NBConfiguration cfg) { + String namer = cfg.get("namer"); + DataMapper namerFunc = VirtData.getMapper(namer); + this.remapper.setNamingFunction(namerFunc); + this.mapfile = cfg.getOptional("mapfile").orElse(null); + } + + @Override + public NBConfigModel getConfigModel() { + return ConfigModel.of(CGNameObfuscator.class) + .add(Param.defaultTo("namer", "Combinations('0-9;0-9;0-9;0-9;0-9')")) + .add(Param.optional("mapfile", String.class)) + .asReadOnly(); + } + + @Override + public String getName() { + return this.name; } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioCalculator.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioCalculator.java index 4d4f45916..ccd8049c7 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioCalculator.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioCalculator.java @@ -23,6 +23,8 @@ import io.nosqlbench.cqlgen.model.CqlTable; public class CGRatioCalculator implements CGModelTransformer { + private String name; + @Override public CqlModel apply(CqlModel model) { if (!model.hasStats()) { @@ -77,5 +79,14 @@ public class CGRatioCalculator implements CGModelTransformer { return model; } + @Override + public void setName(String name) { + this.name = name; + } + + @Override + public String getName() { + return this.name; + } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioSuffixer.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioSuffixer.java index 7c2f1bdce..f99c9ace3 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioSuffixer.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioSuffixer.java @@ -29,6 +29,7 @@ public class CGRatioSuffixer implements CGModelTransformer, NBConfigurable { private double resolution; private String format; + private String name; @Override public CqlModel apply(CqlModel model) { @@ -39,26 +40,34 @@ public class CGRatioSuffixer implements CGModelTransformer, NBConfigurable { for (CqlTable tableDef : model.getTableDefs()) { double opshare = tableDef.getComputedStats().getOpShareOfTotalOps(); - String newname = String.format(this.format, tableDef.getName(), opshare); - model.renameTable(tableDef,newname); + double multiplier = Math.pow(10.0, resolution+1); + long value = (long) (opshare*multiplier); + String newname = String.format(this.format, tableDef.getName(), value); + tableDef.setName(newname); } return model; } + @Override + public void setName(String name) { + this.name = name; + } + @Override public void applyConfig(NBConfiguration cfg) { this.format = cfg.get("format", String.class); - if (!format.contains("NAME")) { - throw new RuntimeException("format config param for the CGRatioSuffixer must contain 'NAME', but it is '" + format + "'"); + if (!format.contains("%1$s")) { + throw new RuntimeException("format config param for the CGRatioSuffixer must contain '%1$s', but it is '" + format + "'"); } - Pattern pattern = Pattern.compile("%(2\\$)?(?\\d+)d"); + Pattern pattern = Pattern.compile(".*?%2\\$(?\\d+)d.*"); Matcher matcher = pattern.matcher(format); if (!matcher.matches()) { throw new RuntimeException("Could not find the required decimal format specifier for the format config parameter of " + CGRatioSuffixer.class); } - this.resolution = Double.parseDouble(matcher.group("resolution")); + this.resolution = Double.parseDouble(matcher.group("resolution"))-1; + this.resolution=Math.max(resolution,2); } @@ -70,4 +79,9 @@ public class CGRatioSuffixer implements CGModelTransformer, NBConfigurable { )) .asReadOnly(); } + + @Override + public String getName() { + return this.name; + } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGReplicationSettingInjector.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGReplicationSettingInjector.java index cb1a8b658..0bbba2f27 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGReplicationSettingInjector.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGReplicationSettingInjector.java @@ -18,22 +18,28 @@ package io.nosqlbench.cqlgen.transformers; import io.nosqlbench.cqlgen.api.CGModelTransformer; import io.nosqlbench.cqlgen.api.CGTransformerConfigurable; -import io.nosqlbench.cqlgen.model.CqlKeyspace; +import io.nosqlbench.cqlgen.model.CqlKeyspaceDef; import io.nosqlbench.cqlgen.model.CqlModel; import java.util.Map; public class CGReplicationSettingInjector implements CGModelTransformer, CGTransformerConfigurable { private String replicationFields; + private String name; @Override public CqlModel apply(CqlModel model) { - for (CqlKeyspace keyspace : model.getKeyspaceDefs()) { + for (CqlKeyspaceDef keyspace : model.getKeyspaceDefs()) { keyspace.setReplicationData(this.replicationFields); } return model; } + @Override + public void setName(String name) { + this.name = name; + } + @Override public void accept(Object cfgObject) { if (cfgObject instanceof Map stringMap) { @@ -44,4 +50,9 @@ public class CGReplicationSettingInjector implements CGModelTransformer, CGTrans throw new RuntimeException("replication settings injector requires a map for its config value."); } } + + @Override + public String getName() { + return this.name; + } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGUdtReplacer.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGUdtReplacer.java index 4dbbebe04..ae78fc26a 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGUdtReplacer.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGUdtReplacer.java @@ -17,7 +17,7 @@ package io.nosqlbench.cqlgen.transformers; import io.nosqlbench.cqlgen.api.CGModelTransformer; -import io.nosqlbench.cqlgen.model.CqlColumnDef; +import io.nosqlbench.cqlgen.model.CqlColumnBase; import io.nosqlbench.cqlgen.model.CqlModel; import io.nosqlbench.cqlgen.model.CqlTable; @@ -25,11 +25,13 @@ import java.util.List; public class CGUdtReplacer implements CGModelTransformer { + private String name; + @Override public CqlModel apply(CqlModel model) { - List toReplace = model.getTypeDefs().stream().map(t -> t.getKeyspace() + "." + t.getName()).toList(); + List toReplace = model.getTypeDefs().stream().map(t -> t.getKeyspace().getName() + "." + t.getName()).toList(); for (CqlTable table : model.getTableDefs()) { - for (CqlColumnDef coldef : table.getColumnDefinitions()) { + for (CqlColumnBase coldef : table.getColumnDefs()) { String typedef = coldef.getTrimmedTypedef(); for (String searchFor : toReplace) { if (typedef.contains(searchFor)) { @@ -42,5 +44,14 @@ public class CGUdtReplacer implements CGModelTransformer { return model; } + @Override + public void setName(String name) { + this.name = name; + } + + @Override + public String getName() { + return this.name; + } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/UnusedTableRemover.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/UnusedTableRemover.java index 2ee8cc55d..be92c186e 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/UnusedTableRemover.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/UnusedTableRemover.java @@ -29,6 +29,7 @@ import java.util.Map; public class UnusedTableRemover implements CGModelTransformer, CGTransformerConfigurable { private final static Logger logger = LogManager.getLogger(UnusedTableRemover.class); private double minimumThreshold = 0.0001; + private String name; @Override public CqlModel apply(CqlModel model) { @@ -44,15 +45,20 @@ public class UnusedTableRemover implements CGModelTransformer, CGTransformerConf double weightedOps = Double.parseDouble(weightedOpsSpec); if (weightedOps < minimumThreshold) { logger.info(String.format( - "removing table " + table.getKeySpace() + "." + table.getName() + " with minimum weighted_ops of %1.5f under %1.5f", + "removing table " + table.getKeyspace().getName() + "." + table.getName() + " with minimum weighted_ops of %1.5f under %1.5f", weightedOps, minimumThreshold) ); - model.getTableDefsByKeyspaceThenTable().get(table.getKeySpace()).remove(table.getName()); + table.getKeyspace().removeTable(table); } } return model; } + @Override + public void setName(String name) { + this.name = name; + } + @Override public void accept(Object cfgObj) { if (cfgObj instanceof Map stringMap) { @@ -64,4 +70,9 @@ public class UnusedTableRemover implements CGModelTransformer, CGTransformerConf throw new RuntimeException("unused table remover requires a Map for its config value."); } } + + @Override + public String getName() { + return this.name; + } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NameCache.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NameCache.java new file mode 100644 index 000000000..2099672fa --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NameCache.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.cqlgen.transformers.namecache; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.nosqlbench.cqlgen.model.CqlKeyspaceDef; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.Function; + +public class NameCache { + private final static Logger logger = LogManager.getLogger(NameCache.class); + + private String path; + private final Map keyspaces = new LinkedHashMap(); + + public NamedKeyspace keyspace(String ksname) { + return keyspaces.computeIfAbsent(ksname, v -> new NamedKeyspace(ksname)); + } + + public NamedKeyspace computeAlias(CqlKeyspaceDef labeledKs, Function ksfunc) { + return keyspaces.computeIfAbsent( + labeledKs.getName(), + ksname -> new NamedKeyspace(ksname) + .alias(ksfunc.apply(labeledKs) + )); + } + + public static NameCache loadOrCreate(Path path) { + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + if (Files.exists(path)) { + BufferedReader reader = null; + try { + reader = Files.newBufferedReader(path); + } catch (IOException e) { + throw new RuntimeException(e); + } + NameCache nameCache = gson.fromJson(reader, NameCache.class); + nameCache.setPath(path.toString()); + return nameCache; + } else { + return new NameCache().setPath(path.toString()); + } + } + + public NameCache setPath(String path) { + if (this.path!=null) { + if (this.path.equals(path)) { + logger.debug("mapfile unchanged '" + path + "'"); + } else { + logger.info("mapfile changed from '" + this.path + "' to '" + path + "'"); + this.path = path; + } + } + return this; + } + + public NameCache Save() { + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + String json = gson.toJson(this); + Path saveto = Path.of(this.path); + try { + Files.writeString(saveto, json); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + public Collection keyspaces() { + return this.keyspaces.values(); + } + +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedColumn.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedColumn.java new file mode 100644 index 000000000..ad6c17ef2 --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedColumn.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.cqlgen.transformers.namecache; + +import io.nosqlbench.api.labels.Labeled; + +import java.util.Map; +import java.util.function.Function; + +public class NamedColumn{ + private final String name; + private String alias; + + public NamedColumn(String name) { + this.name = name; + } + + public void alias(String alias) { + this.alias = alias; + } + + public String computeAlias(Labeled labeled, Function namer) { + if (this.alias==null) { + this.alias = namer.apply(labeled); + } + return this.alias; + } + + public String computeAlias(Map labels, Function,String> namer) { + if (this.alias==null) { + this.alias= namer.apply(labels); + } + return this.alias; + } + +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedKeyspace.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedKeyspace.java new file mode 100644 index 000000000..be833b278 --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedKeyspace.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.cqlgen.transformers.namecache; + +import io.nosqlbench.api.labels.Labeled; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.Function; + +public class NamedKeyspace { + private final String ksname; + private final Map tables = new LinkedHashMap<>(); + private final Map types = new LinkedHashMap<>(); + private String alias; + + public NamedKeyspace(String ksname) { + this.ksname = ksname; + } + + public NamedType type(String typename) { + return types.computeIfAbsent(typename, NamedType::new); + } + + public NamedTable table(String tablename) { + return tables.computeIfAbsent(tablename, NamedTable::new); + } + + public NamedKeyspace alias(String alias) { + this.alias = alias; + return this; + } + + public String computeAlias(Labeled labeled, Function namer) { + if (this.alias==null) { + this.alias = namer.apply(labeled); + } + return this.alias; + } + + public Collection tables() { + return tables.values(); + } + + public Collection types() { + return types.values(); + } +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedTable.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedTable.java new file mode 100644 index 000000000..2681adb65 --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedTable.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.cqlgen.transformers.namecache; + +import io.nosqlbench.api.labels.Labeled; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.Function; + +public class NamedTable { + private final String tablename; + private final Map columns = new LinkedHashMap<>(); + private String alias; + + public NamedTable(String tablename) { + this.tablename = tablename; + } + + public NamedColumn column(String name) { + return this.columns.computeIfAbsent(name, NamedColumn::new); + } + + public NamedTable alias(String alias) { + this.alias = alias; + return this; + } + + public String computeAlias(Labeled labeled, Function namer) { + if (this.alias==null) { + this.alias = namer.apply(labeled); + } + return this.alias; + } + + public String getAlias() { + return this.alias; + } + + public Collection columns() { + return columns.values(); + } +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedType.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedType.java new file mode 100644 index 000000000..76b98e8ca --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedType.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.cqlgen.transformers.namecache; + +import io.nosqlbench.api.labels.Labeled; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public class NamedType { + private String name; + private String alias; + private final Map columns = new LinkedHashMap<>(); + + public NamedType(String typename) { + this.name = typename; + } + + public void alias(String alias) { + this.alias = alias; + } + + public NamedColumn column(String key) { + return this.columns.computeIfAbsent(key, NamedColumn::new); + } + public List getColumnDefs() { + return new ArrayList<>(columns.values()); + } + + public String computeAlias(Labeled labeled, Function namer) { + if (this.alias==null) { + this.alias = namer.apply(labeled); + } + return this.alias; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/udts/ToUdt.java b/adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/udts/ToUdt.java index 0338c6c1a..b18d45c5a 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/udts/ToUdt.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/udts/ToUdt.java @@ -43,11 +43,11 @@ public class ToUdt implements LongFunction { public ToUdt(String spec) { this.spec=spec; typeinfo = CqlModelParser.parseCqlType(spec); - UserDefinedTypeBuilder builder = new UserDefinedTypeBuilder(typeinfo.getKeyspace(), typeinfo.getName()); - typeinfo.getFields().forEach((name,typedef) -> { - DataType dataType = resolveDataType(typedef); - builder.withField(name,dataType); - }); + UserDefinedTypeBuilder builder = new UserDefinedTypeBuilder(typeinfo.getKeyspace().getName(), typeinfo.getName()); +// typeinfo.getFields().forEach((name,typedef) -> { +// DataType dataType = resolveDataType(typedef); +// builder.withField(name,dataType); +// }); this.udt = builder.build(); { diff --git a/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf b/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf index f82757e1d..982d6576d 100644 --- a/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf +++ b/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf @@ -19,7 +19,8 @@ model_transformers: # filters in or out keyspaces - - class: CGKeyspaceFilter + - name: keyspace_filter + class: CGKeyspaceFilter config: - exclude: system - exclude: system_.* @@ -32,8 +33,9 @@ model_transformers: - include: .* # replaces the replication settings with the provided values here, - # specifed as a text block to be put inside the curly braces - - class: CGReplicationSettingInjector + # specified as a text block to be put inside the curly braces + - name: replication + class: CGReplicationSettingInjector config: replication_fields: | 'class': 'SimpleStrategy', @@ -45,12 +47,14 @@ model_transformers: # - class: CGKeySpaceDDLRemover # Replaces UDTs with blobs until we have full UDT generation capability - - class: CGUdtReplacer + - name: udtskipper + class: CGUdtReplacer # Reads a configured file path containing nodetool histogram stats output # If no histostats file is provided, then this is skipped, including # any downstream usage of this data - - class: CGGenStatsInjector + - name: tablestats + class: CGGenStatsInjector config: path: tablestats @@ -58,7 +62,8 @@ model_transformers: # This depends on data from the stats injector above. If not provided, # this skips modifying ratios gracefully and they re all just set to 1 # as usual. - - class: CGRatioCalculator + - name: ratios + class: CGRatioCalculator # # if this is set, and the fractional rate of operations against a table # # counting reads and writes is less than the percent threshold, then @@ -69,10 +74,21 @@ model_transformers: # minimum_threshold: 0.001 # replaces names of keyspaces, tables, and columns with generated values - - class: CGNameObfuscator + - name: obfuscator + class: CGNameObfuscator config: namer: Combinations('0-9;0-9;0-9;0-9;0-9'); +# mapfile: mapfile.json + # format the table names based on their total share of all ops + # The format is a String.format specifier. + # Arg 1 "%1$" is the original name. + # Arg 2 "%2$" is a double value. + # The resolution is specified by the decimal field width. + - name: ratio_suffix + class: CGRatioSuffixer + config: + format: "%1$s_%2$03d" # This controls how the elements in the schema are named in the yaml. # This affects block names, op template names and so on, and also how # op templates will be named in all logs and metric views. @@ -108,7 +124,6 @@ timeouts: delete: 10.0 -# future use, not active right now blockplan: # not needed when tags=block:'schema.*' # schema: schema-keyspaces, schema-tables, schema-types diff --git a/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpSpace.java b/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpSpace.java index 738e7bf8a..ce1394388 100644 --- a/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpSpace.java +++ b/adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpSpace.java @@ -71,7 +71,7 @@ public class HttpSpace implements NBNamedElement { public synchronized void applyConfig(NBConfiguration cfg) { this.followRedirects = HttpClient.Redirect.valueOf( - cfg.get("follow_redirects").toUpperCase(Locale.ROOT) + cfg.get("follow_redirects", String.class).toUpperCase(Locale.ROOT) ); this.timeout = Duration.ofMillis(cfg.get("timeout", long.class)); this.timeoutMillis = cfg.get("timeout", long.class); diff --git a/driver-kafka/pom.xml b/driver-kafka/pom.xml index c43f3c7b8..a97ee2707 100644 --- a/driver-kafka/pom.xml +++ b/driver-kafka/pom.xml @@ -20,7 +20,7 @@ mvn-defaults io.nosqlbench - 4.17.20-SNAPSHOT + 4.17.21-SNAPSHOT ../mvn-defaults @@ -60,7 +60,7 @@ io.nosqlbench engine-api - 4.17.20-SNAPSHOT + 4.17.21-SNAPSHOT