Merge branch 'cqlast'

adapter-cqld4/pom.xml

@@ -35,6 +35,12 @@
     <dependencies>
 
         <!-- core dependencies -->
+        <dependency>
+            <groupId>io.nosqlbench</groupId>
+            <artifactId>adapters-api</artifactId>
+            <version>4.17.21-SNAPSHOT</version>
+        </dependency>
+
         <dependency>
             <groupId>org.codehaus.groovy</groupId>
             <artifactId>groovy</artifactId>
@@ -52,19 +58,6 @@
             <version>3.4.13</version>
         </dependency>
 
-        <dependency>
-            <groupId>io.nosqlbench</groupId>
-            <artifactId>engine-api</artifactId>
-            <version>4.17.21-SNAPSHOT</version>
-        </dependency>
-
-        <dependency>
-            <groupId>io.nosqlbench</groupId>
-            <artifactId>virtdata-lib-basics</artifactId>
-            <version>4.17.21-SNAPSHOT</version>
-            <scope>compile</scope>
-        </dependency>
-
         <dependency>
             <groupId>com.datastax.oss</groupId>
             <artifactId>java-driver-core</artifactId>
@@ -95,6 +88,12 @@
             <version>2.2.4</version>
         </dependency>
 
+        <dependency>
+            <groupId>info.picocli</groupId>
+            <artifactId>picocli</artifactId>
+            <version>4.6.3</version>
+        </dependency>
+
     </dependencies>
 
     <build>

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/optionhelpers/OptionHelpers.java

@@ -257,8 +257,6 @@ public class OptionHelpers implements NBConfigurable {
         add("defaultidempotence","",(m,v) -> {});
 
         add("drivermetrics","",(m,v) -> {});
-
-
     }
 
     public void add(String name, String description, Modifier modifier) {

adapter-cqld4/src/main/java/io/nosqlbench/analysis/RingAnalyzer.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.analysis;
+
+import io.nosqlbench.api.spi.BundledApp;
+import io.nosqlbench.nb.annotations.Service;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import picocli.CommandLine;
+
+@Service(value=BundledApp.class, selector = "cql-ring-analyzer")
+public class RingAnalyzer implements BundledApp {
+    private final static Logger logger = LogManager.getLogger(RingAnalyzer.class);
+
+    @Override
+    public int appMain(String[] args) {
+        RingAnalyzerConfig cfg = new RingAnalyzerConfig();
+        CommandLine cli = new CommandLine(cfg);
+        CommandLine.ParseResult cl = cli.parseArgs(args);
+
+        logger.info("filename: " + cfg.filename);
+        return 0;
+    }
+}

adapter-cqld4/src/main/java/io/nosqlbench/analysis/RingAnalyzerConfig.java

@@ -14,13 +14,13 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.transformers;
+package io.nosqlbench.analysis;
 
-import io.nosqlbench.cqlgen.model.CqlModel;
+import picocli.CommandLine.*;
 
-import java.util.function.Function;
+public class RingAnalyzerConfig {
+
+    @Option(names={"-i","--input"},required = true,description = "Input files containing `nodetool ring` output")
+    String filename;
 
-public interface CGModelTransformer extends Function<CqlModel,CqlModel> {
-    @Override
-    CqlModel apply(CqlModel model);
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/BindingsLibrary.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.api;
+
+import io.nosqlbench.cqlgen.binders.Binding;
+import io.nosqlbench.cqlgen.model.CqlColumnBase;
+
+import java.util.Optional;
+
+/**
+ * A bindings library is simply a service point for a specific way
+ * to map a column definition to a binding function.
+ */
+public interface BindingsLibrary {
+    Optional<Binding> resolveBindingsFor(CqlColumnBase def);
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGModelTransformer.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.api;
+
+import io.nosqlbench.api.config.NBNamedElement;
+import io.nosqlbench.cqlgen.model.CqlModel;
+
+import java.util.function.Function;
+
+/**
+ * Most of the functionality of {@link CqlModel} preparation is handled with transformers.
+ * The type and order of transformers is important, as one transformer may be responsible
+ * for preparing the model for one or more downstream transformers.
+ */
+public interface CGModelTransformer extends Function<CqlModel,CqlModel>, NBNamedElement {
+    @Override
+    CqlModel apply(CqlModel model);
+    void setName(String name);
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGTextTransformer.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter;
+package io.nosqlbench.cqlgen.api;
 
 import java.util.function.Function;
 

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGTransformerConfigurable.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.transformers;
+package io.nosqlbench.cqlgen.api;
 
 import java.util.function.Consumer;
 

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/Binding.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.binders;
+package io.nosqlbench.cqlgen.binders;
 
 public class Binding {
     String name;

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/BindingsAccumulator.java

@@ -14,9 +14,10 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.binders;
+package io.nosqlbench.cqlgen.binders;
 
-import io.nosqlbench.cqlgen.model.CqlColumnDef;
+import io.nosqlbench.cqlgen.api.BindingsLibrary;
+import io.nosqlbench.cqlgen.model.CqlColumnBase;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -41,10 +42,10 @@ public class BindingsAccumulator {
         this.libraries = libraries;
     }
 
-    public Binding forColumn(CqlColumnDef def, String... prefixes) {
+    public Binding forColumn(CqlColumnBase def, String... prefixes) {
         return forColumn(def,Map.of(), prefixes);
     }
-    public Binding forColumn(CqlColumnDef def, Map<String,String> extra, String... prefixes) {
+    public Binding forColumn(CqlColumnBase def, Map<String,String> extra, String... prefixes) {
         for (BindingsLibrary library : libraries) {
             Optional<Binding> optionalBinding = switch (namingStyle) {
                 case FullyQualified -> this.resolveFullyQualifiedBinding(def, extra);
@@ -71,11 +72,11 @@ public class BindingsAccumulator {
     }
 
 
-    private Optional<Binding> resolvedCondensedBinding(CqlColumnDef def, Map<String,String> extra) {
+    private Optional<Binding> resolvedCondensedBinding(CqlColumnBase def, Map<String,String> extra) {
         throw new RuntimeException("Implement me!");
     }
 
-    private Optional<Binding> resolveSymbolicBinding(CqlColumnDef def, Map<String,String> extra) {
+    private Optional<Binding> resolveSymbolicBinding(CqlColumnBase def, Map<String,String> extra) {
         for (BindingsLibrary library : libraries) {
             Optional<Binding> binding = library.resolveBindingsFor(def);
             if (binding.isPresent()) {
@@ -89,7 +90,7 @@ public class BindingsAccumulator {
 
     }
 
-    private Optional<Binding> resolveFullyQualifiedBinding(CqlColumnDef def, Map<String,String> extra) {
+    private Optional<Binding> resolveFullyQualifiedBinding(CqlColumnBase def, Map<String,String> extra) {
         for (BindingsLibrary library : libraries) {
             Optional<Binding> bindingRecipe = library.resolveBindingsFor(def);
             if (bindingRecipe.isPresent()) {

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/NamingFolio.java

@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.binders;
+package io.nosqlbench.cqlgen.binders;
 
-import io.nosqlbench.cqlgen.model.CqlColumnDef;
+import io.nosqlbench.cqlgen.model.CqlColumnBase;
 import io.nosqlbench.cqlgen.model.CqlModel;
 import io.nosqlbench.cqlgen.model.CqlTable;
-import io.nosqlbench.cqlgen.exporter.CGElementNamer;
+import io.nosqlbench.cqlgen.core.CGElementNamer;
 import io.nosqlbench.api.labels.Labeled;
 
 import java.util.*;
@@ -84,7 +84,7 @@ public class NamingFolio {
 
     public void informNamerOfAllKnownNames(CqlModel model) {
         for (CqlTable table : model.getTableDefs()) {
-            for (CqlColumnDef coldef : table.getColumnDefinitions()) {
+            for (CqlColumnBase coldef : table.getColumnDefs()) {
                 addFieldRef(coldef.getLabels());
             }
         }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/NamingStyle.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.binders;
+package io.nosqlbench.cqlgen.binders;
 
 public enum NamingStyle {
     /**

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/UnresolvedBindingException.java

@@ -14,14 +14,14 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.binders;
+package io.nosqlbench.cqlgen.binders;
 
-import io.nosqlbench.cqlgen.model.CqlColumnDef;
+import io.nosqlbench.cqlgen.model.CqlColumnBase;
 
 public class UnresolvedBindingException extends RuntimeException {
-    private final CqlColumnDef def;
+    private final CqlColumnBase def;
 
-    public UnresolvedBindingException(CqlColumnDef def) {
+    public UnresolvedBindingException(CqlColumnBase def) {
         this.def = def;
     }
 

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingBuilder.java

@@ -14,12 +14,9 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.binders;
+package io.nosqlbench.cqlgen.bindspecs;
 
-import io.nosqlbench.cqlgen.model.CqlColumnDef;
+public interface BindingBuilder {
-
+    BindingBuilder prependRaw(String prefunc);
-import java.util.Optional;
+    BindingSpec build();
-
-public interface BindingsLibrary {
-    Optional<Binding> resolveBindingsFor(CqlColumnDef def);
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingSpec.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.bindspecs;
+
+import io.nosqlbench.api.labels.Labeled;
+
+public interface BindingSpec {
+
+    /**
+     * Is this binding intended to be for a limited population?
+     * If so, the value will be the maximum cardinality of values which the binding
+     * is allowed to produce.
+     * @return The effective cardinality, which could be {@link Double#POSITIVE_INFINITY}
+     */
+    default double getCardinality() {
+        return Double.POSITIVE_INFINITY;
+    }
+
+    /**
+     * The fully qualified name of the entity for which the binding values pertain.
+     * This is
+     * @return
+     */
+    Labeled getTarget();
+
+    String getTypedef();
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingSpecImpl.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.bindspecs;
+
+import io.nosqlbench.api.labels.Labeled;
+
+public class BindingSpecImpl implements BindingSpec {
+    private Labeled target;
+    private double cardinality;
+    private String typedef;
+
+    public BindingSpecImpl(Labeled target) {
+        this.target = target;
+    }
+
+    @Override
+    public Labeled getTarget() {
+        return target;
+    }
+
+    @Override
+    public String getTypedef() {
+        return typedef;
+    }
+
+    @Override
+    public double getCardinality() {
+        return BindingSpec.super.getCardinality();
+    }
+
+    public void setTarget(Labeled target) {
+        this.target = target;
+    }
+
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGColumnRebinder.java

@@ -14,11 +14,12 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter;
+package io.nosqlbench.cqlgen.core;
 
-import io.nosqlbench.cqlgen.model.CqlColumnDef;
+import io.nosqlbench.cqlgen.binders.Binding;
-import io.nosqlbench.cqlgen.exporter.binders.Binding;
+import io.nosqlbench.cqlgen.binders.BindingsAccumulator;
-import io.nosqlbench.cqlgen.exporter.binders.BindingsAccumulator;
+import io.nosqlbench.cqlgen.model.CqlTableColumn;
+import io.nosqlbench.cqlgen.model.FieldPosition;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -35,15 +36,15 @@ public class CGColumnRebinder {
         this.quantizerDigits = quantizerDigits;
     }
 
-    public Binding forColumn(CqlColumnDef cdef) {
+    public Binding forColumn(CqlTableColumn cdef) {
-        if (cdef.isPartitionKey()) {
+        if (cdef.getPosition()== FieldPosition.Partitioning) {
             return dividedBinding(cdef);
         } else {
             return accumulator.forColumn(cdef);
         }
     }
 
-    private Binding dividedBinding(CqlColumnDef column) {
+    private Binding dividedBinding(CqlTableColumn column) {
         CGTableStats stats = column.getTable().getTableAttributes();
         if (stats == null) {
             return accumulator.forColumn(column);

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java

@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter;
+package io.nosqlbench.cqlgen.core;
 
-import io.nosqlbench.cqlgen.model.CqlColumnDef;
+import io.nosqlbench.cqlgen.model.CqlColumnBase;
-import io.nosqlbench.cqlgen.exporter.binders.Binding;
+import io.nosqlbench.cqlgen.binders.Binding;
-import io.nosqlbench.cqlgen.exporter.binders.BindingsLibrary;
+import io.nosqlbench.cqlgen.api.BindingsLibrary;
 import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
 import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
 import io.nosqlbench.api.content.Content;
@@ -58,7 +58,7 @@ public class CGDefaultCqlBindings implements BindingsLibrary {
     }
 
     @Override
-    public Optional<Binding> resolveBindingsFor(CqlColumnDef def) {
+    public Optional<Binding> resolveBindingsFor(CqlColumnBase def) {
         String typedef = def.getTrimmedTypedef();
         String recipe = bindings.get(def.getTrimmedTypedef());
         Binding optionalBinding = new Binding(typedef, recipe);

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGElementNamer.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter;
+package io.nosqlbench.cqlgen.core;
 
 import io.nosqlbench.api.labels.Labeled;
 

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGExporterConfig.java

@@ -14,11 +14,10 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.traverser;
+package io.nosqlbench.cqlgen.core;
 
-import java.nio.file.Path;
+public class CGExporterConfig {
+    public CGExporterConfig(String[] args) {
 
-public class CqlDDlDirectoryTraverser {
-    public void buildWorkloads(Path sourcePath, Path targetPath) {
     }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGKeyspaceStats.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter;
+package io.nosqlbench.cqlgen.core;
 
 import java.util.HashMap;
 import java.util.Map;

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGLiteralFormat.java

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter;
+package io.nosqlbench.cqlgen.core;
 
-import io.nosqlbench.cqlgen.model.CqlColumnDef;
+import io.nosqlbench.cqlgen.model.CqlColumnBase;
 
 import java.util.Locale;
 import java.util.Optional;
@@ -49,7 +49,7 @@ public enum CGLiteralFormat {
         this.literalFormat = modifier;
     }
 
-    public static String formatBindType(CqlColumnDef cd, String valueref) {
+    public static String formatBindType(CqlColumnBase cd, String valueref) {
         return CGLiteralFormat.valueOfCqlType(cd.getTrimmedTypedef()).orElse(CGLiteralFormat.UNKNOWN).format(valueref);
     }
 

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGSchemaStats.java

@@ -14,12 +14,13 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter;
+package io.nosqlbench.cqlgen.core;
 
 import java.util.HashMap;
 import java.util.Map;
 
 public class CGSchemaStats {
+
     Map<String, CGKeyspaceStats> keyspaces = new HashMap<String, CGKeyspaceStats>();
 
     public Map<String, CGKeyspaceStats> getKeyspaces() {

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGTableStats.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter;
+package io.nosqlbench.cqlgen.core;
 
 import java.util.HashMap;
 import java.util.Map;

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGTextTransformers.java

@@ -14,10 +14,11 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter;
+package io.nosqlbench.cqlgen.core;
 
-import io.nosqlbench.cqlgen.exporter.transformers.CGNameObfuscator;
+import io.nosqlbench.cqlgen.api.CGTextTransformer;
-import io.nosqlbench.cqlgen.exporter.transformers.CGTransformerConfigurable;
+import io.nosqlbench.cqlgen.transformers.CGNameObfuscator;
+import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java

@@ -14,17 +14,21 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter;
+package io.nosqlbench.cqlgen.core;
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
-import io.nosqlbench.cqlgen.exporter.binders.BindingsAccumulator;
+import io.nosqlbench.api.content.Content;
-import io.nosqlbench.cqlgen.exporter.binders.BindingsLibrary;
+import io.nosqlbench.api.content.NBIO;
-import io.nosqlbench.cqlgen.exporter.transformers.CGModelTransformers;
+import io.nosqlbench.api.spi.BundledApp;
-import io.nosqlbench.cqlgen.parser.CqlModelParser;
+import io.nosqlbench.cqlgen.binders.Binding;
-import io.nosqlbench.cqlgen.exporter.binders.Binding;
+import io.nosqlbench.cqlgen.binders.BindingsAccumulator;
-import io.nosqlbench.cqlgen.exporter.binders.NamingFolio;
+import io.nosqlbench.cqlgen.api.BindingsLibrary;
+import io.nosqlbench.cqlgen.binders.NamingFolio;
+import io.nosqlbench.cqlgen.transformers.CGModelTransformers;
 import io.nosqlbench.cqlgen.model.*;
+import io.nosqlbench.cqlgen.parser.CqlModelParser;
+import io.nosqlbench.nb.annotations.Service;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.snakeyaml.engine.v2.api.Dump;
@@ -40,7 +44,6 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.*;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 
 /**
@@ -51,11 +54,12 @@ import java.util.stream.Collectors;
  *
  * @see <a href="https://cassandra.apache.org/doc/trunk/cassandra/cql/index.html">Apache Cassandra CQL Docs</a>
  */
-public class CGWorkloadExporter {
+@Service(value = BundledApp.class, selector = "cqlgen")
+public class CGWorkloadExporter implements BundledApp {
     private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.class);
     private CGColumnRebinder binder;
     private NamingFolio namer;
-    private final CqlModel model;
+    private CqlModel model;
 
     private final int DEFAULT_RESOLUTION = 10000;
 
@@ -79,38 +83,16 @@ public class CGWorkloadExporter {
         "update", 10.0
     ));
 
-    public CGWorkloadExporter(CqlModel model, CGModelTransformers transformers) {
+    private CGExporterConfig config;
-        this.model = model;
-        for (Function<CqlModel, CqlModel> transformer : transformers.get()) {
-            CqlModel modified = transformer.apply(this.model);
-            model = modified;
-        }
-    }
-
-    public CGWorkloadExporter(String ddl, Path srcpath, CGModelTransformers transformers) {
-        this(CqlModelParser.parse(ddl, srcpath), transformers);
-    }
-
-    public CGWorkloadExporter(String ddl, CGModelTransformers transformers) {
-        this(ddl, null, transformers);
-    }
-
-    public CGWorkloadExporter(Path path, CGModelTransformers transformers) {
-        this(CqlModelParser.parse(path), transformers);
-    }
-
-    private String loadFile(Path path) {
-        try {
-            String ddl = Files.readString(path);
-            logger.info("read " + ddl.length() + " character DDL file");
-            return ddl;
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-
-    }
 
     public static void main(String[] args) {
+        new CGWorkloadExporter().appMain(args);
+    }
+
+    @Override
+    public int appMain(String[] args) {
+        this.config = new CGExporterConfig(args);
+
         logger.info("running CQL workload exporter with args:" + Arrays.toString(args));
 
         if (args.length == 0) {
@@ -142,24 +124,25 @@ public class CGWorkloadExporter {
         }
 
         Yaml yaml = new Yaml();
-        Path cfgpath = Path.of("cqlgen.conf");
-
         CGWorkloadExporter exporter;
-        if (Files.exists(cfgpath)) {
+
-            try {
+        Content<?> cqlgencfg = NBIO.local().prefix("cqlgen").name("cqlgen").extension("conf").first().orElseThrow();
+        if (cqlgencfg == null) {
+            throw new RuntimeException("Unable to load cqlgen.conf");
+        }
+        Map cfgmap = yaml.loadAs(cqlgencfg.getInputStream(), Map.class);
+
         CGModelTransformers modelTransformers = new CGModelTransformers();
         CGTextTransformers textTransformers = new CGTextTransformers();
-                String configfile = Files.readString(cfgpath);
-                Map cfgmap = yaml.loadAs(configfile, Map.class);
 
         if (cfgmap.containsKey("model_transformers")) {
             modelTransformers.accept((List<Map<String, ?>>) cfgmap.get("model_transformers"));
         }
 
-                Object txtr = cfgmap.get("text_transformers");
+//        Object txtr = cfgmap.get("text_transformers");
-                if (txtr != null) {
+//        if (txtr != null) {
-                    textTransformers.accept((List<Map<String, ?>>) cfgmap.get("text_transformers"));
+//            textTransformers.accept((List<Map<String, ?>>) cfgmap.get("text_transformers"));
-                }
+//        }
 
         String ddl;
         try {
@@ -172,20 +155,27 @@ public class CGWorkloadExporter {
             throw new RuntimeException(e);
         }
 
-                exporter = new CGWorkloadExporter(ddl, srcpath, modelTransformers);
 
         String defaultNamingTemplate = cfgmap.get("naming_template").toString();
-                exporter.setNamingTemplate(defaultNamingTemplate);
+        setNamingTemplate(defaultNamingTemplate);
 
         String partition_multipler = cfgmap.get("partition_multiplier").toString();
-                exporter.setPartitionMultiplier(Double.parseDouble(partition_multipler));
+        setPartitionMultiplier(Double.parseDouble(partition_multipler));
+        configureTimeouts(cfgmap.get("timeouts"));
+        configureBlocks(cfgmap.get("blockplan"));
+        configureQuantizerDigits(cfgmap.get("quantizer_digits"));
 
-                exporter.configureTimeouts(cfgmap.get("timeouts"));
+        this.model = CqlModelParser.parse(ddl, srcpath);
+        List<String> errorlist = model.getReferenceErrors();
+        if (errorlist.size()>0) {
+            for (String error : errorlist) {
+                logger.error(error);
+            }
+            throw new RuntimeException("there were " + errorlist.size() + " reference errors in the model.");
+        }
+        this.model = modelTransformers.apply(this.model);
 
-                exporter.configureBlocks(cfgmap.get("blockplan"));
+        String workload = getWorkloadAsYaml();
-                exporter.configureQuantizerDigits(cfgmap.get("quantizer_digits"));
-
-                String workload = exporter.getWorkloadAsYaml();
         try {
             Files.writeString(
                 target,
@@ -196,12 +186,20 @@ public class CGWorkloadExporter {
             e.printStackTrace();
         }
 
+        return 0;
+    }
 
+    private String loadFile(Path path) {
+        try {
+            String ddl = Files.readString(path);
+            logger.info("read " + ddl.length() + " character DDL file");
+            return ddl;
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
+
     }
-    }
+
 
     private void configureQuantizerDigits(Object quantizer_digits) {
         if (quantizer_digits != null) {
@@ -237,10 +235,10 @@ public class CGWorkloadExporter {
                     case "drop-tables" -> genDropTablesBlock(model, blockname);
                     case "drop-keyspaces" -> genDropKeyspacesOpTemplates(model, blockname);
                     case "truncate-tables" -> genTruncateTablesOpTemplates(model, blockname);
-                    case "insert" -> genInsertOpTemplates(model, blockname);
+                    case "insert-seq" -> genInsertOpTemplates(model, blockname);
-                    case "select" -> genSelectOpTemplates(model, blockname);
+                    case "select-seq" -> genSelectOpTemplates(model, blockname);
-                    case "scan-10" -> genScanOpTemplates(model, blockname);
+                    case "scan-10-seq" -> genScanOpTemplates(model, blockname);
-                    case "update" -> genUpdateOpTemplates(model, blockname);
+                    case "update-seq" -> genUpdateOpTemplates(model, blockname);
                     default -> throw new RuntimeException("Unable to create block entries for " + component + ".");
                 };
                 block.putAll(additions);
@@ -367,7 +365,7 @@ public class CGWorkloadExporter {
             where PREDICATE
             LIMIT;
             """
-            .replace("KEYSPACE", table.getKeySpace())
+            .replace("KEYSPACE", table.getKeyspace().getName())
             .replace("TABLE", table.getName())
             .replace("PREDICATE", genPredicateTemplate(table, -1))
             .replace("LIMIT", genLimitSyntax(table));
@@ -397,7 +395,7 @@ public class CGWorkloadExporter {
             where PREDICATE
             LIMIT;
             """
-            .replace("KEYSPACE", table.getKeySpace())
+            .replace("KEYSPACE", table.getKeyspace().getName())
             .replace("TABLE", table.getName())
             .replace("PREDICATE", genPredicateTemplate(table, 0))
             .replace("LIMIT", genLimitSyntax(table));
@@ -437,16 +435,16 @@ public class CGWorkloadExporter {
             VALUES
             ( BINDINGS );
             """
-            .replace("KEYSPACE", table.getKeySpace())
+            .replace("KEYSPACE", table.getKeyspace().getName())
             .replace("TABLE", table.getName())
             .replace("FIELDNAMES",
                 String.join(", ",
-                    table.getColumnDefinitions().stream()
+                    table.getColumnDefs().stream()
-                        .map(CqlColumnDef::getName).toList()))
+                        .map(CqlTableColumn::getName).toList()))
             .replaceAll("BINDINGS",
                 String.join(", ",
-                    table.getColumnDefinitions().stream()
+                    table.getColumnDefs().stream()
-                        .map(binder::forColumn)
+                        .map(c -> binder.forColumn(c))
                         .map(c -> "{" + c.getName() + "}").toList()));
     }
 
@@ -470,22 +468,23 @@ public class CGWorkloadExporter {
 
 
     private boolean isCounterTable(CqlTable table) {
-        return table.getColumnDefinitions().stream()
+        return table.getColumnDefs().stream()
             .anyMatch(cd -> cd.getTrimmedTypedef().equalsIgnoreCase("counter"));
     }
 
     private int totalRatioFor(CqlTable table) {
-        if (table.getTableAttributes() == null || table.getTableAttributes().size() == 0) {
+        if (table.hasStats()) {
+            return readRatioFor(table) + writeRatioFor(table);
+        } else {
             return 1;
         }
-        return readRatioFor(table) + writeRatioFor(table);
     }
 
     private int readRatioFor(CqlTable table) {
         if (table.getTableAttributes() == null || table.getTableAttributes().size() == 0) {
             return 1;
         }
-        double weighted_reads = Double.parseDouble(table.getTableAttributes().getAttribute("weighted_reads"));
+        double weighted_reads = table.getComputedStats().getWeightedReadsOfTotal();
         return (int) (weighted_reads * DEFAULT_RESOLUTION);
     }
 
@@ -493,7 +492,7 @@ public class CGWorkloadExporter {
         if (table.getTableAttributes() == null || table.getTableAttributes().size() == 0) {
             return 1;
         }
-        double weighted_writes = Double.parseDouble(table.getTableAttributes().getAttribute("weighted_writes"));
+        double weighted_writes = table.getComputedStats().getWeightedWritesOfTotal();
         return (int) (weighted_writes * DEFAULT_RESOLUTION);
     }
 
@@ -511,13 +510,13 @@ public class CGWorkloadExporter {
     private String genPredicateTemplate(CqlTable table, int keycount) {
 
         StringBuilder sb = new StringBuilder();
-        LinkedList<CqlColumnDef> pkeys = new LinkedList<>();
+        LinkedList<CqlTableColumn> pkeys = new LinkedList<>();
         for (String pkey : table.getPartitionKeys()) {
-            CqlColumnDef coldef = table.getColumnDefForName(pkey);
+            CqlTableColumn coldef = table.getColumnDefForName(pkey);
             pkeys.push(coldef);
         }
         for (String ccol : table.getClusteringColumns()) {
-            CqlColumnDef coldef = table.getColumnDefForName(ccol);
+            CqlTableColumn coldef = table.getColumnDefForName(ccol);
             pkeys.push(coldef);
         }
 
@@ -547,7 +546,7 @@ public class CGWorkloadExporter {
         return sb.toString();
     }
 
-    private String genPredicatePart(CqlColumnDef def) {
+    private String genPredicatePart(CqlTableColumn def) {
         String typeName = def.getTrimmedTypedef();
         Binding binding = binder.forColumn(def);
         return def.getName() + "={" + binding.getName() + "}";
@@ -559,7 +558,7 @@ public class CGWorkloadExporter {
             set ASSIGNMENTS
             where PREDICATES;
             """
-            .replaceAll("KEYSPACE", table.getKeySpace())
+            .replaceAll("KEYSPACE", table.getKeyspace().getName())
             .replaceAll("TABLE", table.getName())
             .replaceAll("PREDICATES", genPredicateTemplate(table, 0))
             .replaceAll("ASSIGNMENTS", genAssignments(table));
@@ -567,7 +566,7 @@ public class CGWorkloadExporter {
 
     private String genAssignments(CqlTable table) {
         StringBuilder sb = new StringBuilder();
-        for (CqlColumnDef coldef : table.getNonKeyColumnDefinitions()) {
+        for (CqlTableColumn coldef : table.getNonKeyColumnDefinitions()) {
             if (coldef.isCounter()) {
                 sb.append(coldef.getName()).append("=")
                     .append(coldef.getName()).append("+").append("{").append(binder.forColumn(coldef).getName()).append("}")
@@ -699,7 +698,7 @@ public class CGWorkloadExporter {
         Map<String, Object> schemablock = new LinkedHashMap<>();
         Map<String, Object> ops = new LinkedHashMap<>();
 
-        for (CqlKeyspace ks : model.getKeyspacesByName().values()) {
+        for (CqlKeyspaceDef ks : model.getKeyspaceDefs()) {
             ops.put(
                 namer.nameFor(ks, "optype", "create", "blockname", blockname),
                 Map.of(
@@ -718,22 +717,21 @@ public class CGWorkloadExporter {
         Map<String, Object> ops = new LinkedHashMap<>();
         blockdata.put("ops", ops);
 
-        for (String keyspace : model.getTypesByKeyspaceThenName().keySet()) {
+        model.getTypeDefs().forEach(type -> {
-            for (CqlType type : model.getTypesByKeyspaceThenName().get(keyspace).values()) {
             ops.put(
-                    namer.nameFor(type, "optype", "create", "blockname", blockname),
+                namer.nameFor(type,"optype","create","blockname",blockname),
                 Map.of(
-                        "simple", genTypeDDL(type),
+                    "simple",genTypeDDL(type),
-                        "timeout", timeouts.get("create")
+                    "timeout",timeouts.get("create")
                 )
             );
-            }
+        });
-        }
+
         return blockdata;
 
     }
 
-    private String genKeyspaceDDL(CqlKeyspace keyspace) {
+    private String genKeyspaceDDL(CqlKeyspaceDef keyspace) {
         return """
             create keyspace KEYSPACE
             with replication = {REPLICATION}DURABLEWRITES?;
@@ -748,17 +746,15 @@ public class CGWorkloadExporter {
         Map<String, Object> schemablock = new LinkedHashMap<>();
         Map<String, Object> ops = new LinkedHashMap<>();
 
-        for (String ksname : model.getTableDefsByKeyspaceThenTable().keySet()) {
+        model.getTableDefs().forEach(table -> {
-            for (CqlTable cqltable : model.getTableDefsByKeyspaceThenTable().get(ksname).values()) {
             ops.put(
-                    namer.nameFor(cqltable, "optype", "create", "blockname", blockname),
+                namer.nameFor(table, "optype","create","blockname",blockname),
                 Map.of(
-                        "simple", genTableDDL(cqltable),
+                    "simple",genTableDDL(table),
-                        "timeout", timeouts.get("create")
+                    "timeout",timeouts.get("create")
                 )
             );
-            }
+        });
-        }
 
         schemablock.put("ops", ops);
         return schemablock;
@@ -771,10 +767,10 @@ public class CGWorkloadExporter {
             TYPEDEF
             );
             """
-            .replace("KEYSPACE", type.getKeyspace())
+            .replace("KEYSPACE", type.getKeyspace().getName())
             .replace("TYPENAME", type.getName())
-            .replace("TYPEDEF", type.getFields().entrySet().stream()
+            .replace("TYPEDEF", type.getColumnDefs().stream()
-                .map(entry -> entry.getKey() + " " + entry.getValue()).collect(Collectors.joining(",\n")));
+                .map(def -> def.getName() + " " + def.getTypedef()).collect(Collectors.joining(",\n")));
     }
 
     private Object genTableDDL(CqlTable cqltable) {
@@ -788,7 +784,7 @@ public class CGWorkloadExporter {
             primary key (PRIMARYKEY)
             )CLUSTERING;
             """
-            .replace("KEYSPACE", cqltable.getKeySpace())
+            .replace("KEYSPACE", cqltable.getKeyspace().getName())
             .replace("TABLE", cqltable.getName())
             .replace("COLUMN_DEFS", genTableColumnDDL(cqltable))
             .replace("PRIMARYKEY", genPrimaryKeyDDL(cqltable))
@@ -829,7 +825,7 @@ public class CGWorkloadExporter {
     }
 
     private String genTableColumnDDL(CqlTable cqltable) {
-        return cqltable.getColumnDefinitions().stream()
+        return cqltable.getColumnDefs().stream()
             .map(cd -> cd.getName() + " " + cd.getTrimmedTypedef())
             .collect(Collectors.joining(",\n"));
     }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGCachingNameRemapper.java

@@ -1,64 +0,0 @@
-/*
- * Copyright (c) 2022 nosqlbench
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.nosqlbench.cqlgen.exporter.transformers;
-
-import io.nosqlbench.api.config.NBNamedElement;
-import io.nosqlbench.api.labels.Labeled;
-import io.nosqlbench.virtdata.library.basics.shared.from_long.to_string.Combinations;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.function.Function;
-import java.util.function.LongFunction;
-
-public class CGCachingNameRemapper {
-    private LongFunction<String> namefunc;
-    private final Map<String,String> remapped = new HashMap<>();
-    private long index=0;
-
-    public CGCachingNameRemapper() {
-        this.namefunc = new Combinations("a-z;a-z;a-z;a-z;a-z;a-z;");
-    }
-    public CGCachingNameRemapper(LongFunction<String> function) {
-        this.namefunc = function;
-    }
-
-    public synchronized String nameForType(String type, String originalName, String prefix) {
-        String canonical = type+"_"+originalName;
-        return getOrCreateName(canonical, prefix);
-    }
-    public synchronized String nameFor(NBNamedElement element, String prefix) {
-        String canonical = element.getClass().getSimpleName()+"-"+element.getName();
-        return getOrCreateName(canonical, prefix);
-    }
-
-    private String getOrCreateName(String canonical, String prefix) {
-        if (!remapped.containsKey(canonical)) {
-            String newname = prefix+namefunc.apply(index++);
-            remapped.put(canonical,newname);
-        }
-        return remapped.get(canonical);
-    }
-
-    public Function<String, String> mapperForType(Labeled cqlTable, String prefix) {
-        return in -> this.nameForType(cqlTable.getClass().getSimpleName(),in, prefix);
-    }
-
-    public void setNamingFunction(LongFunction<String> namerFunc) {
-        this.namefunc = namerFunc;
-    }
-}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGIfNotExistsInjector.java

@@ -1,52 +0,0 @@
-/*
- * Copyright (c) 2022 nosqlbench
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.nosqlbench.cqlgen.exporter.transformers;
-
-import io.nosqlbench.cqlgen.model.CqlModel;
-
-/**
- * @deprecated Superseded by direct rendering from AST in generator
- */
-public class CGIfNotExistsInjector implements CGModelTransformer {
-
-    @Override
-    public CqlModel apply(CqlModel model) {
-//        for (CqlKeyspace keyspace : model.getKeyspaceDefs()) {
-//            keyspace.setRefDdl(keyspace.getRefddl().replaceAll(
-//                "(?m)(?s)(?i)(\\s*CREATE (TABLE|KEYSPACE|TYPE) +)(?!IF NOT EXISTS)",
-//                "$1IF NOT EXISTS "
-//            ));
-//        }
-//        for (CqlTable table : model.getTableDefs()) {
-//            String refddl = table.getRefDdl();
-//            String replaced = refddl.replaceAll(
-//                "(?m)(?s)(?i)(\\s*CREATE (TABLE|KEYSPACE|TYPE) +)(?!IF NOT EXISTS)",
-//                "$1IF NOT EXISTS "
-//            );
-//
-//            table.setRefDdl(replaced);
-//        }
-//        for (CqlType type : model.getTypes()) {
-//            type.setRefddl(type.getRefDdl().replaceAll(
-//                "(?m)(?s)(?i)(\\s*CREATE (TABLE|KEYSPACE|TYPE) +)(?!IF NOT EXISTS)",
-//                "$1IF NOT EXISTS "
-//            ));
-//
-//        }
-        return model;
-    }
-}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGNameObfuscator.java

@@ -1,81 +0,0 @@
-/*
- * Copyright (c) 2022 nosqlbench
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Create random but stable names for all elements.
- * Use a deterministic method for creating names.
- * once an element is named, use the same name throughout
- * prefix each element type with a code for the type
- */
-package io.nosqlbench.cqlgen.exporter.transformers;
-
-import io.nosqlbench.cqlgen.model.CqlModel;
-import io.nosqlbench.cqlgen.model.CqlTable;
-import io.nosqlbench.cqlgen.model.CqlType;
-import io.nosqlbench.virtdata.core.bindings.DataMapper;
-import io.nosqlbench.virtdata.core.bindings.VirtData;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.Map;
-import java.util.Optional;
-import java.util.function.LongFunction;
-
-public class CGNameObfuscator implements CGModelTransformer, CGTransformerConfigurable {
-    private final static Logger logger = LogManager.getLogger(CGNameObfuscator.class);
-
-    private final CGCachingNameRemapper remapper = new CGCachingNameRemapper();
-
-    @Override
-    public CqlModel apply(CqlModel model) {
-
-        for (String keyspaceName : model.getAllKnownKeyspaceNames()) {
-            String newKeyspaceName = remapper.nameForType("keyspace", keyspaceName, "ks_");
-            model.renamekeyspace(keyspaceName, newKeyspaceName);
-        }
-
-        for (CqlTable cqlTable : model.getTableDefs()) {
-            String tablename = cqlTable.getName();
-            String newTableName = remapper.nameFor(cqlTable, "tbl_");
-            model.renameTable(cqlTable, newTableName);
-            cqlTable.renameColumns(remapper.mapperForType(cqlTable, "col_"));
-        }
-
-        for (CqlType type : model.getTypeDefs()) {
-            String typeName = type.getName();
-            String newTypeName = remapper.nameFor(type, "typ_");
-            model.renameType(type.getKeyspace(), typeName, newTypeName);
-            type.renameColumns(remapper.mapperForType(type, "typ"));
-        }
-
-        return model;
-
-    }
-
-    @Override
-    public void accept(Object configObject) {
-        if (configObject instanceof Map cfgmap) {
-            Object namer = cfgmap.get("namer");
-            Optional<DataMapper<String>> optionalMapper = VirtData.getOptionalMapper(namer.toString());
-            LongFunction<String> namerFunc = optionalMapper.orElseThrow(
-                () -> new RuntimeException("Unable to resolve obfuscator namer '" + namer + "'")
-            );
-            remapper.setNamingFunction(namerFunc);
-        } else {
-            throw new RuntimeException("name obfuscator requires a map for its configuration value.");
-        }
-    }
-}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/ComputedSchemaStats.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.model;
+
+public class ComputedSchemaStats {
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlColumnBase.java

@@ -21,42 +21,42 @@ import io.nosqlbench.api.labels.Labeled;
 
 import java.util.Map;
 
-public class CqlColumnDef implements NBNamedElement, Labeled {
+/**
-    private CqlTable table;
+ * Not anchored to a parent, as it could be a table or a type.
-    private String keyspace;
+ * All access to these must be through their parent element.
-    private String name;
+ */
-    private String type;
+public abstract class CqlColumnBase implements NBNamedElement, Labeled {
-    private final int position;
 
-    public CqlColumnDef(CqlTable table, int position, String colname, String typedef) {
+    private String name;
-        this.table = table;
+    private String typedef;
-        this.position = position;
+    private FieldPosition position;
-        this.type = typedef;
+
+    public CqlColumnBase(String colname, String typedef) {
+        this.typedef = typedef;
         this.name = colname;
     }
 
+    public void setPosition(FieldPosition position) {
+        this.position = position;
+    }
+
+    public FieldPosition getPosition() {
+        return this.position;
+    }
     public void setTypeDef(String type) {
-        this.type = type;
+        this.typedef = type;
     }
 
     public String getName() {
         return name;
     }
 
-    public String getType() {
+    public String getTypedef() {
-        return type;
+        return typedef;
     }
 
     public String getTrimmedTypedef() {
-        return type.replaceAll(" ", "");
+        return typedef.replaceAll(" ", "");
-    }
-
-    public String getTableName() {
-        return table.getName();
-    }
-
-    public String getKeyspace() {
-        return keyspace;
     }
 
     @Override
@@ -68,21 +68,11 @@ public class CqlColumnDef implements NBNamedElement, Labeled {
     public Map<String, String> getLabels() {
         return Map.of(
             "name", name,
-            "typedef", type,
+            "typedef", typedef,
-            "table", table.getName(),
-            "keyspace", keyspace,
             "type", "column"
         );
     }
 
-    public void setKeyspace(String keyspace) {
-        this.keyspace = keyspace;
-    }
-
-    public void setTable(CqlTable table) {
-        this.table = table;
-    }
-
     public boolean isCounter() {
         return getTrimmedTypedef().equalsIgnoreCase("counter");
     }
@@ -95,27 +85,10 @@ public class CqlColumnDef implements NBNamedElement, Labeled {
         return getName() + " " + getTrimmedTypedef();
     }
 
-    public boolean isPartitionKey() {
-        return table.isPartitionKey(position);
-    }
-
-    public boolean isLastPartitionKey() {
-        return table.isLastPartitionKey(position);
-    }
-
-    public boolean isClusteringColumn() {
-        return table.isClusteringColumn(position);
-    }
-
-    public boolean isLastClusteringColumn() {
-        return table.isLastClusteringColumn(position);
-    }
-
-    public CqlTable getTable() {
-        return this.table;
-    }
-
     public String getFullName() {
-        return getKeyspace() + "." + getTable().getName() + "." + getName() + "(column)";
+        return getParentFullName() + "." + getName();
     }
+
+    protected abstract String getParentFullName();
+
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlKeyspaceDef.java

@@ -16,19 +16,31 @@
 
 package io.nosqlbench.cqlgen.model;
 
+import com.datastax.oss.driver.internal.core.util.Strings;
 import io.nosqlbench.api.config.NBNamedElement;
 import io.nosqlbench.api.labels.Labeled;
-import io.nosqlbench.cqlgen.exporter.CGKeyspaceStats;
+import io.nosqlbench.cqlgen.core.CGKeyspaceStats;
 
-import java.util.Map;
+import java.util.*;
 
-public class CqlKeyspace implements NBNamedElement, Labeled {
+public class CqlKeyspaceDef implements NBNamedElement, Labeled {
     String keyspaceName= "";
     CGKeyspaceStats stats;
     private boolean isDurableWrites;
     private String replicationData;
+    private final List<CqlTable> tableDefs = new ArrayList<>();
+    private final List<CqlType> typeDefs = new ArrayList<>();
+    /**
+     * Has this been populated by keyspace definition? If false, it is only
+     * here because it was vivified by a reference.
+     */
+    private transient boolean defined;
 
-    public CqlKeyspace() {
+    public CqlKeyspaceDef() {
+    }
+
+    public CqlKeyspaceDef(String ksname) {
+        setKeyspaceName(ksname);
     }
 
     public void setKeyspaceName(String newname) {
@@ -39,7 +51,6 @@ public class CqlKeyspace implements NBNamedElement, Labeled {
         return this.keyspaceName;
     }
 
-
     @Override
     public String toString() {
         return "CqlKeyspace{" +
@@ -77,4 +88,53 @@ public class CqlKeyspace implements NBNamedElement, Labeled {
     public String getReplicationData() {
         return this.replicationData;
     }
+
+    public CqlTable getTable(String table) {
+        return this.tableDefs.stream().filter(t -> t.getName().equals(table)).findAny().orElse(null);
+    }
+
+    public void addTable(CqlTable table) {
+        table.setKeyspace(this);
+        this.tableDefs.add(table);
+    }
+
+    public List<CqlType> getTypeDefs() {
+        return this.typeDefs;
+    }
+
+    public List<CqlTable> getTableDefs() {
+        return this.tableDefs;
+
+    }
+
+    public void removeTable(CqlTable table) {
+        this.tableDefs.remove(table.getName());
+    }
+
+    public void getReferenceErrors(List<String> errors) {
+        if (!defined) {
+            errors.add("keyspace " + this.getName() + " was referenced but not defined.");
+        }
+        for (CqlType typedef : typeDefs) {
+            typedef.getReferenceErrors(errors);
+        }
+        for (CqlTable value : tableDefs) {
+            value.getReferenceErrors(errors);
+        }
+    }
+
+    public void setDefined() {
+        if (this.keyspaceName==null) {
+            throw new RuntimeException("nuh uh");
+        }
+        this.defined=true;
+    }
+
+    public void validate() {
+        Strings.requireNotEmpty(this.keyspaceName, "keyspace name");
+    }
+
+    public void addType(CqlType usertype) {
+        this.typeDefs.add(usertype);
+    }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModel.java

@@ -16,77 +16,86 @@
 
 package io.nosqlbench.cqlgen.model;
 
-import io.nosqlbench.cqlgen.exporter.CGKeyspaceStats;
+import io.nosqlbench.cqlgen.core.CGKeyspaceStats;
-import io.nosqlbench.cqlgen.exporter.CGSchemaStats;
+import io.nosqlbench.cqlgen.core.CGSchemaStats;
-import io.nosqlbench.cqlgen.exporter.CGTableStats;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 import java.util.function.Supplier;
 
 /**
+ * <p>
  * This model contains definition level details for schema elements which are parsed from the
  * Antlr4 CQL grammar.
- * Because keyspace, table, column, and type elements are handled sometimes in different ways,
+ * Key elements include:
- * these are stored in separate data structures.
+ * <UL>
- * When you see a *refddl or similar field, this is a copy of the text image from the original
+ * <LI>keyspace definitions, organized by keyspace name</LI>
- * parsed syntax. These are used for populating schema blocks without doing a full parse.
+ * <li>type definitions, organized by keyspace name</li>
- * If you update either the refddl or the actual AST level elements for any of the types in this
+ * <li>table definitions with included column definitions, organized by keyspace name</li>
- * model, you are required to update the other version along with it, using string substitution
+ * </UL>
- * if necessary.
+ * </p>
+ *
+ * <p>Because keyspace, table, and type elements are handled sometimes in different ways,
+ * these are stored in separate data structures, mapped by the logical keyspace name. This means
+ * that you will see table definitions for named keyspaces even if those named keyspaces are not represented
+ * in the keyspace definitions. This allows for sub-selecting of rendered elements by logical
+ * name without requiring a fully-interconnected keyspace->table->column object graph.
+ * </p>
  */
 public class CqlModel {
     private final static Logger logger = LogManager.getLogger(CqlModel.class);
 
     private final Supplier<List<String>> errors;
-    Map<String, CqlKeyspace> keyspaceDefs = new LinkedHashMap<>();
+    private final List<CqlKeyspaceDef> keyspaceDefs = new ArrayList();
-    Map<String, Map<String, CqlTable>> tableDefs = new LinkedHashMap<>();
-    Map<String, Map<String, CqlType>> typeDefs = new LinkedHashMap<>();
 
-   CGSchemaStats schemaStats = null;
+    private CGSchemaStats schemaStats = null;
+    private ComputedSchemaStats computedSchemaStats;
+    private Map<String,CqlKeyspaceDef> ksNameCache;
 
-    public CGSchemaStats getKeyspaceAttributes() {
+    public CGSchemaStats getStats() {
         return schemaStats;
     }
 
+    public boolean hasStats() {
+        return schemaStats != null;
+    }
+
+    public ComputedSchemaStats getComputedStats() {
+        return computedSchemaStats;
+    }
+
     public void setKeyspaceAttributes(CGSchemaStats schemaStats) {
         this.schemaStats = schemaStats;
         for (String statsKeyspacename : schemaStats.getKeyspaces().keySet()) {
             CGKeyspaceStats keyspaceStats = schemaStats.getKeyspace(statsKeyspacename);
-            if (keyspaceDefs.containsKey(statsKeyspacename)) {
+
+            CqlKeyspaceDef ksdef = getKeyspace(statsKeyspacename);
+            if (ksdef !=null) {
                 logger.debug("setting         keyspace stats for '" + statsKeyspacename + "'");
-                keyspaceDefs.get(statsKeyspacename).setStats(keyspaceStats);
+                ksdef.setStats(keyspaceStats);
+                keyspaceStats.getKeyspaceTables().forEach((tbname, tbstats) -> {
+                    CqlTable table = ksdef.getTable(tbname);
+                    if (table != null) {
+                        table.setStats(tbstats);
+                    } else {
+                        logger.debug(" skipping table '" + statsKeyspacename + "." + tbname + ", since it was not found in the model.");
+                    }
+                });
             } else {
                 logger.debug("       skipping keyspace stats for '" + statsKeyspacename + "'");
             }
 
-            for (String statsTableName : keyspaceStats.getKeyspaceTables().keySet()) {
-                CGTableStats tableStats = keyspaceStats.getKeyspaceTables().get(statsTableName);
-                Map<String, CqlTable> modelTables = tableDefs.get(statsKeyspacename);
-                if (modelTables!=null) {
-                    CqlTable modelTable = modelTables.get(statsTableName);
-                    if (modelTable!=null) {
-                        logger.debug("setting table stats for '" + statsKeyspacename+"."+statsTableName+"'");
-                        modelTable.setTableAttributes(tableStats);
-                    } else {
-                        logger.debug("       skipping table stats for '" + statsKeyspacename + "."+statsTableName+"'");
-                    }
-                } else {
-                    logger.debug("       SKIPPING stats for all tables in keyspace '" + statsKeyspacename + "'");
-                }
-            }
         }
     }
 
-    transient CqlKeyspace keyspace = null;
+    private CqlKeyspaceDef getKeyspace(String ksname) {
-    transient CqlTable table;
+        return this.keyspaceDefs.stream().filter(ksd -> ksd.getName().equals(ksname)).findAny().orElse(null);
-    transient CqlType udt;
-
-    public boolean hasStats() {
-        return schemaStats!=null;
     }
 
+
     public CqlModel(Supplier<List<String>> errorSource) {
         this.errors = errorSource;
     }
@@ -95,200 +104,68 @@ public class CqlModel {
         return errors.get();
     }
 
-    public void newKeyspace() {
+    public CqlKeyspaceDef refKeyspace(String ksname) {
-        keyspace = new CqlKeyspace();
+        CqlKeyspaceDef keyspace = getKeyspace(ksname);
+        if (getKeyspace(ksname)==null) {
+            keyspace = new CqlKeyspaceDef(ksname);
+            keyspaceDefs.add(keyspace);
+        }
+        return keyspace;
     }
 
-    public void saveKeyspace(String text,String refddl) {
-        keyspace.setKeyspaceName(text);
-        this.keyspaceDefs.put(text, keyspace);
-        keyspace=null;
-    }
 
-    public void newTable() {
+    public List<CqlKeyspaceDef> getKeyspaceDefs() {
-        table = new CqlTable();
+        return this.keyspaceDefs;
-    }
-
-    public void saveTable(String keyspace, String text) {
-        table.setKeyspace(keyspace);
-        table.setName(text);
-        this.tableDefs.computeIfAbsent(keyspace, ks->new LinkedHashMap<>()).put(text, table);
-        table = null;
-    }
-
-    public void saveColumnDefinition(String colname, String typedef, boolean isPrimaryKey, int position) {
-        this.table.addcolumnDef(colname, typedef, position);
-        if (isPrimaryKey) {
-            this.table.addPartitionKey(colname);
-        }
-    }
-
-    public Map<String, CqlKeyspace> getKeyspacesByName() {
-        return keyspaceDefs;
-    }
-
-    public List<CqlKeyspace> getKeyspaceDefs() {
-        return new ArrayList<>(this.keyspaceDefs.values());
-    }
-
-    public Map<String, Map<String, CqlTable>> getTableDefsByKeyspaceThenTable() {
-        return tableDefs;
-    }
-
-    public List<CqlTable> getTablesForKeyspace(String ksname) {
-        Map<String, CqlTable> tables = this.tableDefs.get(ksname);
-        if (tables!=null) {
-            return new ArrayList<>(tables.values());
-        }
-        return List.of();
-    }
-
-    public List<CqlTable> getTableDefs() {
-        return tableDefs.values().stream().flatMap(m->m.values().stream()).toList();
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder sb = new StringBuilder();
-        for (String ks : keyspaceDefs.keySet()) {
-            CqlKeyspace keyspace = keyspaceDefs.get(ks);
-            sb.append("keyspace '").append(keyspace.getName()).append("':\n");
-            sb.append(keyspace).append("\n");
-
-            tableDefs.getOrDefault(ks,Map.of()).values().stream()
-                .forEach(table -> {
-                    sb.append("table '").append(table.getName()).append("':\n");
-                    sb.append(table);
-                });
-        }
-        return sb.toString();
-    }
-
-    /**
-     * Get all the keyspace names which have been referenced in any way, whether or not
-     * this was in a keyspace definition or some other DDL like table or udt names.
-     * @return A list of all known keyspace names
-     */
-    public Set<String> getAllKnownKeyspaceNames() {
-        Set<String> ksnames = new LinkedHashSet<>();
-        ksnames.addAll(this.keyspaceDefs.keySet());
-        ksnames.addAll(this.tableDefs.keySet());
-        return ksnames;
-    }
-
-    public void addPartitionKey(String partitionKey) {
-        table.addPartitionKey(partitionKey);
-    }
-
-    public void addClusteringColumn(String ccolumn) {
-        table.addClusteringColumn(ccolumn);
-    }
-
-    public void newType() {
-        udt = new CqlType();
-    }
-
-    public void addTypeField(String name, String typedef) {
-        udt.addField(name, typedef);
-    }
-
-    public void saveType(String keyspace, String name) {
-        udt.setKeyspace(keyspace);
-        udt.setName(name);
-        Map<String, CqlType> ksTypes = this.typeDefs.computeIfAbsent(keyspace, ks -> new LinkedHashMap<>());
-        ksTypes.put(udt.getName(),udt);
-        udt=null;
     }
 
     public List<CqlType> getTypeDefs() {
-        ArrayList<CqlType> list = new ArrayList<>();
+        return this.keyspaceDefs.stream().flatMap(ks -> ks.getTypeDefs().stream()).toList();
-        for (Map<String, CqlType> cqlTypesByKeyspace : typeDefs.values()) {
-            for (CqlType cqlType : cqlTypesByKeyspace.values()) {
-                list.add(cqlType);
-            }
-        }
-        return list;
     }
 
     public void removeKeyspaceDef(String ksname) {
         this.keyspaceDefs.remove(ksname);
     }
 
-    public void removeTablesForKeyspace(String ksname) {
-        this.tableDefs.remove(ksname);
-    }
-
-    public void removeTypesForKeyspace(String name) {
-        this.typeDefs.remove(name);
-    }
-
     public String getSummaryLine() {
         return "keyspaces: " + keyspaceDefs.size() + ", tables: " + getTableDefs().size() +
-            ", columns: " + getTableDefs().stream().mapToInt(t -> t.getColumnDefinitions().size()).sum() +
+            ", columns: " + getTableDefs().stream().mapToInt(t -> t.getColumnDefs().size()).sum() +
             ", types: " + getTypeDefs().size();
     }
 
-    public void renamekeyspace(String keyspaceName, String newKeyspaceName) {
+    public List<CqlTable> getTableDefs() {
-        if (this.keyspaceDefs.containsKey(keyspaceName)) {
+        return this.keyspaceDefs.stream().flatMap(ks -> ks.getTableDefs().stream()).toList();
-            CqlKeyspace keyspace = this.keyspaceDefs.remove(keyspaceName);
-            keyspace.setKeyspaceName(newKeyspaceName);
-            this.keyspaceDefs.put(newKeyspaceName, keyspace);
-        }
-        if (this.tableDefs.containsKey(keyspaceName)) {
-            Map<String, CqlTable> tablesForKeyspace = this.tableDefs.remove(keyspaceName);
-            if (tablesForKeyspace!=null) {
-                for (CqlTable table : tablesForKeyspace.values()) {
-                    table.setKeyspace(newKeyspaceName);
-                }
-            }
-            this.tableDefs.put(newKeyspaceName, tablesForKeyspace);
-        }
-        if (this.typeDefs.containsKey(keyspaceName)) {
-            Map<String, CqlType> typesForKeyspace = this.typeDefs.remove(keyspaceName);
-            if (typesForKeyspace!=null) {
-                for (CqlType cqltype : typesForKeyspace.values()) {
-                    cqltype.setKeyspace(newKeyspaceName);
-                }
-            }
-            this.typeDefs.put(newKeyspaceName,typesForKeyspace);
-        }
     }
 
-    public void renameTable(CqlTable extant, String newTableName) {
+    public void renameColumn(CqlColumnBase extant, String newColName) {
-        Map<String, CqlTable> tablesInKs = tableDefs.get(extant.getKeySpace());
+        extant.setName(newColName);
-        CqlTable table = tablesInKs.get(extant.getName());
-        table.setName(newTableName);
-        tablesInKs.put(table.getName(),table);
-    }
-
-    public void renameType(String keyspaceName, String typeName, String newTypeName) {
-        Map<String,CqlType> typesInKeyspace = typeDefs.get(keyspaceName);
-        CqlType cqlType = typesInKeyspace.remove(typeName);
-        cqlType.setName(newTypeName);
-        typesInKeyspace.put(newTypeName,cqlType);
-    }
-
-    public void setTableCompactStorage(boolean isCompactStorage) {
-        table.setCompactStorage(isCompactStorage);
-    }
-
-    public void setKeyspaceDurableWrites(String booleanLiteral) {
-        keyspace.setDurableWrites(Boolean.parseBoolean(booleanLiteral));
-    }
-
-    public void setReplicationData(String repldata) {
-        keyspace.setReplicationData(repldata);
-    }
-
-    public Map<String, Map<String, CqlType>> getTypesByKeyspaceThenName() {
-        return typeDefs;
-    }
-
-    public void addClusteringOrder(String colname, String order) {
-        table.addTableClusteringOrder(colname, order);
     }
 
     public boolean isEmpty() {
-        return this.keyspaceDefs.size()==0 && this.tableDefs.size()==0 && this.typeDefs.size()==0;
+        return this.keyspaceDefs.size() == 0;
+    }
+
+    public List<String> getReferenceErrors() {
+        List<String> errors = new ArrayList<>();
+        for (CqlKeyspaceDef keyspace : this.keyspaceDefs) {
+            keyspace.getReferenceErrors(errors);
+        }
+
+        return errors;
+    }
+
+    public void addKeyspace(CqlKeyspaceDef keyspace) {
+        this.keyspaceDefs.add(keyspace);
+    }
+
+    public void addType(String ksname, CqlType usertype) {
+        CqlKeyspaceDef refks = this.refKeyspace(ksname);
+        usertype.setKeyspace(refks);
+        refks.addType(usertype);
+    }
+
+    public void addTable(String ksname, CqlTable table) {
+        CqlKeyspaceDef cqlKeyspaceDef = refKeyspace(ksname);
+        table.setKeyspace(cqlKeyspaceDef);
+        cqlKeyspaceDef.addTable(table);
     }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModelBuilder.java

@@ -36,6 +36,13 @@ public class CqlModelBuilder extends CqlParserBaseListener {
     private final CqlModel model;
     private long counted;
     private int colindex;
+    private CqlKeyspaceDef keyspace;
+    private CqlType usertype;
+    transient CqlTable table;
+
+
+
+
 
     public CqlModelBuilder(CGErrorListener errorListener) {
         this.errorListener = errorListener;
@@ -62,26 +69,34 @@ public class CqlModelBuilder extends CqlParserBaseListener {
 
     @Override
     public void enterCreateKeyspace(CqlParser.CreateKeyspaceContext ctx) {
-        model.newKeyspace();
+        this.keyspace=new CqlKeyspaceDef();
     }
 
     @Override
     public void exitCreateKeyspace(CqlParser.CreateKeyspaceContext ctx) {
-        model.saveKeyspace(
+        saveKeyspace(
-            ctx.keyspace().getText(),
+            ctx.keyspace().getText()
-            textOf(ctx)
         );
     }
 
+    public void saveKeyspace(String keyspaceName) {
+        this.keyspace.setKeyspaceName(keyspaceName);
+        keyspace.validate();
+        keyspace.setDefined();
+        model.addKeyspace(keyspace);
+        this.keyspace = null;
+    }
+
+
     @Override
     public void exitReplicationList(CqlParser.ReplicationListContext ctx) {
         String repldata = textOf(ctx);
-        model.setReplicationData(repldata);
+        keyspace.setReplicationData(repldata);
     }
 
     @Override
     public void enterCreateTable(CqlParser.CreateTableContext ctx) {
-        model.newTable();
+        this.table = new CqlTable();
     }
 
     @Override
@@ -92,37 +107,40 @@ public class CqlModelBuilder extends CqlParserBaseListener {
     @Override
     public void exitPrimaryKeyDefinition(CqlParser.PrimaryKeyDefinitionContext ctx) {
         if (ctx.singlePrimaryKey() != null) {
-            model.addPartitionKey(ctx.singlePrimaryKey().column().getText());
+            addPartitionKey(ctx.singlePrimaryKey().column().getText());
         } else if (ctx.compositeKey() != null) {
             if (ctx.compositeKey().partitionKeyList() != null) {
                 for (CqlParser.PartitionKeyContext pkctx : ctx.compositeKey().partitionKeyList().partitionKey()) {
-                    model.addPartitionKey(pkctx.column().getText());
+                    addPartitionKey(pkctx.column().getText());
                 }
             }
             if (ctx.compositeKey().clusteringKeyList() != null) {
                 for (CqlParser.ClusteringKeyContext ccol : ctx.compositeKey().clusteringKeyList().clusteringKey()) {
-                    model.addClusteringColumn(ccol.column().getText());
+                    addClusteringColumn(ccol.column().getText());
                 }
             }
         } else if (ctx.compoundKey() != null) {
-            model.addPartitionKey(ctx.compoundKey().partitionKey().getText());
+            addPartitionKey(ctx.compoundKey().partitionKey().getText());
             for (CqlParser.ClusteringKeyContext ccol : ctx.compoundKey().clusteringKeyList().clusteringKey()) {
-                model.addClusteringColumn(ccol.column().getText());
+                addClusteringColumn(ccol.column().getText());
             }
         }
     }
 
-
     @Override
     public void enterCreateType(CqlParser.CreateTypeContext ctx) {
-        model.newType();
+        this.usertype = new CqlType();
     }
 
     @Override
     public void exitCreateType(CqlParser.CreateTypeContext ctx) {
         String keyspace = ctx.keyspace().getText();
         String name = ctx.type_().getText();
-        model.saveType(keyspace, name);
+        usertype.setName(name);
+        usertype.setDefined();
+        model.addType(keyspace, usertype);
+        usertype.validate();
+        usertype=null;
     }
 
 
@@ -132,10 +150,8 @@ public class CqlModelBuilder extends CqlParserBaseListener {
         List<CqlParser.ColumnContext> columns = ctx.column();
         List<CqlParser.DataTypeContext> dataTypes = ctx.dataType();
         for (int idx = 0; idx < columns.size(); idx++) {
-            model.addTypeField(
+            addTypeField(
-                columns.get(idx).getText(),
+                new CqlTypeColumn(columns.get(idx).getText(),dataTypes.get(idx).getText()));
-                dataTypes.get(idx).getText()
-            );
         }
 
 //        dataTypes.get(0).dataType().get(0).dataType().get(0)
@@ -148,29 +164,32 @@ public class CqlModelBuilder extends CqlParserBaseListener {
 
     @Override
     public void exitCreateTable(CqlParser.CreateTableContext ctx) {
-        model.saveTable(
+        table.setName(ctx.table().getText());
+        saveTable(
             ctx.keyspace().getText(),
             ctx.table().getText()
         );
     }
 
+    private void saveTable(String ksname, String tableName) {
+        table.setName(tableName);
+        model.addTable(ksname, table);
+        table=null;
+    }
+
+
     @Override
     public void exitOrderDirection(CqlParser.OrderDirectionContext ctx) {
     }
 
     @Override
     public void exitTableOptionItem(CqlParser.TableOptionItemContext ctx) {
-        if (ctx.kwCompactStorage()!=null) {
+        table.setCompactStorage(ctx.kwCompactStorage()!=null);
-            model.setTableCompactStorage(true);
-        }
-        super.exitTableOptionItem(ctx);
     }
 
     @Override
     public void exitDurableWrites(CqlParser.DurableWritesContext ctx) {
-        model.setKeyspaceDurableWrites(ctx.booleanLiteral().getText());
+        keyspace.setDurableWrites(Boolean.parseBoolean(ctx.booleanLiteral().getText()));
-
-
     }
 
     @Override
@@ -187,9 +206,10 @@ public class CqlModelBuilder extends CqlParserBaseListener {
             .toList();
 
         IntStream.range(0, columns.size())
-            .forEach(i -> model.addClusteringOrder(columns.get(i), orders.get(i)));
+            .forEach(i -> table.addTableClusteringOrder(columns.get(i), orders.get(i)));
     }
 
+
 //    @Override
 //    public void exitColumn(CqlParser.ColumnContext ctx) {
 //        super.exitColumn(ctx);
@@ -214,11 +234,10 @@ public class CqlModelBuilder extends CqlParserBaseListener {
 
     @Override
     public void exitColumnDefinition(CqlParser.ColumnDefinitionContext ctx) {
-        model.saveColumnDefinition(
+        addColumnDefinition(
             ctx.column().getText(),
             textOf(ctx.dataType()),
-            ctx.primaryKeyColumn() != null,
+            ctx.primaryKeyColumn() != null
-            colindex++
         );
     }
 
@@ -236,4 +255,31 @@ public class CqlModelBuilder extends CqlParserBaseListener {
         return model.getErrors();
     }
 
+    private void addColumnDefinition(String colname, String typedef, boolean isPrimaryKey) {
+        if (table != null) {
+            table.addcolumnDef(new CqlTableColumn(colname, typedef));
+            if (isPrimaryKey) {
+                this.table.addPartitionKey(colname);
+            }
+        } else if (usertype != null) {
+            usertype.addColumn(
+                new CqlTypeColumn(colname, typedef)
+            );
+        }
+    }
+
+    public void addPartitionKey(String partitionKey) {
+        table.addPartitionKey(partitionKey);
+    }
+
+    public void addClusteringColumn(String ccolumn) {
+        table.addClusteringColumn(ccolumn);
+    }
+
+    public void addTypeField(CqlTypeColumn coldef) {
+        this.usertype.addColumn(coldef);
+    }
+
+
+
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTable.java

@@ -18,21 +18,23 @@ package io.nosqlbench.cqlgen.model;
 
 import io.nosqlbench.api.config.NBNamedElement;
 import io.nosqlbench.api.labels.Labeled;
-import io.nosqlbench.cqlgen.exporter.CGTableStats;
+import io.nosqlbench.cqlgen.core.CGTableStats;
+import io.nosqlbench.cqlgen.transformers.ComputedTableStats;
 
 import java.util.*;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
 public class CqlTable implements NBNamedElement, Labeled {
+    private CqlKeyspaceDef keyspace;
     String name = "";
-    String keyspace = "";
     CGTableStats tableAttributes = null;
     int[] partitioning = new int[0];
     int[] clustering = new int[0];
     List<String> clusteringOrders = new ArrayList<>();
-    List<CqlColumnDef> coldefs = new ArrayList<>();
+    List<CqlTableColumn> coldefs = new ArrayList<>();
     private boolean compactStorage;
+    private ComputedTableStats computedTableStats;
 
     public CqlTable() {
     }
@@ -45,11 +47,11 @@ public class CqlTable implements NBNamedElement, Labeled {
         return tableAttributes;
     }
 
-    public void setTableAttributes(CGTableStats tableAttributes) {
+    public void setStats(CGTableStats tableAttributes) {
         this.tableAttributes = tableAttributes;
     }
 
-    public void addcolumnDef(CqlColumnDef cqlField) {
+    public void addcolumnDef(CqlTableColumn cqlField) {
         this.coldefs.add(cqlField);
     }
 
@@ -57,10 +59,6 @@ public class CqlTable implements NBNamedElement, Labeled {
         this.name = tableName;
     }
 
-    public void addcolumnDef(String colname, String typedef, int position) {
-        coldefs.add(new CqlColumnDef(this, coldefs.size(), colname, typedef));
-    }
-
     @Override
     public String toString() {
         return "cql table: '" + this.name + "':\n"
@@ -70,7 +68,7 @@ public class CqlTable implements NBNamedElement, Labeled {
             .collect(Collectors.joining("\n"));
     }
 
-    public List<CqlColumnDef> getColumnDefinitions() {
+    public List<CqlTableColumn> getColumnDefs() {
         return this.coldefs;
     }
 
@@ -78,22 +76,14 @@ public class CqlTable implements NBNamedElement, Labeled {
         return this.name;
     }
 
-    public void setKeyspace(String newKsName) {
+    public void setKeyspace(CqlKeyspaceDef keyspace) {
-        for (CqlColumnDef coldef : coldefs) {
+        this.keyspace = keyspace;
-            coldef.setKeyspace(newKsName);
-        }
-        this.keyspace = newKsName;
-
-    }
-
-    public String getKeySpace() {
-        return this.keyspace;
     }
 
     @Override
     public Map<String, String> getLabels() {
         return Map.of(
-            "keyspace", this.keyspace,
+            "keyspace", this.getName(),
             "name", this.name,
             "type", "table"
         );
@@ -140,8 +130,8 @@ public class CqlTable implements NBNamedElement, Labeled {
         return Arrays.stream(clustering).mapToObj(i -> this.coldefs.get(i).getName()).toList();
     }
 
-    public CqlColumnDef getColumnDefForName(String colname) {
+    public CqlTableColumn getColumnDefForName(String colname) {
-        Optional<CqlColumnDef> def = coldefs
+        Optional<CqlTableColumn> def = coldefs
             .stream()
             .filter(c -> c.getName().equalsIgnoreCase(colname))
             .findFirst();
@@ -153,15 +143,15 @@ public class CqlTable implements NBNamedElement, Labeled {
     }
 
     public void renameColumns(Function<String, String> renamer) {
-        for (CqlColumnDef coldef : coldefs) {
+        for (CqlTableColumn coldef : coldefs) {
             coldef.setName(renamer.apply(coldef.getName()));
         }
     }
 
-    public List<CqlColumnDef> getNonKeyColumnDefinitions() {
+    public List<CqlTableColumn> getNonKeyColumnDefinitions() {
         int last = partitioning[partitioning.length - 1];
         last = (clustering.length > 0 ? clustering[clustering.length - 1] : last);
-        List<CqlColumnDef> nonkeys = new ArrayList<>();
+        List<CqlTableColumn> nonkeys = new ArrayList<>();
         for (int nonkey = last; nonkey < coldefs.size(); nonkey++) {
             nonkeys.add(coldefs.get(nonkey));
         }
@@ -173,7 +163,7 @@ public class CqlTable implements NBNamedElement, Labeled {
     }
 
     public String getFullName() {
-        return (this.keyspace != null ? this.keyspace + "." : "") + this.name;
+        return (this.keyspace != null ? this.keyspace.getName() + "." : "") + this.name;
     }
 
     public boolean isPartitionKey(int position) {
@@ -191,4 +181,23 @@ public class CqlTable implements NBNamedElement, Labeled {
     public boolean isLastClusteringColumn(int position) {
         return clustering.length > 0 && position == clustering[clustering.length - 1];
     }
+
+    public ComputedTableStats getComputedStats() {
+        return this.computedTableStats;
+    }
+
+    public void setComputedStats(ComputedTableStats stats) {
+        this.computedTableStats = stats;
+    }
+
+    public boolean hasStats() {
+        return this.computedTableStats!=null;
+    }
+
+    public CqlKeyspaceDef getKeyspace() {
+        return this.keyspace;
+    }
+
+    public void getReferenceErrors(List<String> errors) {
+    }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTableColumn.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.model;
+
+public class CqlTableColumn extends CqlColumnBase {
+
+    private CqlTable table;
+    public CqlTableColumn(String colname, String typedef) {
+        super(colname, typedef);
+    }
+
+    @Override
+    protected String getParentFullName() {
+        return table.getFullName();
+    }
+
+    public CqlTable getTable() {
+        return table;
+    }
+
+    public void setTable(CqlTable table) {
+        this.table = table;
+    }
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlType.java

@@ -19,27 +19,26 @@ package io.nosqlbench.cqlgen.model;
 import io.nosqlbench.api.config.NBNamedElement;
 import io.nosqlbench.api.labels.Labeled;
 
-import java.util.LinkedHashMap;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
-import java.util.function.Function;
+import java.util.Objects;
 
 public class CqlType implements NBNamedElement, Labeled {
-    private String keyspace;
-    private String name;
-    private String refddl;
-    private Map<String,String> fields = new LinkedHashMap<>();
 
-    public void setKeyspace(String newksname) {
+    private String name;
-        this.keyspace = newksname;
+    private CqlKeyspaceDef keyspace;
-        if (refddl!=null) {
+    private List<CqlTypeColumn> columnDefs = new ArrayList<>();
-            this.refddl = this.refddl.replaceAll(this.keyspace,newksname);
+    private volatile boolean defined;
-        }
+
+    public void setKeyspace(CqlKeyspaceDef keyspace) {
+        this.keyspace = keyspace;
     }
     public void setName(String name) {
         this.name = name;
     }
 
-    public String getKeyspace() {
+    public CqlKeyspaceDef getKeyspace() {
         return keyspace;
     }
 
@@ -47,26 +46,47 @@ public class CqlType implements NBNamedElement, Labeled {
         return this.name;
     }
 
-    public void addField(String name, String typedef) {
+    public void addColumn(CqlTypeColumn def) {
-        this.fields.put(name, typedef);
+        this.columnDefs.add(this.columnDefs.size(),def);
     }
 
-    public Map<String, String> getFields() {
+    public List<CqlTypeColumn> columns() {
-        return fields;
+        return columnDefs;
     }
 
     @Override
     public Map<String, String> getLabels() {
         return Map.of(
-            "keyspace", this.keyspace,
+            "keyspace", keyspace.getName(),
-            "type","udt",
+            "type","type",
             "name",name
         );
     }
 
-    public void renameColumns(Function<String, String> renamer) {
+    public void setColumnDefs(List<CqlTypeColumn> columnDefs) {
-        Map<String,String> newColumns = new LinkedHashMap<>();
+        this.columnDefs = columnDefs;
-        fields.forEach((k,v)->newColumns.put(renamer.apply(k),v));
+    }
-        this.fields = newColumns;
+
+    public List<CqlTypeColumn> getColumnDefs() {
+        return columnDefs;
+    }
+
+    public String getFullName() {
+        return keyspace.getName()+"."+getName();
+    }
+
+    public void getReferenceErrors(List<String> errors) {
+        if (!defined) {
+            errors.add("type " + this.getName() + " was referenced but not defined.");
+        }
+    }
+
+    public void validate() {
+        Objects.requireNonNull(this.name);
+        Objects.requireNonNull(this.keyspace);
+    }
+
+    public void setDefined() {
+        this.defined=true;
     }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTypeColumn.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.model;
+
+public class CqlTypeColumn extends CqlColumnBase {
+    CqlType type;
+
+    public CqlTypeColumn(String colname, String typedef) {
+        super(colname, typedef);
+    }
+
+    @Override
+    protected String getParentFullName() {
+        return type.getFullName();
+    }
+
+    public CqlType getType() {
+        return type;
+    }
+
+    public void setType(CqlType type) {
+        this.type = type;
+    }
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/FieldPosition.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.model;
+
+public enum FieldPosition {
+    /**
+     * This field is used in the partitioning key(s) for a table
+     */
+    Partitioning,
+    /**
+     * This field is used in the clustering value(s) for a table
+     */
+    Clustering,
+    /**
+     * This field is a non-key field for a table
+     */
+    NonKey,
+
+    /**
+     * This field is used in a type definition
+     */
+    TypeDef
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGCachingNameRemapper.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.transformers;
+
+import io.nosqlbench.api.labels.Labeled;
+import io.nosqlbench.virtdata.library.basics.shared.from_long.to_string.Combinations;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.LongFunction;
+
+public class CGCachingNameRemapper {
+    private LongFunction<String> namefunc;
+    private final Map<String,String> remapped = new HashMap<>();
+    private final Map<String,String> prefixmap = new HashMap<>();
+    private final Map<String,Long> indexmap = new HashMap<>();
+
+    public CGCachingNameRemapper() {
+        this.namefunc = new Combinations("a-z;a-z;a-z;a-z;a-z;a-z;");
+    }
+    public CGCachingNameRemapper(LongFunction<String> function) {
+        this.namefunc = function;
+    }
+
+//    public synchronized String nameForType(String type, String originalName, String prefix) {
+//        String canonical = type+"_"+originalName;
+//        return getOrCreateName(canonical, prefix);
+//    }
+
+//    public synchronized String nameFor(NBNamedElement element) {
+//        String prefix = prefixmap.get(element.getClass().getSimpleName());
+//        String canonical = element.getClass().getSimpleName()+"-"+element.getName();
+//        return getOrCreateName(canonical, prefix);
+//    }
+
+
+    private long indexforType(String type) {
+        long newvalue = indexmap.computeIfAbsent(type, t -> 0L)+1;
+        indexmap.put(type,newvalue);
+        return newvalue;
+    }
+
+    public synchronized String nameFor(Map<String,String> labels) {
+        String type = labels.get("type");
+        Objects.requireNonNull(type);
+        String name = labels.get("name");
+        Objects.requireNonNull(name);
+        String canonical = type+"-"+name;
+        String prefix = prefixmap.getOrDefault(type,"");
+        if (!remapped.containsKey(canonical)) {
+            long indexForType=indexforType(type);
+            String newname = (prefix!=null?prefix:"")+namefunc.apply(indexForType);
+            remapped.put(canonical,newname);
+        }
+        return remapped.get(canonical);
+    }
+
+    public synchronized String nameFor(Labeled element) {
+        Map<String, String> labels = element.getLabels();
+        return nameFor(labels);
+    }
+
+    //    public Function<String, String> mapperForType(Labeled cqlTable, String prefix) {
+//        return in -> this.nameForType(cqlTable.getClass().getSimpleName(),in, prefix);
+//    }
+//
+    public void setNamingFunction(LongFunction<String> namerFunc) {
+        this.namefunc = namerFunc;
+    }
+    public void setTypePrefixes(Map<String,String> prefixesByLabeledType) {
+        this.prefixmap.clear();
+        this.prefixmap.putAll(prefixesByLabeledType);
+    }
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGGenStatsInjector.java

@@ -14,9 +14,11 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.transformers;
+package io.nosqlbench.cqlgen.transformers;
 
-import io.nosqlbench.cqlgen.exporter.CGSchemaStats;
+import io.nosqlbench.cqlgen.api.CGModelTransformer;
+import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
+import io.nosqlbench.cqlgen.core.CGSchemaStats;
 import io.nosqlbench.cqlgen.model.CqlModel;
 
 import java.io.IOException;
@@ -25,6 +27,7 @@ import java.util.Map;
 
 public class CGGenStatsInjector implements CGModelTransformer, CGTransformerConfigurable {
     private CGSchemaStats schemaStats = null;
+    private String name;
 
     public CGGenStatsInjector() {
     }
@@ -37,6 +40,11 @@ public class CGGenStatsInjector implements CGModelTransformer, CGTransformerConf
         return model;
     }
 
+    @Override
+    public void setName(String name) {
+        this.name = name;
+    }
+
     @Override
     public void accept(Object configObject) {
         if (configObject instanceof Map config) {
@@ -59,4 +67,9 @@ public class CGGenStatsInjector implements CGModelTransformer, CGTransformerConf
             throw new RuntimeException("stats injector requires a map for it's config value");
         }
     }
+
+    @Override
+    public String getName() {
+        return this.name;
+    }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeySpaceDDLRemover.java

@@ -14,22 +14,33 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.transformers;
+package io.nosqlbench.cqlgen.transformers;
 
-import io.nosqlbench.cqlgen.model.CqlKeyspace;
+import io.nosqlbench.cqlgen.api.CGModelTransformer;
+import io.nosqlbench.cqlgen.model.CqlKeyspaceDef;
 import io.nosqlbench.cqlgen.model.CqlModel;
 
 import java.util.List;
 
 public class CGKeySpaceDDLRemover implements CGModelTransformer {
     private List<String> includes;
+    private String name;
 
     @Override
     public CqlModel apply(CqlModel model) {
-        for (CqlKeyspace keyspace : model.getKeyspaceDefs()) {
+        for (CqlKeyspaceDef keyspace : model.getKeyspaceDefs()) {
             model.removeKeyspaceDef(keyspace.getName());
         }
         return model;
     }
 
+    @Override
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    @Override
+    public String getName() {
+        return this.name;
+    }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeyspaceFilter.java

@@ -14,8 +14,11 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.transformers;
+package io.nosqlbench.cqlgen.transformers;
 
+import io.nosqlbench.cqlgen.api.CGModelTransformer;
+import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
+import io.nosqlbench.cqlgen.model.CqlKeyspaceDef;
 import io.nosqlbench.cqlgen.model.CqlModel;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -23,7 +26,6 @@ import org.apache.logging.log4j.Logger;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.function.Function;
 import java.util.regex.Pattern;
 
@@ -31,6 +33,12 @@ public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfig
 
     private final static Logger logger = LogManager.getLogger(CGKeyspaceFilter.class);
     private List<TriStateFilter> patterns;
+    private String name;
+
+    @Override
+    public String getName() {
+        return this.name;
+    }
 
     private enum InclExcl {
         include,
@@ -45,8 +53,8 @@ public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfig
 
     @Override
     public CqlModel apply(CqlModel model) {
-        Set<String> keyspacenames = model.getAllKnownKeyspaceNames();
+        List<String> ksnames = model.getKeyspaceDefs().stream().map(CqlKeyspaceDef::getName).toList();
-        for (String keyspace : keyspacenames) {
+        for (String keyspace : ksnames) {
             Action action = Action.inderminate;
             for (TriStateFilter pattern : patterns) {
                 action = pattern.apply(keyspace);
@@ -57,8 +65,6 @@ public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfig
                     case remove:
                         logger.info("removing all definitions in " + keyspace + " with exclusion pattern " + pattern);
                         model.removeKeyspaceDef(keyspace);
-                        model.removeTablesForKeyspace(keyspace);
-                        model.removeTypesForKeyspace(keyspace);
                     case inderminate:
                 }
             }
@@ -71,6 +77,11 @@ public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfig
         return model;
     }
 
+    @Override
+    public void setName(String name) {
+        this.name = name;
+    }
+
     private static class TriStateFilter implements Function<String, Action> {
         private final InclExcl filterType;
         private final Pattern pattern;

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGModelTransformers.java

@@ -14,8 +14,13 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.transformers;
+package io.nosqlbench.cqlgen.transformers;
 
+import io.nosqlbench.api.config.standard.NBConfigurable;
+import io.nosqlbench.api.config.standard.NBConfiguration;
+import io.nosqlbench.cqlgen.api.CGModelTransformer;
+import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
+import io.nosqlbench.cqlgen.model.CqlModel;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -24,10 +29,15 @@ import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Consumer;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
-public class CGModelTransformers implements Consumer<List<Map<String, ?>>>, Supplier<List<CGModelTransformer>> {
+public class CGModelTransformers implements
+    Consumer<List<Map<String, ?>>>,
+    Supplier<List<CGModelTransformer>>,
+    Function<CqlModel,CqlModel> {
     private final static Logger logger = LogManager.getLogger(CGModelTransformers.class);
     private final List<CGModelTransformer> transformers = new ArrayList<>();
 
@@ -43,9 +53,11 @@ public class CGModelTransformers implements Consumer<List<Map<String, ?>>>, Supp
             // Instantiate Transformer
 
             String classname = cfgmap.get("class").toString();
+            String name = Optional.ofNullable(cfgmap.get("name")).orElseThrow().toString();
+
             if (!classname.contains(".")) {
                 String newname = CGNameObfuscator.class.getPackageName() + "." + classname;
-                logger.info("qualified transformer '" + classname + "' as '" + newname + "'");
+                logger.debug("qualified transformer '" + classname + "' as '" + newname + "'");
                 classname = newname;
             }
             Class<?> txclass = null;
@@ -56,6 +68,7 @@ public class CGModelTransformers implements Consumer<List<Map<String, ?>>>, Supp
                 Object instance = ctor.newInstance();
                 if (instance instanceof CGModelTransformer t) {
                     transformer = t;
+                    t.setName(name);
                 } else {
                     throw new RuntimeException("Object " + instance.getClass().getName() + " is not a " + CGModelTransformer.class.getName());
                 }
@@ -63,7 +76,16 @@ public class CGModelTransformers implements Consumer<List<Map<String, ?>>>, Supp
                 throw new RuntimeException(e);
             }
 
+            if (transformer instanceof NBConfigurable nbc) {
+                Object cfg = cfgmap.get("config");
+                if (cfg instanceof Map tcfgmap) {
+                    NBConfiguration configuration = nbc.getConfigModel().apply((Map<String, ?>) cfg);
+                    nbc.applyConfig(configuration);
+                } else {
+                    throw new RuntimeException("config for " + nbc.getClass().getSimpleName() + " must be map.");
+                }
 
+            }
             // Configure Transformer IFF ...
             if (transformer instanceof CGTransformerConfigurable configurable) {
                 Object cfgvalues = cfgmap.get("config");
@@ -84,4 +106,12 @@ public class CGModelTransformers implements Consumer<List<Map<String, ?>>>, Supp
     public List<CGModelTransformer> get() {
         return this.transformers;
     }
+
+    @Override
+    public CqlModel apply(CqlModel cqlModel) {
+        for (CGModelTransformer transformer : transformers) {
+            cqlModel=transformer.apply(cqlModel);
+        }
+        return cqlModel;
+    }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGNameObfuscator.java

@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Create random but stable names for all elements.
+ * Use a deterministic method for creating names.
+ * once an element is named, use the same name throughout
+ * prefix each element type with a code for the type
+ */
+package io.nosqlbench.cqlgen.transformers;
+
+import io.nosqlbench.api.config.standard.*;
+import io.nosqlbench.cqlgen.api.CGModelTransformer;
+import io.nosqlbench.cqlgen.model.*;
+import io.nosqlbench.cqlgen.transformers.namecache.*;
+import io.nosqlbench.virtdata.core.bindings.DataMapper;
+import io.nosqlbench.virtdata.core.bindings.VirtData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.nio.file.Path;
+import java.util.Map;
+
+public class CGNameObfuscator implements CGModelTransformer, NBConfigurable {
+    private final static Logger logger = LogManager.getLogger(CGNameObfuscator.class);
+
+    NameCache cache = new NameCache();
+    private final CGCachingNameRemapper remapper = new CGCachingNameRemapper();
+    private String name;
+    private String mapfile;
+
+    @Override
+    public CqlModel apply(CqlModel model) {
+        remapper.setTypePrefixes(Map.of("keyspace", "ks_", "type", "typ_", "table", "tb_", "column","col_"));
+
+        if (mapfile != null) {
+            cache = NameCache.loadOrCreate(Path.of(mapfile));
+        }
+
+        for (CqlKeyspaceDef keyspaceDef : model.getKeyspaceDefs()) {
+            NamedKeyspace namedKeyspace = cache.keyspace(keyspaceDef.getName());
+            String newKeyspaceName = namedKeyspace.computeAlias(keyspaceDef, remapper::nameFor);
+            keyspaceDef.setKeyspaceName(newKeyspaceName);
+
+            for (CqlType typeDef : keyspaceDef.getTypeDefs()) {
+                NamedType namedType = namedKeyspace.type(typeDef.getName());
+                String typeDefName = namedType.computeAlias(typeDef,remapper::nameFor);
+                namedType.setName(typeDefName);
+
+                for (CqlTypeColumn columnDef : typeDef.getColumnDefs()) {
+                    NamedColumn namedTypeColumn = namedType.column(columnDef.getName());
+                    String newColumnName = namedTypeColumn.computeAlias(columnDef,remapper::nameFor);
+                    columnDef.setName(newColumnName);
+                }
+            }
+
+            for (CqlTable table : keyspaceDef.getTableDefs()) {
+
+                NamedTable namedTable = namedKeyspace.table(table.getName());
+                String newTableName = namedTable.computeAlias(table,remapper::nameFor);
+                table.setName(newTableName);
+
+                for (CqlColumnBase columnDef : table.getColumnDefs()) {
+                    NamedColumn namedTableColumn = namedTable.column(columnDef.getName());
+                    String newColumnName = namedTableColumn.computeAlias(columnDef,remapper::nameFor);
+                    columnDef.setName(newColumnName);
+                }
+
+            }
+
+        }
+
+//        for (String keyspaceName : model.getAllKnownKeyspaceNames()) {
+//            Map<String, String> labels = Map.of("type", "keyspace", "name", keyspaceName);
+//            NamedKeyspace cachedKeyspace = cache.keyspace(keyspaceName);
+//            cachedKeyspace.computeAlias(labels, remapper::nameFor);
+////            model.renamekeyspace(keyspaceName, alias);
+//        }
+//
+//        for (CqlTable cqlTable : model.getTableDefs()) {
+//            String tablename = cqlTable.getName();
+//            NamedTable cachedTable = cache.keyspace(cqlTable.getKeyspaceName()).table(tablename);
+//            String alias = cachedTable.computeAlias(cqlTable, remapper::nameFor);
+//            model.renameTable(cqlTable, alias);
+//
+//            for (CqlColumnBase coldef : cqlTable.getColumnDefs()) {
+//                NamedColumn cachedColumn = cache.keyspace(cqlTable.getKeyspaceName()).table(tablename).column(coldef.getName());
+//                cachedColumn.computeAlias(coldef, remapper::nameFor);
+////                model.renameColumn(coldef, colalias);
+//            }
+//        }
+//
+//        for (CqlType type : model.getTypeDefs()) {
+//            String typeName = type.getName();
+//            NamedType cachedType = cache.keyspace(type.getKeyspace()).type(typeName);
+//            cachedType.computeAlias(type, remapper::nameFor);
+////            model.renameType(type.getKeyspace(), typeName, alias);
+//
+////            Function<String, String> colmapper = remapper.mapperForType(type, "typ");
+//            Map<String, String> newdefs = new LinkedHashMap<>();
+//
+//            Set<String> keys = type.getFields().keySet();
+//            for (String key : keys) {
+//                NamedColumn cachedColdef = cache.keyspace(type.getKeyspace()).type(typeName).column(key);
+//                cachedColdef.computeAlias(Map.of("type", "column", "name", key), remapper::nameFor);
+////                String def = type.getFields().get(key);
+////                newdefs.put(colalias, def);
+////                type.setFields(newdefs);
+//            }
+//        }
+
+        if (mapfile!=null) {
+            cache.setPath(mapfile);
+            cache.Save();
+        }
+        return model;
+
+    }
+
+    @Override
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    @Override
+    public void applyConfig(NBConfiguration cfg) {
+        String namer = cfg.get("namer");
+        DataMapper<String> namerFunc = VirtData.getMapper(namer);
+        this.remapper.setNamingFunction(namerFunc);
+        this.mapfile = cfg.getOptional("mapfile").orElse(null);
+    }
+
+    @Override
+    public NBConfigModel getConfigModel() {
+        return ConfigModel.of(CGNameObfuscator.class)
+            .add(Param.defaultTo("namer", "Combinations('0-9;0-9;0-9;0-9;0-9')"))
+            .add(Param.optional("mapfile", String.class))
+            .asReadOnly();
+    }
+
+    @Override
+    public String getName() {
+        return this.name;
+    }
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioCalculator.java

@@ -14,43 +14,47 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.transformers;
+package io.nosqlbench.cqlgen.transformers;
 
-import io.nosqlbench.cqlgen.exporter.CGTableStats;
+import io.nosqlbench.cqlgen.api.CGModelTransformer;
+import io.nosqlbench.cqlgen.core.CGTableStats;
 import io.nosqlbench.cqlgen.model.CqlModel;
 import io.nosqlbench.cqlgen.model.CqlTable;
 
 public class CGRatioCalculator implements CGModelTransformer {
 
+    private String name;
+
     @Override
     public CqlModel apply(CqlModel model) {
         if (!model.hasStats()) {
             // TODO: True this up
             return model;
         }
+
         double totalReads = 0.0d;
         double totalWrites = 0.0d;
         double totalSpace = 0.0d;
-        double totalOps=0.0d;
+        double totalOps = 0.0d;
 
         for (CqlTable table : model.getTableDefs()) {
             CGTableStats tableAttributes = table.getTableAttributes();
-            if (tableAttributes==null) {
+            if (tableAttributes == null) {
                 continue;
             }
             String local_read_count = tableAttributes.getAttribute("Local read count");
             double reads = Double.parseDouble(local_read_count);
-            totalReads+=reads;
+            totalReads += reads;
-            totalOps+=reads;
+            totalOps += reads;
 
             String local_write_count = table.getTableAttributes().getAttribute("Local write count");
             double writes = Double.parseDouble(local_write_count);
             totalWrites += writes;
-            totalOps+=writes;
+            totalOps += writes;
 
             String space_used_total = table.getTableAttributes().getAttribute("Space used (total)");
             double space = Double.parseDouble(space_used_total);
-            totalSpace+=space;
+            totalSpace += space;
         }
 
         for (CqlTable table : model.getTableDefs()) {
@@ -59,15 +63,30 @@ public class CGRatioCalculator implements CGModelTransformer {
 
             double totalTableReads = reads / totalOps;
             double totalTableWrites = writes / totalOps;
-            table.getTableAttributes().setAttribute("weighted_reads", String.valueOf(totalTableReads));
-            table.getTableAttributes().setAttribute("weighted_writes", String.valueOf(totalTableWrites));
-            table.getTableAttributes().setAttribute("weighted_ops", String.valueOf(totalTableReads+totalTableWrites));
             double tableSpaceUsed = Double.parseDouble(table.getTableAttributes().getAttribute("Space used (total)"));
-            table.getTableAttributes().setAttribute("weighted_space", String.valueOf(tableSpaceUsed / totalSpace));
+            double weighted_space = tableSpaceUsed / totalSpace;
+            double op_share_of_total_ops = totalTableReads + totalTableWrites;
+            ComputedTableStats computedTableStats = new ComputedTableStats()
+                .setReadShareOfTotalOps(reads / totalOps)
+                .setReadShareOfTotalReads(reads / totalReads)
+                .setWriteShareOfTotalOps(writes / totalOps)
+                .setWriteShareOfTotalWrites(writes / totalWrites)
+                .setOpShareOfTotalOps(op_share_of_total_ops)
+                .setSpaceUsedOfTotalSpace(weighted_space);
+            table.setComputedStats(computedTableStats);
         }
 
         return model;
     }
 
+    @Override
+    public void setName(String name) {
+        this.name = name;
+    }
 
+
+    @Override
+    public String getName() {
+        return this.name;
+    }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioSuffixer.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.transformers;
+
+import io.nosqlbench.api.config.standard.*;
+import io.nosqlbench.cqlgen.api.CGModelTransformer;
+import io.nosqlbench.cqlgen.core.CGSchemaStats;
+import io.nosqlbench.cqlgen.model.CqlModel;
+import io.nosqlbench.cqlgen.model.CqlTable;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class CGRatioSuffixer implements CGModelTransformer, NBConfigurable {
+
+    private double resolution;
+    private String format;
+    private String name;
+
+    @Override
+    public CqlModel apply(CqlModel model) {
+        CGSchemaStats schemastats = model.getStats();
+        if (schemastats == null) {
+            return model;
+        }
+
+        for (CqlTable tableDef : model.getTableDefs()) {
+            double opshare = tableDef.getComputedStats().getOpShareOfTotalOps();
+            double multiplier = Math.pow(10.0, resolution+1);
+            long value = (long) (opshare*multiplier);
+            String newname = String.format(this.format, tableDef.getName(), value);
+            tableDef.setName(newname);
+        }
+
+        return model;
+    }
+
+    @Override
+    public void setName(String name) {
+        this.name = name;
+    }
+
+
+    @Override
+    public void applyConfig(NBConfiguration cfg) {
+        this.format = cfg.get("format", String.class);
+        if (!format.contains("%1$s")) {
+            throw new RuntimeException("format config param for the CGRatioSuffixer must contain '%1$s', but it is '" + format + "'");
+        }
+        Pattern pattern = Pattern.compile(".*?%2\\$(?<resolution>\\d+)d.*");
+        Matcher matcher = pattern.matcher(format);
+        if (!matcher.matches()) {
+            throw new RuntimeException("Could not find the required decimal format specifier for the format config parameter of " + CGRatioSuffixer.class);
+        }
+        this.resolution = Double.parseDouble(matcher.group("resolution"))-1;
+        this.resolution=Math.max(resolution,2);
+
+    }
+
+    @Override
+    public NBConfigModel getConfigModel() {
+        return ConfigModel.of(CGRatioSuffixer.class)
+            .add(Param.defaultTo("format","%1$s_%2$5d").setDescription(
+                "The format specifier as in Java String.format, with a required string format for the first arg, and a required decimal format for the second."
+            ))
+            .asReadOnly();
+    }
+
+    @Override
+    public String getName() {
+        return this.name;
+    }
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRegexReplacer.java

@@ -14,9 +14,10 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.transformers;
+package io.nosqlbench.cqlgen.transformers;
 
-import io.nosqlbench.cqlgen.exporter.CGTextTransformer;
+import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
+import io.nosqlbench.cqlgen.api.CGTextTransformer;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGReplicationSettingInjector.java

@@ -14,24 +14,32 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.transformers;
+package io.nosqlbench.cqlgen.transformers;
 
-import io.nosqlbench.cqlgen.model.CqlKeyspace;
+import io.nosqlbench.cqlgen.api.CGModelTransformer;
+import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
+import io.nosqlbench.cqlgen.model.CqlKeyspaceDef;
 import io.nosqlbench.cqlgen.model.CqlModel;
 
 import java.util.Map;
 
 public class CGReplicationSettingInjector implements CGModelTransformer, CGTransformerConfigurable {
     private String replicationFields;
+    private String name;
 
     @Override
     public CqlModel apply(CqlModel model) {
-        for (CqlKeyspace keyspace : model.getKeyspaceDefs()) {
+        for (CqlKeyspaceDef keyspace : model.getKeyspaceDefs()) {
             keyspace.setReplicationData(this.replicationFields);
         }
         return model;
     }
 
+    @Override
+    public void setName(String name) {
+        this.name = name;
+    }
+
     @Override
     public void accept(Object cfgObject) {
         if (cfgObject instanceof Map stringMap) {
@@ -42,4 +50,9 @@ public class CGReplicationSettingInjector implements CGModelTransformer, CGTrans
             throw new RuntimeException("replication settings injector requires a map for its config value.");
         }
     }
+
+    @Override
+    public String getName() {
+        return this.name;
+    }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGUdtReplacer.java

@@ -14,9 +14,10 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.transformers;
+package io.nosqlbench.cqlgen.transformers;
 
-import io.nosqlbench.cqlgen.model.CqlColumnDef;
+import io.nosqlbench.cqlgen.api.CGModelTransformer;
+import io.nosqlbench.cqlgen.model.CqlColumnBase;
 import io.nosqlbench.cqlgen.model.CqlModel;
 import io.nosqlbench.cqlgen.model.CqlTable;
 
@@ -24,11 +25,13 @@ import java.util.List;
 
 public class CGUdtReplacer implements CGModelTransformer {
 
+    private String name;
+
     @Override
     public CqlModel apply(CqlModel model) {
-        List<String> toReplace = model.getTypeDefs().stream().map(t -> t.getKeyspace() + "." + t.getName()).toList();
+        List<String> toReplace = model.getTypeDefs().stream().map(t -> t.getKeyspace().getName() + "." + t.getName()).toList();
         for (CqlTable table : model.getTableDefs()) {
-            for (CqlColumnDef coldef : table.getColumnDefinitions()) {
+            for (CqlColumnBase coldef : table.getColumnDefs()) {
                 String typedef = coldef.getTrimmedTypedef();
                 for (String searchFor : toReplace) {
                     if (typedef.contains(searchFor)) {
@@ -41,5 +44,14 @@ public class CGUdtReplacer implements CGModelTransformer {
         return model;
     }
 
+    @Override
+    public void setName(String name) {
+        this.name = name;
+    }
 
+
+    @Override
+    public String getName() {
+        return this.name;
+    }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/ComputedTableStats.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.transformers;
+
+public class ComputedTableStats {
+    private double readShareOfTotalOps;
+    private double readShareOfTotalReads;
+    private double writeShareOfTotalOps;
+    private double writeShareOfTotalWrites;
+    private double opShareOfTotalOps;
+    private double spaceUsedOfTotalSpace;
+
+    public ComputedTableStats setReadShareOfTotalOps(double v) {
+        this.readShareOfTotalOps = v;
+        return this;
+    }
+
+    public ComputedTableStats setReadShareOfTotalReads(double v) {
+        this.readShareOfTotalReads =v;
+        return this;
+    }
+
+    public ComputedTableStats setWriteShareOfTotalOps(double v) {
+        this.writeShareOfTotalOps = v;
+        return this;
+    }
+
+    public ComputedTableStats setWriteShareOfTotalWrites(double v) {
+        this.writeShareOfTotalWrites = v;
+        return this;
+    }
+
+    public ComputedTableStats setOpShareOfTotalOps(double op_share_total) {
+        this.opShareOfTotalOps = op_share_total;
+        return this;
+    }
+
+    public ComputedTableStats setSpaceUsedOfTotalSpace(double weightedSpace) {
+        this.spaceUsedOfTotalSpace = weightedSpace;
+        return this;
+    }
+
+    public double getOpShareOfTotalOps() {
+        return opShareOfTotalOps;
+    }
+
+    public double getWeightedReadsOfTotal() {
+        return this.readShareOfTotalOps;
+    }
+
+    public double getWeightedWritesOfTotal() {
+        return this.writeShareOfTotalWrites;
+    }
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CqlSchemaStatsParser.java

@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.transformers;
+package io.nosqlbench.cqlgen.transformers;
 
-import io.nosqlbench.cqlgen.exporter.CGKeyspaceStats;
+import io.nosqlbench.cqlgen.core.CGKeyspaceStats;
-import io.nosqlbench.cqlgen.exporter.CGSchemaStats;
+import io.nosqlbench.cqlgen.core.CGSchemaStats;
-import io.nosqlbench.cqlgen.exporter.CGTableStats;
+import io.nosqlbench.cqlgen.core.CGTableStats;
 import org.apache.commons.math4.util.Pair;
 
 import java.io.BufferedReader;

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/UnusedTableRemover.java

@@ -14,8 +14,10 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.cqlgen.exporter.transformers;
+package io.nosqlbench.cqlgen.transformers;
 
+import io.nosqlbench.cqlgen.api.CGModelTransformer;
+import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
 import io.nosqlbench.cqlgen.model.CqlModel;
 import io.nosqlbench.cqlgen.model.CqlTable;
 import org.apache.logging.log4j.LogManager;
@@ -27,6 +29,7 @@ import java.util.Map;
 public class UnusedTableRemover implements CGModelTransformer, CGTransformerConfigurable {
     private final static Logger logger = LogManager.getLogger(UnusedTableRemover.class);
     private double minimumThreshold = 0.0001;
+    private String name;
 
     @Override
     public CqlModel apply(CqlModel model) {
@@ -42,15 +45,20 @@ public class UnusedTableRemover implements CGModelTransformer, CGTransformerConf
             double weightedOps = Double.parseDouble(weightedOpsSpec);
             if (weightedOps < minimumThreshold) {
                 logger.info(String.format(
-                    "removing table " + table.getKeySpace() + "." + table.getName() + " with minimum weighted_ops of %1.5f under %1.5f",
+                    "removing table " + table.getKeyspace().getName() + "." + table.getName() + " with minimum weighted_ops of %1.5f under %1.5f",
                     weightedOps, minimumThreshold)
                 );
-                model.getTableDefsByKeyspaceThenTable().get(table.getKeySpace()).remove(table.getName());
+                table.getKeyspace().removeTable(table);
             }
         }
         return model;
     }
 
+    @Override
+    public void setName(String name) {
+        this.name = name;
+    }
+
     @Override
     public void accept(Object cfgObj) {
         if (cfgObj instanceof Map stringMap) {
@@ -62,4 +70,9 @@ public class UnusedTableRemover implements CGModelTransformer, CGTransformerConf
             throw new RuntimeException("unused table remover requires a Map for its config value.");
         }
     }
+
+    @Override
+    public String getName() {
+        return this.name;
+    }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NameCache.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.transformers.namecache;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import io.nosqlbench.cqlgen.model.CqlKeyspaceDef;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.function.Function;
+
+public class NameCache {
+    private final static Logger logger = LogManager.getLogger(NameCache.class);
+
+    private String path;
+    private final Map<String, NamedKeyspace> keyspaces = new LinkedHashMap();
+
+    public NamedKeyspace keyspace(String ksname) {
+        return keyspaces.computeIfAbsent(ksname, v -> new NamedKeyspace(ksname));
+    }
+
+    public NamedKeyspace computeAlias(CqlKeyspaceDef labeledKs, Function<CqlKeyspaceDef, String> ksfunc) {
+        return keyspaces.computeIfAbsent(
+            labeledKs.getName(),
+            ksname -> new NamedKeyspace(ksname)
+                .alias(ksfunc.apply(labeledKs)
+                ));
+    }
+
+    public static NameCache loadOrCreate(Path path) {
+        Gson gson = new GsonBuilder().setPrettyPrinting().create();
+        if (Files.exists(path)) {
+            BufferedReader reader = null;
+            try {
+                reader = Files.newBufferedReader(path);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+            NameCache nameCache = gson.fromJson(reader, NameCache.class);
+            nameCache.setPath(path.toString());
+            return nameCache;
+        } else {
+            return new NameCache().setPath(path.toString());
+        }
+    }
+
+    public NameCache setPath(String path) {
+        if (this.path!=null) {
+            if (this.path.equals(path)) {
+                logger.debug("mapfile unchanged '" + path + "'");
+            } else {
+                logger.info("mapfile changed from '" + this.path + "' to '" + path + "'");
+                this.path = path;
+            }
+        }
+        return this;
+    }
+
+    public NameCache Save() {
+        Gson gson = new GsonBuilder().setPrettyPrinting().create();
+        String json = gson.toJson(this);
+        Path saveto = Path.of(this.path);
+        try {
+            Files.writeString(saveto, json);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return this;
+    }
+
+    public Collection<NamedKeyspace> keyspaces() {
+        return this.keyspaces.values();
+    }
+
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedColumn.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.transformers.namecache;
+
+import io.nosqlbench.api.labels.Labeled;
+
+import java.util.Map;
+import java.util.function.Function;
+
+public class NamedColumn{
+    private final String name;
+    private String alias;
+
+    public NamedColumn(String name) {
+        this.name = name;
+    }
+
+    public void alias(String alias) {
+        this.alias = alias;
+    }
+
+    public String computeAlias(Labeled labeled, Function<Labeled, String> namer) {
+        if (this.alias==null) {
+            this.alias = namer.apply(labeled);
+        }
+        return this.alias;
+    }
+
+    public String computeAlias(Map<String,String> labels, Function<Map<String,String>,String> namer) {
+        if (this.alias==null) {
+            this.alias= namer.apply(labels);
+        }
+        return this.alias;
+    }
+
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedKeyspace.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.transformers.namecache;
+
+import io.nosqlbench.api.labels.Labeled;
+
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.function.Function;
+
+public class NamedKeyspace {
+    private final String ksname;
+    private final Map<String, NamedTable> tables = new LinkedHashMap<>();
+    private final Map<String, NamedType> types = new LinkedHashMap<>();
+    private String alias;
+
+    public NamedKeyspace(String ksname) {
+        this.ksname = ksname;
+    }
+
+    public NamedType type(String typename) {
+        return types.computeIfAbsent(typename, NamedType::new);
+    }
+
+    public NamedTable table(String tablename) {
+        return tables.computeIfAbsent(tablename, NamedTable::new);
+    }
+
+    public NamedKeyspace alias(String alias) {
+        this.alias = alias;
+        return this;
+    }
+
+    public String computeAlias(Labeled labeled, Function<Labeled,String> namer) {
+        if (this.alias==null) {
+            this.alias = namer.apply(labeled);
+        }
+        return this.alias;
+    }
+
+    public Collection<NamedTable> tables() {
+        return tables.values();
+    }
+
+    public Collection<NamedType> types() {
+        return types.values();
+    }
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedTable.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.transformers.namecache;
+
+import io.nosqlbench.api.labels.Labeled;
+
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.function.Function;
+
+public class NamedTable {
+    private final String tablename;
+    private final Map<String, NamedColumn> columns = new LinkedHashMap<>();
+    private String alias;
+
+    public NamedTable(String tablename) {
+        this.tablename = tablename;
+    }
+
+    public NamedColumn column(String name) {
+        return this.columns.computeIfAbsent(name, NamedColumn::new);
+    }
+
+    public NamedTable alias(String alias) {
+        this.alias = alias;
+        return this;
+    }
+
+    public String computeAlias(Labeled labeled, Function<Labeled,String> namer) {
+        if (this.alias==null) {
+            this.alias = namer.apply(labeled);
+        }
+        return this.alias;
+    }
+
+    public String getAlias() {
+        return this.alias;
+    }
+
+    public Collection<NamedColumn> columns() {
+        return columns.values();
+    }
+}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NamedType.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.cqlgen.transformers.namecache;
+
+import io.nosqlbench.api.labels.Labeled;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+public class NamedType {
+    private String name;
+    private String alias;
+    private final Map<String,NamedColumn> columns = new LinkedHashMap<>();
+
+    public NamedType(String typename) {
+        this.name = typename;
+    }
+
+    public void alias(String alias) {
+        this.alias = alias;
+    }
+
+    public NamedColumn column(String key) {
+        return this.columns.computeIfAbsent(key, NamedColumn::new);
+    }
+    public List<NamedColumn> getColumnDefs() {
+        return new ArrayList<>(columns.values());
+    }
+
+    public String computeAlias(Labeled labeled, Function<Labeled, String> namer) {
+        if (this.alias==null) {
+            this.alias = namer.apply(labeled);
+        }
+        return this.alias;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+}

adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/udts/ToUdt.java

@@ -43,11 +43,11 @@ public class ToUdt implements LongFunction<UdtValue> {
     public ToUdt(String spec) {
         this.spec=spec;
         typeinfo = CqlModelParser.parseCqlType(spec);
-        UserDefinedTypeBuilder builder = new UserDefinedTypeBuilder(typeinfo.getKeyspace(), typeinfo.getName());
+        UserDefinedTypeBuilder builder = new UserDefinedTypeBuilder(typeinfo.getKeyspace().getName(), typeinfo.getName());
-        typeinfo.getFields().forEach((name,typedef) -> {
+//        typeinfo.getFields().forEach((name,typedef) -> {
-            DataType dataType = resolveDataType(typedef);
+//            DataType dataType = resolveDataType(typedef);
-            builder.withField(name,dataType);
+//            builder.withField(name,dataType);
-        });
+//        });
         this.udt = builder.build();
 
         {

adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf

@@ -8,18 +8,19 @@
 #      outfile: _replaced.cql
 #      replacers:
 #        - - '^(\s*?)\b(options|role|roles|permissions|permission|date|key|timestamp|type|keys)\b(\s+[a-zA-Z][a-zA-Z<>,_ -]*?,?)$'
-#          - '$1PREFIX$2SUFFIX$3'
+#          - '$1"$2"$3'
 #        - - '^(.*?PRIMARY KEY.*?)\b(options|role|roles|permissions|permission|date|key|timestamp|type|keys)\b(.*?)$'
-#          - '$1PREFIX$2SUFFIX$3'
+#          - '$1"$2"$3'
 #        - - '^(.*?CLUSTERING ORDER BY.+?)\b(options|role|roles|permissions|permission|date|key|timestamp|type|keys)\b(.*?)$'
-#          - '$1PREFIX$2SUFFIX$3'
+#          - '$1"$2"$3'
 #        - - '^(\s*?CREATE TABLE.+?)\b(options|role|roles|permissions|permission|date|key|timestamp|type|keys)\b(.*?)$'
-#          - '$1PREFIX$2SUFFIX$3'
+#          - '$1"$2"$3'
 #
 model_transformers:
 
  # filters in or out keyspaces
-  - class: CGKeyspaceFilter
+  - name: keyspace_filter
+    class: CGKeyspaceFilter
     config:
       - exclude: system
       - exclude: system_.*
@@ -27,13 +28,14 @@ model_transformers:
       - exclude: dsefs_.*
       - exclude: cfs_.*
       - exclude: cfs
-      - exclude: HiveMetaStore
+      - exclude: Hive.*
       - exclude: spark_system
       - include: .*
 
   # replaces the replication settings with the provided values here,
-  # specifed as a text block to be put inside the curly braces
+  # specified as a text block to be put inside the curly braces
-  - class: CGReplicationSettingInjector
+  - name: replication
+    class: CGReplicationSettingInjector
     config:
       replication_fields: |
         'class': 'SimpleStrategy',
@@ -44,17 +46,15 @@ model_transformers:
 #  # Removes Keyspace DDL statements
 #  - class: CGKeySpaceDDLRemover
 
-#  This is now a generator behavior that is done automatically
-#  # Adds IF NOT EXIST to all DDL
-#  - class: CGIfNotExistsInjector
-
   # Replaces UDTs with blobs until we have full UDT generation capability
-  - class: CGUdtReplacer
+  - name: udtskipper
+    class: CGUdtReplacer
 
   # Reads a configured file path containing nodetool histogram stats output
   # If no histostats file is provided, then this is skipped, including
   # any downstream usage of this data
-  - class: CGGenStatsInjector
+  - name: tablestats
+    class: CGGenStatsInjector
     config:
       path: tablestats
 
@@ -62,21 +62,33 @@ model_transformers:
   # This depends on data from the stats injector above. If not provided,
   # this skips modifying ratios gracefully and they re all just set to 1
   # as usual.
-  - class: CGRatioCalculator
+  - name: ratios
+    class: CGRatioCalculator
 
-  # if this is set, and the fractional rate of operations against a table
+#   # if this is set, and the fractional rate of operations against a table
-  # counting reads and writes is less than the percent threshold, then
+#   # counting reads and writes is less than the percent threshold, then
-  # the table will be excluded from all op template generation.
+#   # the table will be excluded from all op template generation.
-  - class: UnusedTableRemover
+#   - class: UnusedTableRemover
-    config:
+#     config:
-      # this is as a fractional number, so 0.1 is the same as 10%
+#       # this is as a fractional number, so 0.1 is the same as 10%
-      minimum_threshold: 0.001
+#       minimum_threshold: 0.001
 
   # replaces names of keyspaces, tables, and columns with generated values
-  - class: CGNameObfuscator
+  - name: obfuscator
+    class: CGNameObfuscator
     config:
       namer: Combinations('0-9;0-9;0-9;0-9;0-9');
+#       mapfile: mapfile.json
 
+  # format the table names based on their total share of all ops
+  # The format is a String.format specifier.
+  #   Arg 1 "%1$" is the original name.
+  #   Arg 2 "%2$" is a double value.
+  # The resolution is specified by the decimal field width.
+  - name: ratio_suffix
+    class: CGRatioSuffixer
+    config:
+      format: "%1$s_%2$03d"
 # This controls how the elements in the schema are named in the yaml.
 # This affects block names, op template names and so on, and also how
 # op templates will be named in all logs and  metric views.
@@ -112,7 +124,6 @@ timeouts:
   delete: 10.0
 
 
-# future use, not active right now
 blockplan:
   # not needed when tags=block:'schema.*'
   # schema: schema-keyspaces, schema-tables, schema-types
@@ -125,11 +136,11 @@ blockplan:
   drop-keyspaces: drop-keyspaces
   # not needed when tags=block:'drop.*'
   # drop: drop-types, drop-tables, drop-keyspaces
-  rampup: insert
+  rampup: insert-seq
-  main-insert: insert
+  main-insert: insert-seq
-  main-select: select
+  main-select: select-seq
-  main-scan: scan-10
+  main-scan: scan-10-seq
-  main-update: update
+  main-update: update-seq
   # not needed when tags=block:'main.*'
   # main: insert, select, scan-10, update
 

adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/cql/parser/CqlParserHarnessTest.java

@@ -16,16 +16,11 @@
 
 package io.nosqlbench.converters.cql.cql.parser;
 
-import io.nosqlbench.cqlgen.exporter.CGWorkloadExporter;
+import io.nosqlbench.cqlgen.core.CGWorkloadExporter;
-import io.nosqlbench.cqlgen.exporter.transformers.CGModelTransformers;
 import io.nosqlbench.cqlgen.parser.CqlModelParser;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
-import java.nio.file.Path;
-
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
 public class CqlParserHarnessTest {
 
     private final static String ksddl = """
@@ -47,19 +42,13 @@ public class CqlParserHarnessTest {
 
     @Test
     public void testAllTypes() {
-        CGWorkloadExporter exporter = new CGWorkloadExporter(Path.of("src/test/resources/testschemas/cql_alltypes.cql"), new CGModelTransformers());
+        CGWorkloadExporter exporter = new CGWorkloadExporter();
+        exporter.appMain(new String[]{"src/test/resources/testschemas/cql_alltypes.cql"});
         exporter.setNamingTemplate("[OPTYPE-][COLUMN-][TYPEDEF-][TABLE!]-[KEYSPACE]");
         var data = exporter.getWorkloadAsYaml();
 
     }
 
-    @Disabled
-    @Test
-    public void testGenBasicWorkload() {
-        CGWorkloadExporter exporter = new CGWorkloadExporter(ddl, new CGModelTransformers());
-        assertThatThrownBy(() ->  exporter.getWorkloadAsYaml()).isInstanceOf(RuntimeException.class);
-    }
-
     @Test
     public void testCqlParserHarnessCombined() {
         CqlModelParser.parse(ddl, null);

adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/CGElementNamerTest.java

@@ -17,7 +17,7 @@
 package io.nosqlbench.converters.cql.exporters;
 
 import io.nosqlbench.api.labels.Labeled;
-import io.nosqlbench.cqlgen.exporter.CGElementNamer;
+import io.nosqlbench.cqlgen.core.CGElementNamer;
 import org.junit.jupiter.api.Test;
 
 import java.util.Map;

adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/CGWorkloadExporterTest.java

@@ -16,7 +16,7 @@
 
 package io.nosqlbench.converters.cql.exporters;
 
-import io.nosqlbench.cqlgen.exporter.CGColumnRebinder;
+import io.nosqlbench.cqlgen.core.CGColumnRebinder;
 import org.junit.jupiter.api.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;

adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/NamingFolioTest.java

@@ -16,7 +16,7 @@
 
 package io.nosqlbench.converters.cql.exporters;
 
-import io.nosqlbench.cqlgen.exporter.binders.NamingFolio;
+import io.nosqlbench.cqlgen.binders.NamingFolio;
 import org.junit.jupiter.api.Test;
 
 import java.util.Map;

adapter-http/src/main/java/io/nosqlbench/adapter/http/core/HttpSpace.java

@@ -71,7 +71,7 @@ public class HttpSpace implements NBNamedElement {
     public synchronized void applyConfig(NBConfiguration cfg) {
         this.followRedirects =
             HttpClient.Redirect.valueOf(
-                cfg.get("follow_redirects").toUpperCase(Locale.ROOT)
+                cfg.get("follow_redirects", String.class).toUpperCase(Locale.ROOT)
             );
         this.timeout = Duration.ofMillis(cfg.get("timeout", long.class));
         this.timeoutMillis = cfg.get("timeout", long.class);

docsys/src/main/java/io/nosqlbench/docsys/core/NBWebServerApp.java

@@ -16,7 +16,9 @@
 
 package io.nosqlbench.docsys.core;
 
+import io.nosqlbench.api.spi.BundledApp;
 import io.nosqlbench.docsys.endpoints.DocsysMarkdownEndpoint;
+import io.nosqlbench.nb.annotations.Service;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -27,24 +29,12 @@ import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.Arrays;
 
-public class NBWebServerApp {
+@Service(value=NBWebServerApp.class,selector="appserver")
+public class NBWebServerApp implements BundledApp {
     private static final Logger logger = LogManager.getLogger(NBWebServerApp.class);
 
     public static void main(String[] args) {
-        if (args.length > 0 && args[0].contains("help")) {
+        new NBWebServerApp().appMain(args);
-            showHelp();
-        } else if (args.length > 0 && args[0].contains("generate")) {
-            try {
-                String[] genargs = Arrays.copyOfRange(args, 1, args.length);
-                logger.info("Generating with args [" + String.join("][", args) + "]");
-                generate(genargs);
-            } catch (IOException e) {
-                logger.error("could not generate files with command " + String.join(" ", args));
-                e.printStackTrace();
-            }
-        } else {
-            runServer(args);
-        }
     }
 
     private static boolean deleteDirectory(File directoryToBeDeleted) {
@@ -152,4 +142,23 @@ public class NBWebServerApp {
     private static void listTopics() {
 
     }
+
+    @Override
+    public int appMain(String[] args) {
+        if (args.length > 0 && args[0].contains("help")) {
+            showHelp();
+        } else if (args.length > 0 && args[0].contains("generate")) {
+            try {
+                String[] genargs = Arrays.copyOfRange(args, 1, args.length);
+                logger.info("Generating with args [" + String.join("][", args) + "]");
+                generate(genargs);
+            } catch (IOException e) {
+                logger.error("could not generate files with command " + String.join(" ", args));
+                e.printStackTrace();
+            }
+        } else {
+            runServer(args);
+        }
+        return 0;
+    }
 }

driver-cockroachdb/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <groupId>io.nosqlbench</groupId>
         <artifactId>mvn-defaults</artifactId>
-        <version>4.17.20-SNAPSHOT</version>
+        <version>4.17.21-SNAPSHOT</version>
         <relativePath>../mvn-defaults</relativePath>
     </parent>
 
@@ -37,7 +37,7 @@
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>driver-jdbc</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>org.postgresql</groupId>

driver-jdbc/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <artifactId>nosqlbench</artifactId>
         <groupId>io.nosqlbench</groupId>
-        <version>4.17.20-SNAPSHOT</version>
+        <version>4.17.21-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -35,7 +35,7 @@
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>engine-api</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
             <scope>compile</scope>
         </dependency>
 

driver-jms/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <artifactId>mvn-defaults</artifactId>
         <groupId>io.nosqlbench</groupId>
-        <version>4.17.20-SNAPSHOT</version>
+        <version>4.17.21-SNAPSHOT</version>
         <relativePath>../mvn-defaults</relativePath>
     </parent>
 
@@ -56,7 +56,7 @@
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>engine-api</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
         </dependency>
 
         <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->

driver-jmx/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <artifactId>mvn-defaults</artifactId>
         <groupId>io.nosqlbench</groupId>
-        <version>4.17.20-SNAPSHOT</version>
+        <version>4.17.21-SNAPSHOT</version>
         <relativePath>../mvn-defaults</relativePath>
     </parent>
 
@@ -38,7 +38,7 @@
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>engine-api</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
             <scope>compile</scope>
         </dependency>
 

driver-kafka/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <artifactId>mvn-defaults</artifactId>
         <groupId>io.nosqlbench</groupId>
-        <version>4.17.20-SNAPSHOT</version>
+        <version>4.17.21-SNAPSHOT</version>
         <relativePath>../mvn-defaults</relativePath>
     </parent>
 
@@ -60,7 +60,7 @@
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>engine-api</artifactId>
-          <version>4.17.20-SNAPSHOT</version>
+          <version>4.17.21-SNAPSHOT</version>
         </dependency>
 
     </dependencies>

driver-pulsar/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <artifactId>mvn-defaults</artifactId>
         <groupId>io.nosqlbench</groupId>
-        <version>4.17.20-SNAPSHOT</version>
+        <version>4.17.21-SNAPSHOT</version>
         <relativePath>../mvn-defaults</relativePath>
     </parent>
 
@@ -56,7 +56,7 @@
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>engine-api</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
         </dependency>
 
         <!-- https://mvnrepository.com/artifact/commons-beanutils/commons-beanutils -->

engine-api/src/main/java/io/nosqlbench/engine/api/activityapi/errorhandling/modular/ResultCode.java

@@ -36,7 +36,7 @@ public class ResultCode implements ErrorHandler, NBMapConfigurable {
 
     @Override
     public void applyConfig(Map<String, ?> providedConfig) {
-        this.code = Byte.valueOf(providedConfig.get("code").toString());
+        this.code = Byte.parseByte(providedConfig.get("code").toString());
     }
 
     @Override

engine-api/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionCountMetrics.java

@@ -29,10 +29,12 @@ import java.util.concurrent.ConcurrentHashMap;
  */
 public class ExceptionCountMetrics {
     private final ConcurrentHashMap<String, Counter> counters = new ConcurrentHashMap<>();
+    private final Counter allerrors;
     private final ActivityDef activityDef;
 
     public ExceptionCountMetrics(ActivityDef activityDef) {
         this.activityDef = activityDef;
+        allerrors=ActivityMetrics.counter(activityDef, "errorcounts.ALL");
     }
 
     public void count(String name) {
@@ -46,6 +48,7 @@ public class ExceptionCountMetrics {
             }
         }
         c.inc();
+        allerrors.inc();
     }
 
     public List<Counter> getCounters() {

engine-api/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionHistoMetrics.java

@@ -31,10 +31,12 @@ import java.util.concurrent.ConcurrentHashMap;
  */
 public class ExceptionHistoMetrics {
     private final ConcurrentHashMap<String, Histogram> histos = new ConcurrentHashMap<>();
+    private final Histogram allerrors;
     private final ActivityDef activityDef;
 
     public ExceptionHistoMetrics(ActivityDef activityDef) {
         this.activityDef = activityDef;
+        allerrors = ActivityMetrics.histogram(activityDef, "errorhistos.ALL", activityDef.getParams().getOptionalInteger("hdr_digits").orElse(4));
     }
 
     public void update(String name, long magnitude) {
@@ -48,6 +50,7 @@ public class ExceptionHistoMetrics {
             }
         }
         h.update(magnitude);
+        allerrors.update(magnitude);
     }
 
 

engine-api/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionMeterMetrics.java

@@ -29,10 +29,12 @@ import java.util.concurrent.ConcurrentHashMap;
  */
 public class ExceptionMeterMetrics {
     private final ConcurrentHashMap<String, Meter> meters = new ConcurrentHashMap<>();
+    private final Meter allerrors;
     private final ActivityDef activityDef;
 
     public ExceptionMeterMetrics(ActivityDef activityDef) {
         this.activityDef = activityDef;
+        allerrors = ActivityMetrics.meter(activityDef, "errormeters.ALL");
     }
 
     public void mark(String name) {
@@ -41,11 +43,12 @@ public class ExceptionMeterMetrics {
             synchronized (meters) {
                 c = meters.computeIfAbsent(
                     name,
-                    k -> ActivityMetrics.meter(activityDef, "exceptions." + name)
+                    k -> ActivityMetrics.meter(activityDef, "errormeters." + name)
                 );
             }
         }
         c.mark();
+        allerrors.mark();
     }
 
     public List<Meter> getMeters() {

engine-api/src/main/java/io/nosqlbench/engine/api/metrics/ExceptionTimerMetrics.java

@@ -30,10 +30,16 @@ import java.util.concurrent.TimeUnit;
  */
 public class ExceptionTimerMetrics {
     private final ConcurrentHashMap<String, Timer> timers = new ConcurrentHashMap<>();
+    private final Timer allerrors;
     private final ActivityDef activityDef;
 
     public ExceptionTimerMetrics(ActivityDef activityDef) {
         this.activityDef = activityDef;
+        allerrors = ActivityMetrics.timer(
+            activityDef,
+            "errortimers.ALL",
+            activityDef.getParams().getOptionalInteger("hdr_digits").orElse(4)
+        );
     }
 
     public void update(String name, long nanosDuration) {
@@ -42,11 +48,12 @@ public class ExceptionTimerMetrics {
             synchronized (timers) {
                 timer = timers.computeIfAbsent(
                     name,
-                    k -> ActivityMetrics.timer(activityDef, "exceptions." + name, activityDef.getParams().getOptionalInteger("hdr_digits").orElse(4))
+                    k -> ActivityMetrics.timer(activityDef, "errortimers." + name, activityDef.getParams().getOptionalInteger("hdr_digits").orElse(4))
                 );
             }
         }
         timer.update(nanosDuration, TimeUnit.NANOSECONDS);
+        allerrors.update(nanosDuration, TimeUnit.NANOSECONDS);
     }
 
     public List<Timer> getTimers() {

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java

@@ -16,14 +16,21 @@
 
 package io.nosqlbench.engine.cli;
 
-import io.nosqlbench.api.docsapi.docexporter.BundledMarkdownExporter;
+import io.nosqlbench.api.annotations.Annotation;
-import io.nosqlbench.docsys.core.NBWebServerApp;
+import io.nosqlbench.api.annotations.Layer;
+import io.nosqlbench.api.content.Content;
+import io.nosqlbench.api.content.NBIO;
+import io.nosqlbench.api.engine.metrics.ActivityMetrics;
+import io.nosqlbench.api.errors.BasicError;
+import io.nosqlbench.api.logging.NBLogLevel;
+import io.nosqlbench.api.metadata.SessionNamer;
+import io.nosqlbench.api.metadata.SystemId;
+import io.nosqlbench.api.spi.BundledApp;
 import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogDumperUtility;
 import io.nosqlbench.engine.api.activityapi.cyclelog.outputs.cyclelog.CycleLogImporterUtility;
 import io.nosqlbench.engine.api.activityapi.input.InputType;
 import io.nosqlbench.engine.api.activityapi.output.OutputType;
 import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
-import io.nosqlbench.api.engine.metrics.ActivityMetrics;
 import io.nosqlbench.engine.core.annotation.Annotators;
 import io.nosqlbench.engine.core.lifecycle.*;
 import io.nosqlbench.engine.core.logging.LoggerConfig;
@@ -35,16 +42,8 @@ import io.nosqlbench.engine.core.script.ScenariosExecutor;
 import io.nosqlbench.engine.core.script.ScriptParams;
 import io.nosqlbench.engine.docker.DockerMetricsManager;
 import io.nosqlbench.nb.annotations.Maturity;
-import io.nosqlbench.api.annotations.Annotation;
+import io.nosqlbench.nb.annotations.Service;
-import io.nosqlbench.api.annotations.Layer;
+import io.nosqlbench.nb.annotations.ServiceSelector;
-import io.nosqlbench.api.content.Content;
-import io.nosqlbench.api.content.NBIO;
-import io.nosqlbench.api.errors.BasicError;
-import io.nosqlbench.api.logging.NBLogLevel;
-import io.nosqlbench.api.markdown.exporter.MarkdownExporter;
-import io.nosqlbench.api.metadata.SessionNamer;
-import io.nosqlbench.api.metadata.SystemId;
-import io.nosqlbench.virtdata.userlibs.apps.VirtDataMainApp;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.config.ConfigurationFactory;
@@ -53,15 +52,10 @@ import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Arrays;
+import java.util.*;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
 import java.util.concurrent.locks.LockSupport;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -178,6 +172,21 @@ public class NBCLI implements Function<String[], Integer> {
         logger.info("command-line: " + Arrays.stream(args).collect(Collectors.joining(" ")));
         logger.info("client-hardware: " + SystemId.getHostSummary());
 
+
+        // Invoke any bundled app which matches the name of the first non-option argument, if it exists.
+        // If it does not, continue with no fanfare. Let it drop through to other command resolution methods.
+        if (args.length>0 && args[0].matches("\\w[\\w\\d-_.]+")) {
+            ServiceSelector<BundledApp> apploader = ServiceSelector.of(args[0], ServiceLoader.load(BundledApp.class));
+            BundledApp app = apploader.get().orElse(null);
+            if (app!=null) {
+                String[] appargs = Arrays.copyOfRange(args, 1, args.length);
+                logger.info("invoking bundled app '" + args[0] + "' (" + app.getClass().getSimpleName() + ").");
+                globalOptions.setWantsStackTraces(true);
+                int result = app.appMain(appargs);
+                return result;
+            }
+        }
+
         boolean dockerMetrics = globalOptions.wantsDockerMetrics();
         String dockerMetricsAt = globalOptions.wantsDockerMetricsAt();
         String reportGraphiteTo = globalOptions.wantsReportGraphiteTo();
@@ -226,40 +235,6 @@ public class NBCLI implements Function<String[], Integer> {
             annotatorsConfig = "[{type:'log',level:'info'}]";
         }
 
-        if (args.length > 0 && args[0].toLowerCase().equals("cqlgen")) {
-            String exporterImpl = "io.nosqlbench.cqlgen.exporter.CGWorkloadExporter";
-            String[] exporterArgs = Arrays.copyOfRange(args, 1, args.length);
-            try {
-                Class<?> genclass = Class.forName(exporterImpl);
-                Method main = genclass.getMethod("main", new String[0].getClass());
-                Object result = main.invoke(null, new Object[]{exporterArgs});
-            } catch (ClassNotFoundException e) {
-                throw new RuntimeException("cql workload exporter implementation " + exporterImpl + " was not found in this runtime.");
-            } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
-                System.out.println("Error in app: " + e.toString());
-                e.printStackTrace();
-                throw new RuntimeException("error while invoking " + exporterImpl + ": " + e.toString(),e);
-            }
-            return EXIT_OK;
-        }
-        if (args.length > 0 && args[0].toLowerCase().equals("export-docs")) {
-            BundledMarkdownExporter.main(Arrays.copyOfRange(args,1,args.length));
-            return EXIT_OK;
-        }
-        if (args.length > 0 && args[0].toLowerCase().equals("virtdata")) {
-            VirtDataMainApp.main(Arrays.copyOfRange(args, 1, args.length));
-            return EXIT_OK;
-        }
-        if (args.length > 0 && args[0].toLowerCase().matches("docserver|appserver")) {
-            NBWebServerApp.main(Arrays.copyOfRange(args, 1, args.length));
-            return EXIT_OK;
-        }
-        if (args.length > 0 && args[0].toLowerCase().equals(MarkdownExporter.APP_NAME)
-        ) {
-            MarkdownExporter.main(Arrays.copyOfRange(args, 1, args.length));
-            return EXIT_OK;
-        }
-
         NBCLIOptions options = new NBCLIOptions(args);
         logger = LogManager.getLogger("NBCLI");
 
@@ -282,6 +257,20 @@ public class NBCLI implements Function<String[], Integer> {
             return EXIT_OK;
         }
 
+        if (options.isWantsListApps()) {
+            ServiceLoader<BundledApp> loader = ServiceLoader.load(BundledApp.class);
+            for (ServiceLoader.Provider<BundledApp> provider : loader.stream().toList()) {
+                Class<? extends BundledApp> appType = provider.type();
+                String name = appType.getAnnotation(Service.class).selector();
+                System.out.println(String.format("%-40s %s",name,appType.getCanonicalName()));
+            }
+            return EXIT_OK;
+        }
+
+        if (options.getWantsListCommands()) {
+            NBCLICommandParser.RESERVED_WORDS.forEach(System.out::println);
+            return EXIT_OK;
+        }
         if (options.wantsActivityTypes()) {
             new ActivityTypeLoader().getAllSelectors().forEach(System.out::println);
             return EXIT_OK;
@@ -297,7 +286,7 @@ public class NBCLI implements Function<String[], Integer> {
             return EXIT_OK;
         }
 
-        if (options.wantsScriptList()) {
+        if (options.wantsListScripts()) {
             NBCLIScripts.printScripts(true, options.wantsIncludes());
             return EXIT_OK;
         }

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLICommandParser.java

@@ -16,14 +16,19 @@
 
 package io.nosqlbench.engine.cli;
 
-import io.nosqlbench.engine.api.scenarios.NBCLIScenarioParser;
 import io.nosqlbench.api.content.Content;
 import io.nosqlbench.api.content.NBIO;
+import io.nosqlbench.engine.api.scenarios.NBCLIScenarioParser;
 
-import java.security.InvalidParameterException;
 import java.util.*;
 
+/**
+ * This parser will return a non-empty optional if there is no error.
+ * If the optional is empty, then it means some part of the command structure
+ * was not recognized.
+ */
 public class NBCLICommandParser {
+
     private static final String FRAGMENT = "fragment";
     private static final String SCRIPT = "script";
     private static final String START = "start";
@@ -37,21 +42,20 @@ public class NBCLICommandParser {
     public static final Set<String> RESERVED_WORDS = new HashSet<>() {{
         addAll(
                 Arrays.asList(
-                        SCRIPT, ACTIVITY, SCENARIO, RUN, START,
+                        FRAGMENT, SCRIPT, START, RUN, AWAIT, STOP, ACTIVITY, SCENARIO, WAIT_MILLIS
-                        FRAGMENT, STOP, AWAIT, WAIT_MILLIS
                 )
         );
     }};
 
-    public static void parse(
+    public static Optional<List<Cmd>> parse(
             LinkedList<String> arglist,
-            LinkedList<Cmd> cmdList,
             String... includes
     ) {
+        List<Cmd> cmdList = new LinkedList<>();
         PathCanonicalizer canonicalizer = new PathCanonicalizer(includes);
         while (arglist.peekFirst() != null) {
             String word = arglist.peekFirst();
-            Cmd cmd = null;
+            Cmd cmd;
             switch (word) {
                 case FRAGMENT:
                 case SCRIPT:
@@ -80,11 +84,12 @@ public class NBCLICommandParser {
                     } else if (NBCLIScenarioParser.isFoundWorkload(word, includes)) {
                         NBCLIScenarioParser.parseScenarioCommand(arglist, RESERVED_WORDS, includes);
                     } else {
-                        throw new InvalidParameterException("unrecognized option:" + word);
+                        return Optional.empty();
                     }
                     break;
             }
         }
+        return Optional.of(cmdList);
 
     }
 }

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java

@@ -61,6 +61,7 @@ public class NBCLIOptions {
 
     // Discovery
     private static final String HELP = "--help";
+    private static final String LIST_COMMANDS = "--list-commands";
     private static final String LIST_METRICS = "--list-metrics";
     private static final String LIST_DRIVERS = "--list-drivers";
     private static final String LIST_ACTIVITY_TYPES = "--list-activity-types";
@@ -69,6 +70,7 @@ public class NBCLIOptions {
     private static final String LIST_SCENARIOS = "--list-scenarios";
     private static final String LIST_INPUT_TYPES = "--list-input-types";
     private static final String LIST_OUTPUT_TYPES = "--list-output-types";
+    private static final String LIST_APPS = "--list-apps";
     private static final String VERSION_COORDS = "--version-coords";
     private static final String VERSION = "--version";
     private static final String SHOW_SCRIPT = "--show-script";
@@ -128,7 +130,7 @@ public class NBCLIOptions {
     //    private static final String DEFAULT_CONSOLE_LOGGING_PATTERN = "%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n";
 
 
-    private final LinkedList<Cmd> cmdList = new LinkedList<>();
+    private final List<Cmd> cmdList = new ArrayList<>();
     private int logsMax = 0;
     private boolean wantsVersionShort = false;
     private boolean wantsVersionCoords = false;
@@ -160,8 +162,8 @@ public class NBCLIOptions {
     private Map<String, String> logLevelsOverrides = new HashMap<>();
     private boolean enableChart = false;
     private boolean dockerMetrics = false;
-    private boolean wantsScenariosList = false;
+    private boolean wantsListScenarios = false;
-    private boolean wantsScriptList = false;
+    private boolean wantsListScripts = false;
     private String wantsToCopyWorkload = null;
     private boolean wantsWorkloadsList = false;
     private final List<String> wantsToIncludePaths = new ArrayList<>();
@@ -185,7 +187,16 @@ public class NBCLIOptions {
     private boolean enableAnsi = System.getenv("TERM")!=null && !System.getenv("TERM").isEmpty();
     private Maturity minMaturity = Maturity.Unspecified;
     private String graphitelogLevel="info";
+    private boolean wantsListCommands = false;
+    private boolean wantsListApps = false;
 
+    public boolean isWantsListApps() {
+        return wantsListApps;
+    }
+
+    public boolean getWantsListCommands() {
+        return wantsListCommands;
+    }
     public String getAnnotatorsConfig() {
         return annotatorsConfig;
     }
@@ -517,6 +528,10 @@ public class NBCLIOptions {
                     arglist.removeFirst();
                     showScript = true;
                     break;
+                case LIST_COMMANDS:
+                    arglist.removeFirst();
+                    this.wantsListCommands = true;
+                    break;
                 case LIST_METRICS:
                     arglist.removeFirst();
                     arglist.addFirst("start");
@@ -596,16 +611,20 @@ public class NBCLIOptions {
                     break;
                 case LIST_SCENARIOS:
                     arglist.removeFirst();
-                    wantsScenariosList = true;
+                    wantsListScenarios = true;
                     break;
                 case LIST_SCRIPTS:
                     arglist.removeFirst();
-                    wantsScriptList = true;
+                    wantsListScripts = true;
                     break;
                 case LIST_WORKLOADS:
                     arglist.removeFirst();
                     wantsWorkloadsList = true;
                     break;
+                case LIST_APPS:
+                    arglist.removeFirst();
+                    wantsListApps= true;
+                    break;
                 case SCRIPT_FILE:
                     arglist.removeFirst();
                     scriptFile = readWordOrThrow(arglist, "script file");
@@ -619,7 +638,31 @@ public class NBCLIOptions {
             }
         }
         arglist = nonincludes;
-        NBCLICommandParser.parse(arglist, cmdList);
+        Optional<List<Cmd>> commands = NBCLICommandParser.parse(arglist);
+        if (commands.isPresent()) {
+            this.cmdList.addAll(commands.get());
+        } else {
+            String arg = arglist.peekFirst();
+            Objects.requireNonNull(arg);
+            String helpmsg = """
+                Could not recognize command 'ARG'.
+                This means that all of the following searches for a compatible command failed:
+                1. commands: no scenario command named 'ARG' is known. (start, run, await, ...)
+                2. scripts: no auto script named './scripts/auto/ARG.js' in the local filesystem.
+                3. scripts: no auto script named 'scripts/auto/ARG.js' was found in the PROG binary.
+                4. workloads: no workload file named ARG[.yaml] was found in the local filesystem, even in include paths INCLUDES.
+                5. workloads: no workload file named ARG[.yaml] was bundled in PROG binary, even in include paths INCLUDES.
+                6. apps: no application named ARG was bundled in PROG.
+
+                You can discover available ways to invoke PROG by using the various --list-* commands:
+                [ --list-commands, --list-scripts, --list-workloads (and --list-scenarios), --list-apps ]
+                """
+                .replaceAll("ARG",arg)
+                .replaceAll("PROG","nb5")
+                .replaceAll("INCLUDES", String.join(",",this.wantsIncludes()));
+            throw new BasicError(helpmsg);
+
+        }
     }
 
 
@@ -859,11 +902,11 @@ public class NBCLIOptions {
     }
 
     public boolean wantsScenariosList() {
-        return wantsScenariosList;
+        return wantsListScenarios;
     }
 
-    public boolean wantsScriptList() {
+    public boolean wantsListScripts() {
-        return wantsScriptList;
+        return wantsListScripts;
     }
 
     public boolean wantsToCopyResource() {

engine-cli/src/main/resources/error-handlers.md

@@ -15,12 +15,15 @@ The currently supported handler verbs, like `stop` above, are:
 
 * **ignore** If an error matches this verb in a handler chain, then it
   will be silently ignored.
-* **counter** Count each uniquely named error with a counter metric.
+* **counter** Count each uniquely named error with a counter metric. These will show up in
-* **meter** Meter each uniquely named error with a meter metric.
+  metrics as `...errorcounts.<name>` and `...errorcounts.ALL`.
+* **meter** Meter each uniquely named error with a meter metric. These will show up in metrics
+  as `...errormeters.<name>` and `...errormeters.ALL`.
 * **histogram** Track the session time of each uniquely named error with a
-  histogram.
+  histogram. This will show as `...errorhistos.<name>` and `...errorhistos.ALL`.
 * **timer** Count, Meter, and Track session times of each uniquely named
   error with a timer metric, which combines the three forms above.
+  This will show as `...errortimers.<name>` and `...errortimers.ALL`.
 * **warn** Log a warning to the log with the error details.
 * **stop** Allow the error to propagate through the stack to cause the
   activity to be stopped.

engine-cli/src/test/java/io/nosqlbench/engine/cli/TestNBCLIOptions.java

@@ -205,7 +205,7 @@ public class TestNBCLIOptions {
     @Test
     public void listScripts() {
         NBCLIOptions opts = new NBCLIOptions(new String[]{ "--list-scripts"});
-        assertThat(opts.wantsScriptList()).isTrue();
+        assertThat(opts.wantsListScripts()).isTrue();
     }
 
     @Test

engine-core/src/main/java/io/nosqlbench/engine/core/logging/LoggerConfig.java

@@ -62,7 +62,7 @@ public class LoggerConfig extends ConfigurationFactory {
     );
 
     /**
-     * Some included libraries are spammy and intefere with normal diagnostic visibility, so
+     * Some included libraries are spammy and interfere with normal diagnostic visibility, so
      * we squelch them to some reasonable level so they aren't a nuisance.
      */
     public static Map<String, Level> BUILTIN_OVERRIDES = Map.of(

engine-rest/src/main/java/io/nosqlbench/engine/rest/resources/ScenarioExecutorEndpoint.java

@@ -122,7 +122,10 @@ public class ScenarioExecutorEndpoint implements WebServiceObject {
         }
 
         args = substituteFilenames(rq, args);
-        NBCLICommandParser.parse(args, cmdList, workspace.asIncludes());
+        Optional<List<Cmd>> parsed = NBCLICommandParser.parse(args, workspace.asIncludes());
+        if (!parsed.isPresent()) {
+            return Response.serverError().entity("Unable to render command stream from provided command spec.").build();
+        }
         ScriptBuffer buffer = new BasicScriptBuffer();
         buffer.add(cmdList.toArray(new Cmd[0]));
 

nb-api/src/main/java/io/nosqlbench/api/config/standard/NBConfiguration.java

@@ -83,11 +83,29 @@ public class NBConfiguration {
         } else {
             return value;
         }
-
     }
 
-    public String get(String name) {
+    /**
-        return get(name, String.class);
+     * Get a config value or object by name. This uses type inference (as a generic method)
+     * in addition to the internal model for type checking and ergonomic use. If you do not
+     * call this within an assignment or context where the Java compiler knows what type you
+     * are expecting, then use {@link #get(String, Class)} instead.
+     * @param name The name of the configuration parameter
+     * @param <T> The (inferred) generic type of the configuration value
+     * @return The value of type T, matching the config model type for the provided field name
+     */
+    public <T> T get(String name) {
+        Param<T> param = (Param<T>)model.getNamedParams().get(name);
+        Object object = this.data.get(name);
+        if (param.type.isInstance(object)) {
+            return (T) object;
+        } else if (param.type.isAssignableFrom(object.getClass())) {
+            return param.type.cast(object);
+        } else if (NBTypeConverter.canConvert(object, param.type)) {
+            return NBTypeConverter.convert(object, param.type);
+        } else {
+            throw new NBConfigError("Unable to assign config value for field '" + name + "' of type '" + object.getClass().getCanonicalName() + "' to the required return type '" + param.type.getCanonicalName() + "' as specified in the config model for '" + model.getOf().getCanonicalName());
+        }
     }
 
     public <T> T get(String name, Class<? extends T> type) {

nb-api/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledMarkdownExporter.java

@@ -16,6 +16,8 @@
 
 package io.nosqlbench.api.docsapi.docexporter;
 
+import io.nosqlbench.api.spi.BundledApp;
+import io.nosqlbench.nb.annotations.Service;
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
@@ -24,10 +26,15 @@ import java.io.IOException;
 import java.nio.file.Path;
 import java.util.List;
 
-public class BundledMarkdownExporter {
+@Service(value=BundledApp.class,selector = "export-docs")
+public class BundledMarkdownExporter implements BundledApp {
 
     public static void main(String[] args) {
+        new BundledMarkdownExporter().appMain(args);
 
+    }
+    @Override
+    public int appMain(String[] args) {
         final OptionParser parser = new OptionParser();
 
         OptionSpec<String> zipfileSpec = parser.accepts("zipfile", "zip file to write to")
@@ -46,6 +53,6 @@ public class BundledMarkdownExporter {
         String zipfile = options.valueOf(zipfileSpec);
 
         new BundledMarkdownZipExporter(new BundledFrontmatterInjector()).exportDocs(Path.of(zipfile));
+        return 0;
     }
-
 }

nb-api/src/main/java/io/nosqlbench/api/labels/Labeled.java

@@ -16,6 +16,8 @@
 
 package io.nosqlbench.api.labels;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
@@ -29,6 +31,7 @@ public interface Labeled {
         }
         return map;
     }
+
     default Map<String, String> getLabelsAnd(Map<String,String> extra) {
         LinkedHashMap<String,String> map = new LinkedHashMap<>(getLabels());
         map.putAll(extra);
@@ -51,4 +54,20 @@ public interface Labeled {
             return labels;
         }
     }
+
+    default String linearized(Map<String,String> and) {
+        StringBuilder sb= new StringBuilder();
+        Map<String, String> allLabels = this.getLabelsAnd(and);
+        ArrayList<String> sortedLabels = new ArrayList<>(allLabels.keySet());
+        Collections.sort(sortedLabels);
+        for (String label : sortedLabels) {
+            sb.append(label).append(":").append(allLabels.get(label)).append((","));
+        }
+        sb.setLength(sb.length()-",".length());
+        return sb.toString();
+    }
+
+    default String linearized(String... and) {
+        return linearized(getLabelsAnd(and));
+    }
 }

nb-api/src/main/java/io/nosqlbench/api/markdown/exporter/MarkdownExporter.java

@@ -19,6 +19,8 @@ package io.nosqlbench.api.markdown.exporter;
 import io.nosqlbench.api.markdown.aggregator.MarkdownDocs;
 import io.nosqlbench.api.markdown.types.DocScope;
 import io.nosqlbench.api.markdown.types.MarkdownInfo;
+import io.nosqlbench.api.spi.BundledApp;
+import io.nosqlbench.nb.annotations.Service;
 import joptsimple.*;
 
 import java.nio.file.Path;
@@ -27,35 +29,15 @@ import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-public class MarkdownExporter implements Runnable {
+@Service(value = BundledApp.class, selector = "markdown-exporter")
+public class MarkdownExporter implements BundledApp, Runnable {
 
     public static final String APP_NAME = "exporter";
-    private final Path basePath;
+    private Path basePath;
-    private final Set<DocScope> scopeSet;
+    private Set<DocScope> scopeSet;
-
-    public MarkdownExporter(Path basePath, Set<DocScope> scopeSet) {
-        this.basePath = basePath;
-        this.scopeSet = scopeSet;
-    }
 
     public static void main(String[] args) {
-        final OptionParser parser = new OptionParser();
+        new MarkdownExporter().appMain(args);
-
-        OptionSpec<String> basedir = parser.accepts("basedir", "base directory to write to")
-            .withRequiredArg().ofType(String.class).defaultsTo(".");
-
-        OptionSpec<String> docScopes = parser.accepts("scopes", "scopes of documentation to export")
-            .withRequiredArg().ofType(String.class).defaultsTo(DocScope.ANY.toString());
-
-        parser.acceptsAll(List.of("-h","--help","help"),"Display help").forHelp();
-
-        OptionSet options = parser.parse(args);
-
-        Path basePath = Path.of(basedir.value(options));
-        Set<DocScope> scopeSet = docScopes.values(options).stream().map(DocScope::valueOf).collect(Collectors.toSet());
-
-
-        new MarkdownExporter(basePath,scopeSet).run();
     }
 
     @Override
@@ -65,4 +47,25 @@ public class MarkdownExporter implements Runnable {
     }
 
 
+    @Override
+    public int appMain(String[] args) {
+        final OptionParser parser = new OptionParser();
+
+        OptionSpec<String> basedir = parser.accepts("basedir", "base directory to write to")
+            .withRequiredArg().ofType(String.class).defaultsTo(".");
+
+        OptionSpec<String> docScopes = parser.accepts("scopes", "scopes of documentation to export")
+            .withRequiredArg().ofType(String.class).defaultsTo(DocScope.ANY.toString());
+
+        parser.acceptsAll(List.of("-h", "--help", "help"), "Display help").forHelp();
+
+        OptionSet options = parser.parse(args);
+
+        Path basePath = Path.of(basedir.value(options));
+        Set<DocScope> scopeSet = docScopes.values(options).stream().map(DocScope::valueOf).collect(Collectors.toSet());
+        this.basePath = basePath;
+        this.scopeSet = scopeSet;
+        run();
+        return 0;
+    }
 }

nb-api/src/main/java/io/nosqlbench/api/spi/BundledApp.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.api.spi;
+
+public interface BundledApp {
+    int appMain(String[] args);
+}

nb/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <artifactId>mvn-defaults</artifactId>
         <groupId>io.nosqlbench</groupId>
-        <version>4.17.20-SNAPSHOT</version>
+        <version>4.17.21-SNAPSHOT</version>
         <relativePath>../mvn-defaults</relativePath>
     </parent>
 
@@ -40,31 +40,31 @@
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>nbr</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
         </dependency>
 
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>adapter-stdout</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
         </dependency>
 
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>adapter-diag</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
         </dependency>
 
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>adapter-dynamodb</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
         </dependency>
 
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>adapter-cqld4</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
         </dependency>
 
         <!-- Everything below this line constitutes the delta between nb and nb5 -->
@@ -73,51 +73,51 @@
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>driver-http</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
         </dependency>
 
 
         <!--        <dependency>-->
         <!--            <groupId>io.nosqlbench</groupId>-->
         <!--            <artifactId>driver-kafka</artifactId>-->
-        <!--            <version>4.17.20-SNAPSHOT</version>-->
+        <!--            <version>4.17.21-SNAPSHOT</version>-->
         <!--        </dependency>-->
 
 
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>driver-tcp</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
         </dependency>
 
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>driver-jmx</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
         </dependency>
 
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>driver-mongodb</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
         </dependency>
 
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>driver-pulsar</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
         </dependency>
 
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>driver-cockroachdb</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
         </dependency>
 
         <dependency>
             <groupId>io.nosqlbench</groupId>
             <artifactId>driver-jms</artifactId>
-            <version>4.17.20-SNAPSHOT</version>
+            <version>4.17.21-SNAPSHOT</version>
         </dependency>
 
 
@@ -193,7 +193,7 @@
                 <dependency>
                     <groupId>io.nosqlbench</groupId>
                     <artifactId>driver-mongodb</artifactId>
-                    <version>4.17.20-SNAPSHOT</version>
+                    <version>4.17.21-SNAPSHOT</version>
                 </dependency>
             </dependencies>
         </profile>

virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/VirtDataMainApp.java

@@ -16,6 +16,8 @@
 
 package io.nosqlbench.virtdata.userlibs.apps;
 
+import io.nosqlbench.api.spi.BundledApp;
+import io.nosqlbench.nb.annotations.Service;
 import io.nosqlbench.virtdata.userlibs.apps.diagnoseapp.VirtDataDiagnoseApp;
 import io.nosqlbench.virtdata.userlibs.apps.docsapp.VirtDataGenDocsApp;
 import io.nosqlbench.virtdata.userlibs.apps.valuesapp.VirtDataCheckPerfApp;
@@ -25,7 +27,8 @@ import java.util.Arrays;
 /**
  * This just routes the user to the correct sub-app depending on the leading verb, stripping it off in the process.
  */
-public class VirtDataMainApp {
+@Service(value=BundledApp.class, selector = "virtdata")
+public class VirtDataMainApp implements BundledApp {
 
     private final static String APP_TESTMAPPER = "testmapper";
     private final static String APP_GENDOCS = "gendocs";
@@ -37,9 +40,14 @@ public class VirtDataMainApp {
     }
 
     public static void main(String[] args) {
+        new VirtDataMainApp().appMain(args);
+    }
+
+    @Override
+    public int appMain(String[] args) {
         if (args.length == 0) {
             System.out.println("Usage: app (" + APP_TESTMAPPER + "|" + APP_GENDOCS + "|" + APP_DIAGNOSE +")");
-            return;
+            return 1;
         }
 
         String appSelection = args[0];
@@ -57,5 +65,6 @@ public class VirtDataMainApp {
         } else {
             System.err.println("Error in command line. The first argument must one of " + String.join(",", names));
         }
+        return 0;
     }
 }