cqlgen updates

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/BindingsAccumulator.java

@@ -17,14 +17,17 @@
 package io.nosqlbench.cqlgen.binders;
 
 import io.nosqlbench.cqlgen.api.BindingsLibrary;
+import io.nosqlbench.cqlgen.core.CGDefaultCqlBindings;
+import io.nosqlbench.cqlgen.core.CGWorkloadExporter;
 import io.nosqlbench.cqlgen.model.CqlColumnBase;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.io.File;
 import java.util.*;
 
 public class BindingsAccumulator {
-    private final static Logger logger = LogManager.getLogger("CQL-GENERATOR");
+    private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.APPNAME+"/bindings-cache");
 
     private final NamingFolio namer;
     private final List<BindingsLibrary> libraries;
@@ -78,7 +81,9 @@ public class BindingsAccumulator {
             Optional<Binding> binding = library.resolveBindingsFor(def);
             if (binding.isPresent()) {
                 if (binding.get().getRecipe()==null) {
-                    throw new RuntimeException("Binding returned from library " + library + "' was null, for def '" + def + "'");
+                    throw new RuntimeException("Binding returned from library " + library + "' was null, for def '" + def + "'. " +
+                        "This probably means you need to add a default binding for '" + def.getTrimmedTypedef() + "' to " +
+                    CGDefaultCqlBindings.DEFAULT_CFG_DIR+ File.separator+CGDefaultCqlBindings.DEFAULT_BINDINGS_FILE+" (see cqlgen help for details)");
                 }
                 return binding;
             }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGColumnRebinder.java

@@ -25,7 +25,7 @@ import org.apache.logging.log4j.Logger;
 
 public class CGColumnRebinder {
 
-    private final static Logger logger = LogManager.getLogger(CGColumnRebinder.class);
+    private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.APPNAME+"/rebinder");
     private final BindingsAccumulator accumulator;
     private final double partitionMultiplier;
     private final int quantizerDigits;

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java

@@ -32,23 +32,23 @@ import java.util.Optional;
 public class CGDefaultCqlBindings implements BindingsLibrary {
 
     private final Map<String, String> bindings;
-    private final static String exporterCfgDir = "cqlgen";
+    public final static String DEFAULT_CFG_DIR = "cqlgen";
-    private final static String bindingsFileName = "bindings-cqlgen.yaml";
+    public final static String DEFAULT_BINDINGS_FILE = "bindings-cqlgen.yaml";
 
     public CGDefaultCqlBindings() {
         String yamlContent = NBIO.all()
-            .name(bindingsFileName)
+            .name(DEFAULT_BINDINGS_FILE)
             .first()
             .map(Content::asString)
-            .or(() -> loadLocal(bindingsFileName))
+            .or(() -> loadLocal(DEFAULT_BINDINGS_FILE))
-            .orElseThrow(() -> new RuntimeException("Unable to load " + bindingsFileName + ", from local dir or internally as cqlgen/" + bindingsFileName));
+            .orElseThrow(() -> new RuntimeException("Unable to load " + DEFAULT_BINDINGS_FILE + ", from local dir or internally as cqlgen/" + DEFAULT_BINDINGS_FILE));
         StmtsDocList stmtsDocs = StatementsLoader.loadString(yamlContent, Map.of());
         this.bindings = stmtsDocs.getDocBindings();
     }
 
     private Optional<String> loadLocal(String path) {
         try {
-            String resourceName = exporterCfgDir + File.separator + path;
+            String resourceName = DEFAULT_CFG_DIR + File.separator + path;
             InputStream stream = getClass().getClassLoader().getResourceAsStream(resourceName);
             byte[] bytes = stream.readAllBytes();
             return Optional.of(new String(bytes));

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGExporterConfig.java

@@ -1,23 +0,0 @@
-/*
- * Copyright (c) 2022 nosqlbench
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.nosqlbench.cqlgen.core;
-
-public class CGExporterConfig {
-    public CGExporterConfig(String[] args) {
-
-    }
-}

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGTextTransformers.java

@@ -31,7 +31,7 @@ import java.util.function.Consumer;
 import java.util.function.Supplier;
 
 public class CGTextTransformers implements Consumer<List<Map<String, ?>>>, Supplier<List<CGTextTransformer>> {
-    private final static Logger logger = LogManager.getLogger(CGTextTransformers.class);
+    private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.APPNAME+"/text-transformers");
     private final List<CGTextTransformer> transformers = new ArrayList<>();
 
     @Override

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java

@@ -56,7 +56,9 @@ import java.util.stream.Collectors;
  */
 @Service(value = BundledApp.class, selector = "cqlgen")
 public class CGWorkloadExporter implements BundledApp {
-    private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.class);
+    public static final String APPNAME = "cqlgen";
+    private final static Logger logger = LogManager.getLogger(APPNAME);
+
     private CGColumnRebinder binder;
     private NamingFolio namer;
     private CqlModel model;
@@ -83,15 +85,12 @@ public class CGWorkloadExporter implements BundledApp {
         "update", 10.0
     ));
 
-    private CGExporterConfig config;
-
     public static void main(String[] args) {
         new CGWorkloadExporter().applyAsInt(args);
     }
 
     @Override
     public int applyAsInt(String[] args) {
-        this.config = new CGExporterConfig(args);
 
         logger.info("running CQL workload exporter with args:" + Arrays.toString(args));
 
@@ -182,8 +181,12 @@ public class CGWorkloadExporter implements BundledApp {
                 workload,
                 StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING
             );
+            logger.info("Wrote workload template as '" + target + "'. Bear in mind that this is simply one version " +
+                "of a workload using this schema, and may not be representative of actual production usage patterns.");
         } catch (IOException e) {
-            e.printStackTrace();
+            String errmsg = "There was an error writing '" + target + "'.";
+            logger.error(errmsg);
+            throw new RuntimeException(errmsg);
         }
 
         return 0;

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModel.java

@@ -18,6 +18,7 @@ package io.nosqlbench.cqlgen.model;
 
 import io.nosqlbench.cqlgen.core.CGKeyspaceStats;
 import io.nosqlbench.cqlgen.core.CGSchemaStats;
+import io.nosqlbench.cqlgen.core.CGWorkloadExporter;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -46,7 +47,7 @@ import java.util.function.Supplier;
  * </p>
  */
 public class CqlModel {
-    private final static Logger logger = LogManager.getLogger(CqlModel.class);
+    private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.APPNAME+"/model");
 
     private final Supplier<List<String>> errors;
     private final List<CqlKeyspaceDef> keyspaceDefs = new ArrayList();

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModelBuilder.java

@@ -17,6 +17,7 @@
 package io.nosqlbench.cqlgen.model;
 
 
+import io.nosqlbench.cqlgen.core.CGWorkloadExporter;
 import io.nosqlbench.cqlgen.generated.CqlParser;
 import io.nosqlbench.cqlgen.generated.CqlParserBaseListener;
 import org.antlr.v4.runtime.ParserRuleContext;
@@ -30,7 +31,7 @@ import java.util.List;
 import java.util.stream.IntStream;
 
 public class CqlModelBuilder extends CqlParserBaseListener {
-    private final static Logger logger = LogManager.getLogger(CqlModelBuilder.class);
+    private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.APPNAME+"/buidler");
 
     private final CGErrorListener errorListener;
     private final CqlModel model;

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/parser/CqlModelParser.java

@@ -16,6 +16,7 @@
 
 package io.nosqlbench.cqlgen.parser;
 
+import io.nosqlbench.cqlgen.core.CGWorkloadExporter;
 import io.nosqlbench.cqlgen.generated.CqlLexer;
 import io.nosqlbench.cqlgen.generated.CqlParser;
 import io.nosqlbench.cqlgen.model.CGErrorListener;
@@ -35,7 +36,7 @@ import java.util.List;
 
 
 public class CqlModelParser {
-    private final static Logger logger  = LogManager.getLogger(CqlModelParser.class);
+    private final static Logger logger  = LogManager.getLogger(CGWorkloadExporter.APPNAME+"/parser");
 
 
     public static CqlModel parse(Path path) {

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGGenStatsInjector.java

@@ -19,6 +19,7 @@ package io.nosqlbench.cqlgen.transformers;
 import io.nosqlbench.cqlgen.api.CGModelTransformer;
 import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
 import io.nosqlbench.cqlgen.core.CGSchemaStats;
+import io.nosqlbench.cqlgen.core.CGWorkloadExporter;
 import io.nosqlbench.cqlgen.model.CqlModel;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -30,7 +31,7 @@ import java.util.Locale;
 import java.util.Map;
 
 public class CGGenStatsInjector implements CGModelTransformer, CGTransformerConfigurable {
-    private final static Logger logger = LogManager.getLogger(CGGenStatsInjector.class);
+    private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.APPNAME+"/stats-injector");
 
     private CGSchemaStats schemaStats = null;
     private String name;
@@ -58,9 +59,10 @@ public class CGGenStatsInjector implements CGModelTransformer, CGTransformerConf
             String histogramPath = config.get("path").toString();
             if (histogramPath != null) {
                 if (!Files.exists(Path.of(histogramPath))) {
-                    logger.error("Unable to load tablestats file from '" + histogramPath + "' because it doesn't exists.");
+                    logger.info("No tablestats file was found. at '" + histogramPath + "'.");
                     Object onmissing = config.get("onmissing");
                     if (onmissing==null || !String.valueOf(onmissing).toLowerCase(Locale.ROOT).equals("skip")) {
+                        logger.error("Unable to load tablestats file from '" + histogramPath + "' because it doesn't exists, and onmissing!=skip.");
                         throw new RuntimeException("Unable to continue. onmissing=" + onmissing.toString());
                     } else {
                         return;

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeyspaceFilter.java

@@ -18,6 +18,7 @@ package io.nosqlbench.cqlgen.transformers;
 
 import io.nosqlbench.cqlgen.api.CGModelTransformer;
 import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
+import io.nosqlbench.cqlgen.core.CGWorkloadExporter;
 import io.nosqlbench.cqlgen.model.CqlKeyspaceDef;
 import io.nosqlbench.cqlgen.model.CqlModel;
 import org.apache.logging.log4j.LogManager;
@@ -31,7 +32,7 @@ import java.util.regex.Pattern;
 
 public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfigurable {
 
-    private final static Logger logger = LogManager.getLogger(CGKeyspaceFilter.class);
+    private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.APPNAME+"/keyspace-filter");
     private List<TriStateFilter> patterns;
     private String name;
 

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGModelTransformers.java

@@ -20,6 +20,7 @@ import io.nosqlbench.api.config.standard.NBConfigurable;
 import io.nosqlbench.api.config.standard.NBConfiguration;
 import io.nosqlbench.cqlgen.api.CGModelTransformer;
 import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
+import io.nosqlbench.cqlgen.core.CGWorkloadExporter;
 import io.nosqlbench.cqlgen.model.CqlModel;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -38,7 +39,8 @@ public class CGModelTransformers implements
     Consumer<List<Map<String, ?>>>,
     Supplier<List<CGModelTransformer>>,
     Function<CqlModel,CqlModel> {
-    private final static Logger logger = LogManager.getLogger(CGModelTransformers.class);
+
+    private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.APPNAME+"/transformers");
     private final List<CGModelTransformer> transformers = new ArrayList<>();
 
     public CGModelTransformers() {

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGNameObfuscator.java

@@ -24,6 +24,7 @@ package io.nosqlbench.cqlgen.transformers;
 
 import io.nosqlbench.api.config.standard.*;
 import io.nosqlbench.cqlgen.api.CGModelTransformer;
+import io.nosqlbench.cqlgen.core.CGWorkloadExporter;
 import io.nosqlbench.cqlgen.model.*;
 import io.nosqlbench.cqlgen.transformers.namecache.*;
 import io.nosqlbench.virtdata.core.bindings.DataMapper;
@@ -35,7 +36,7 @@ import java.nio.file.Path;
 import java.util.Map;
 
 public class CGNameObfuscator implements CGModelTransformer, NBConfigurable {
-    private final static Logger logger = LogManager.getLogger(CGNameObfuscator.class);
+    private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.APPNAME+"/name-obfuscator");
 
     NameCache cache = new NameCache();
     private final CGCachingNameRemapper remapper = new CGCachingNameRemapper();

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRegexReplacer.java

@@ -18,6 +18,7 @@ package io.nosqlbench.cqlgen.transformers;
 
 import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
 import io.nosqlbench.cqlgen.api.CGTextTransformer;
+import io.nosqlbench.cqlgen.core.CGWorkloadExporter;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -32,7 +33,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 public class CGRegexReplacer implements CGTextTransformer, CGTransformerConfigurable {
-    private final static Logger logger = LogManager.getLogger(CGRegexReplacer.class);
+    private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.APPNAME+"/replacer");
     private List<Replacer> replacers;
     private String prefix = "";
     private String suffix = "";

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGUdtReplacer.java

@@ -17,25 +17,45 @@
 package io.nosqlbench.cqlgen.transformers;
 
 import io.nosqlbench.cqlgen.api.CGModelTransformer;
+import io.nosqlbench.cqlgen.core.CGWorkloadExporter;
 import io.nosqlbench.cqlgen.model.CqlColumnBase;
 import io.nosqlbench.cqlgen.model.CqlModel;
 import io.nosqlbench.cqlgen.model.CqlTable;
+import io.nosqlbench.cqlgen.model.CqlType;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 
 public class CGUdtReplacer implements CGModelTransformer {
-
+    private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.APPNAME+"/udt-replacer");
     private String name;
 
     @Override
     public CqlModel apply(CqlModel model) {
-        List<String> toReplace = model.getTypeDefs().stream().map(t -> t.getKeyspace().getName() + "." + t.getName()).toList();
+        List<String> toReplace = new ArrayList<>();
+
+        model.getTypeDefs().stream()
+            .map(t -> t.getKeyspace().getName() + "." + t.getName())
+            .forEach(toReplace::add);
+
+        model.getTypeDefs().stream()
+            .map(CqlType::getName)
+            .forEach(toReplace::add);
+
         for (CqlTable table : model.getTableDefs()) {
             for (CqlColumnBase coldef : table.getColumnDefs()) {
                 String typedef = coldef.getTrimmedTypedef();
                 for (String searchFor : toReplace) {
-                    if (typedef.contains(searchFor)) {
+                    String[] words = typedef.split("\\W+");
+                    for (String word : words) {
+                        if (word.toLowerCase(Locale.ROOT).equals(searchFor.toLowerCase(Locale.ROOT))) {
+                            logger.info("replacing '" + typedef + "' with blob");
                             coldef.setTypeDef("blob");
+                            break;
+                        }
                     }
                 }
             }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/UnusedTableRemover.java

@@ -18,6 +18,7 @@ package io.nosqlbench.cqlgen.transformers;
 
 import io.nosqlbench.cqlgen.api.CGModelTransformer;
 import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
+import io.nosqlbench.cqlgen.core.CGWorkloadExporter;
 import io.nosqlbench.cqlgen.model.CqlModel;
 import io.nosqlbench.cqlgen.model.CqlTable;
 import org.apache.logging.log4j.LogManager;
@@ -27,7 +28,7 @@ import java.util.List;
 import java.util.Map;
 
 public class UnusedTableRemover implements CGModelTransformer, CGTransformerConfigurable {
-    private final static Logger logger = LogManager.getLogger(UnusedTableRemover.class);
+    private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.APPNAME+"/unused-table-remover");
     private double minimumThreshold = 0.0001;
     private String name;
 

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/namecache/NameCache.java

@@ -18,6 +18,7 @@ package io.nosqlbench.cqlgen.transformers.namecache;
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import io.nosqlbench.cqlgen.core.CGWorkloadExporter;
 import io.nosqlbench.cqlgen.model.CqlKeyspaceDef;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -32,7 +33,7 @@ import java.util.Map;
 import java.util.function.Function;
 
 public class NameCache {
-    private final static Logger logger = LogManager.getLogger(NameCache.class);
+    private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.APPNAME+"/namecache");
 
     private String path;
     private final Map<String, NamedKeyspace> keyspaces = new LinkedHashMap();

adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/core/MongoDBOpTypes.java

@@ -0,0 +1,2 @@
+package io.nosqlbench.adapter.mongodb.core;public class MongoDBOpTypes {
+}

adapter-mongodb/src/main/java/io/nosqlbench/adapter/mongodb/dispensers/MongoDbUpdateOpDispenser.java

@@ -14,25 +14,16 @@
  * limitations under the License.
  */
 
-package io.nosqlbench.engine.api.activityimpl;
+package io.nosqlbench.adapter.mongodb.dispensers;
 
-import io.nosqlbench.engine.api.activityimpl.uniform.DriverAdapter;
+import io.nosqlbench.adapter.mongodb.core.MongoSpace;
+import io.nosqlbench.engine.api.activityimpl.OpDispenser;
 import io.nosqlbench.engine.api.activityimpl.uniform.flowtypes.Op;
 import io.nosqlbench.engine.api.templating.ParsedOp;
 
 import java.util.function.LongFunction;
 
-public class StandardOpDispenser<T extends Op> extends BaseOpDispenser<T>{
+public class MongoDbUpdateOpDispenser implements OpDispenser<? extends Op> {
-
+    public MongoDbUpdateOpDispenser(LongFunction<MongoSpace> ctxFunc, ParsedOp op) {
-    private final LongFunction<T> opfunc;
-
-    public StandardOpDispenser(DriverAdapter adapter, ParsedOp op, LongFunction<T> opfunc) {
-        super(adapter, op);
-        this.opfunc = opfunc;
-    }
-
-    @Override
-    public T apply(long value) {
-        return opfunc.apply(value);
     }
 }