cqlgen fixes

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java

@@ -18,6 +18,8 @@ package io.nosqlbench.cqlgen.core;
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import io.nosqlbench.api.content.Content;
+import io.nosqlbench.api.content.NBIO;
 import io.nosqlbench.api.spi.BundledApp;
 import io.nosqlbench.cqlgen.binders.Binding;
 import io.nosqlbench.cqlgen.binders.BindingsAccumulator;
@@ -52,7 +54,7 @@ import java.util.stream.Collectors;
  *
  * @see <a href="https://cassandra.apache.org/doc/trunk/cassandra/cql/index.html">Apache Cassandra CQL Docs</a>
  */
-@Service(value= BundledApp.class,selector = "cqlgen")
+@Service(value = BundledApp.class, selector = "cqlgen")
 public class CGWorkloadExporter implements BundledApp {
     private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.class);
     private CGColumnRebinder binder;
@@ -122,67 +124,63 @@ public class CGWorkloadExporter implements BundledApp {
         }
 
         Yaml yaml = new Yaml();
-        Path cfgpath = Path.of("cqlgen.conf");
-
         CGWorkloadExporter exporter;
 
-        if (Files.exists(cfgpath)) {
-            try {
-                CGModelTransformers modelTransformers = new CGModelTransformers();
-                CGTextTransformers textTransformers = new CGTextTransformers();
-                String configfile = Files.readString(cfgpath);
-                Map cfgmap = yaml.loadAs(configfile, Map.class);
-
-                if (cfgmap.containsKey("model_transformers")) {
-                    modelTransformers.accept((List<Map<String, ?>>) cfgmap.get("model_transformers"));
-                }
-
-                Object txtr = cfgmap.get("text_transformers");
-                if (txtr != null) {
-                    textTransformers.accept((List<Map<String, ?>>) cfgmap.get("text_transformers"));
-                }
-
-                String ddl;
-                try {
-                    ddl = Files.readString(srcpath);
-                    logger.info("read " + ddl.length() + " character DDL file, parsing");
-                    if (textTransformers != null) {
-                        ddl = textTransformers.process(ddl);
-                    }
-                } catch (IOException e) {
-                    throw new RuntimeException(e);
-                }
-
-
-                String defaultNamingTemplate = cfgmap.get("naming_template").toString();
-                setNamingTemplate(defaultNamingTemplate);
-
-                String partition_multipler = cfgmap.get("partition_multiplier").toString();
-                setPartitionMultiplier(Double.parseDouble(partition_multipler));
-
-                configureTimeouts(cfgmap.get("timeouts"));
-
-                configureBlocks(cfgmap.get("blockplan"));
-                configureQuantizerDigits(cfgmap.get("quantizer_digits"));
-
-                this.model = CqlModelParser.parse(ddl, srcpath);
-
-                String workload = getWorkloadAsYaml();
-                try {
-                    Files.writeString(
-                        target,
-                        workload,
-                        StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING
-                    );
-                } catch (IOException e) {
-                    e.printStackTrace();
-                }
-
-
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
+        Content<?> cqlgencfg = NBIO.local().prefix("cqlgen").name("cqlgen").extension("conf").one();
+        if (cqlgencfg == null) {
+            throw new RuntimeException("Unable to load cqlgen.conf");
         }
+        Map cfgmap = yaml.loadAs(cqlgencfg.getInputStream(), Map.class);
+
+        CGModelTransformers modelTransformers = new CGModelTransformers();
+        CGTextTransformers textTransformers = new CGTextTransformers();
+
+        if (cfgmap.containsKey("model_transformers")) {
+            modelTransformers.accept((List<Map<String, ?>>) cfgmap.get("model_transformers"));
+        }
+
+//        Object txtr = cfgmap.get("text_transformers");
+//        if (txtr != null) {
+//            textTransformers.accept((List<Map<String, ?>>) cfgmap.get("text_transformers"));
+//        }
+
+        String ddl;
+        try {
+            ddl = Files.readString(srcpath);
+            logger.info("read " + ddl.length() + " character DDL file, parsing");
+            if (textTransformers != null) {
+                ddl = textTransformers.process(ddl);
+            }
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+
+        String defaultNamingTemplate = cfgmap.get("naming_template").toString();
+        setNamingTemplate(defaultNamingTemplate);
+
+        String partition_multipler = cfgmap.get("partition_multiplier").toString();
+        setPartitionMultiplier(Double.parseDouble(partition_multipler));
+
+        configureTimeouts(cfgmap.get("timeouts"));
+
+        configureBlocks(cfgmap.get("blockplan"));
+        configureQuantizerDigits(cfgmap.get("quantizer_digits"));
+
+        this.model = CqlModelParser.parse(ddl, srcpath);
+        this.model = modelTransformers.apply(this.model);
+
+        String workload = getWorkloadAsYaml();
+        try {
+            Files.writeString(
+                target,
+                workload,
+                StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING
+            );
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+
         return 0;
     }
 
@@ -470,17 +468,18 @@ public class CGWorkloadExporter implements BundledApp {
     }
 
     private int totalRatioFor(CqlTable table) {
-        if (table.getTableAttributes() == null || table.getTableAttributes().size() == 0) {
+        if (table.hasStats()) {
+            return readRatioFor(table) + writeRatioFor(table);
+        } else {
             return 1;
         }
-        return readRatioFor(table) + writeRatioFor(table);
     }
 
     private int readRatioFor(CqlTable table) {
         if (table.getTableAttributes() == null || table.getTableAttributes().size() == 0) {
             return 1;
         }
-        double weighted_reads = Double.parseDouble(table.getTableAttributes().getAttribute("weighted_reads"));
+        double weighted_reads = table.getComputedStats().getWeightedReadsOfTotal();
         return (int) (weighted_reads * DEFAULT_RESOLUTION);
     }
 
@@ -488,7 +487,7 @@ public class CGWorkloadExporter implements BundledApp {
         if (table.getTableAttributes() == null || table.getTableAttributes().size() == 0) {
             return 1;
         }
-        double weighted_writes = Double.parseDouble(table.getTableAttributes().getAttribute("weighted_writes"));
+        double weighted_writes = table.getComputedStats().getWeightedWritesOfTotal();
         return (int) (weighted_writes * DEFAULT_RESOLUTION);
     }
 

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTable.java

@@ -201,4 +201,8 @@ public class CqlTable implements NBNamedElement, Labeled {
     public void setComputedStats(ComputedTableStats stats) {
         this.computedTableStats = stats;
     }
+
+    public boolean hasStats() {
+        return this.computedTableStats!=null;
+    }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGModelTransformers.java

@@ -18,6 +18,7 @@ package io.nosqlbench.cqlgen.transformers;
 
 import io.nosqlbench.cqlgen.api.CGModelTransformer;
 import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
+import io.nosqlbench.cqlgen.model.CqlModel;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -27,9 +28,13 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Consumer;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
-public class CGModelTransformers implements Consumer<List<Map<String, ?>>>, Supplier<List<CGModelTransformer>> {
+public class CGModelTransformers implements
+    Consumer<List<Map<String, ?>>>,
+    Supplier<List<CGModelTransformer>>,
+    Function<CqlModel,CqlModel> {
     private final static Logger logger = LogManager.getLogger(CGModelTransformers.class);
     private final List<CGModelTransformer> transformers = new ArrayList<>();
 
@@ -86,4 +91,12 @@ public class CGModelTransformers implements Consumer<List<Map<String, ?>>>, Supp
     public List<CGModelTransformer> get() {
         return this.transformers;
     }
+
+    @Override
+    public CqlModel apply(CqlModel cqlModel) {
+        for (CGModelTransformer transformer : transformers) {
+            cqlModel=transformer.apply(cqlModel);
+        }
+        return cqlModel;
+    }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/ComputedTableStats.java

@@ -57,4 +57,12 @@ public class ComputedTableStats {
     public double getOpShareOfTotalOps() {
         return opShareOfTotalOps;
     }
+
+    public double getWeightedReadsOfTotal() {
+        return this.readShareOfTotalOps;
+    }
+
+    public double getWeightedWritesOfTotal() {
+        return this.writeShareOfTotalWrites;
+    }
 }

adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf

@@ -44,10 +44,6 @@ model_transformers:
 #  # Removes Keyspace DDL statements
 #  - class: CGKeySpaceDDLRemover
 
-#  This is now a generator behavior that is done automatically
-#  # Adds IF NOT EXIST to all DDL
-#  - class: CGIfNotExistsInjector
-
   # Replaces UDTs with blobs until we have full UDT generation capability
   - class: CGUdtReplacer
 
@@ -125,7 +121,7 @@ blockplan:
   drop-keyspaces: drop-keyspaces
   # not needed when tags=block:'drop.*'
   # drop: drop-types, drop-tables, drop-keyspaces
-  rampup: insert
+  rampup: insert-seq
   main-insert: insert-seq
   main-select: select-seq
   main-scan: scan-10-seq

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java

@@ -181,6 +181,7 @@ public class NBCLI implements Function<String[], Integer> {
             if (app!=null) {
                 String[] appargs = Arrays.copyOfRange(args, 1, args.length);
                 logger.info("invoking bundled app '" + args[0] + "' (" + app.getClass().getSimpleName() + ").");
+                globalOptions.setWantsStackTraces(true);
                 int result = app.appMain(appargs);
                 return result;
             }