cqlgen fixes

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java

@@ -18,6 +18,8 @@ package io.nosqlbench.cqlgen.core;
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import io.nosqlbench.api.content.Content;
+import io.nosqlbench.api.content.NBIO;
 import io.nosqlbench.api.spi.BundledApp;
 import io.nosqlbench.cqlgen.binders.Binding;
 import io.nosqlbench.cqlgen.binders.BindingsAccumulator;
@@ -122,25 +124,25 @@ public class CGWorkloadExporter implements BundledApp {
         }
 
         Yaml yaml = new Yaml();
-        Path cfgpath = Path.of("cqlgen.conf");
-
         CGWorkloadExporter exporter;
 
-        if (Files.exists(cfgpath)) {
+        Content<?> cqlgencfg = NBIO.local().prefix("cqlgen").name("cqlgen").extension("conf").one();
-            try {
+        if (cqlgencfg == null) {
+            throw new RuntimeException("Unable to load cqlgen.conf");
+        }
+        Map cfgmap = yaml.loadAs(cqlgencfg.getInputStream(), Map.class);
+
         CGModelTransformers modelTransformers = new CGModelTransformers();
         CGTextTransformers textTransformers = new CGTextTransformers();
-                String configfile = Files.readString(cfgpath);
-                Map cfgmap = yaml.loadAs(configfile, Map.class);
 
         if (cfgmap.containsKey("model_transformers")) {
             modelTransformers.accept((List<Map<String, ?>>) cfgmap.get("model_transformers"));
         }
 
-                Object txtr = cfgmap.get("text_transformers");
+//        Object txtr = cfgmap.get("text_transformers");
-                if (txtr != null) {
+//        if (txtr != null) {
-                    textTransformers.accept((List<Map<String, ?>>) cfgmap.get("text_transformers"));
+//            textTransformers.accept((List<Map<String, ?>>) cfgmap.get("text_transformers"));
-                }
+//        }
 
         String ddl;
         try {
@@ -166,6 +168,7 @@ public class CGWorkloadExporter implements BundledApp {
         configureQuantizerDigits(cfgmap.get("quantizer_digits"));
 
         this.model = CqlModelParser.parse(ddl, srcpath);
+        this.model = modelTransformers.apply(this.model);
 
         String workload = getWorkloadAsYaml();
         try {
@@ -178,11 +181,6 @@ public class CGWorkloadExporter implements BundledApp {
             e.printStackTrace();
         }
 
-
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        }
         return 0;
     }
 
@@ -470,17 +468,18 @@ public class CGWorkloadExporter implements BundledApp {
     }
 
     private int totalRatioFor(CqlTable table) {
-        if (table.getTableAttributes() == null || table.getTableAttributes().size() == 0) {
+        if (table.hasStats()) {
+            return readRatioFor(table) + writeRatioFor(table);
+        } else {
             return 1;
         }
-        return readRatioFor(table) + writeRatioFor(table);
     }
 
     private int readRatioFor(CqlTable table) {
         if (table.getTableAttributes() == null || table.getTableAttributes().size() == 0) {
             return 1;
         }
-        double weighted_reads = Double.parseDouble(table.getTableAttributes().getAttribute("weighted_reads"));
+        double weighted_reads = table.getComputedStats().getWeightedReadsOfTotal();
         return (int) (weighted_reads * DEFAULT_RESOLUTION);
     }
 
@@ -488,7 +487,7 @@ public class CGWorkloadExporter implements BundledApp {
         if (table.getTableAttributes() == null || table.getTableAttributes().size() == 0) {
             return 1;
         }
-        double weighted_writes = Double.parseDouble(table.getTableAttributes().getAttribute("weighted_writes"));
+        double weighted_writes = table.getComputedStats().getWeightedWritesOfTotal();
         return (int) (weighted_writes * DEFAULT_RESOLUTION);
     }
 

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTable.java

@@ -201,4 +201,8 @@ public class CqlTable implements NBNamedElement, Labeled {
     public void setComputedStats(ComputedTableStats stats) {
         this.computedTableStats = stats;
     }
+
+    public boolean hasStats() {
+        return this.computedTableStats!=null;
+    }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGModelTransformers.java

@@ -18,6 +18,7 @@ package io.nosqlbench.cqlgen.transformers;
 
 import io.nosqlbench.cqlgen.api.CGModelTransformer;
 import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
+import io.nosqlbench.cqlgen.model.CqlModel;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -27,9 +28,13 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Consumer;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
-public class CGModelTransformers implements Consumer<List<Map<String, ?>>>, Supplier<List<CGModelTransformer>> {
+public class CGModelTransformers implements
+    Consumer<List<Map<String, ?>>>,
+    Supplier<List<CGModelTransformer>>,
+    Function<CqlModel,CqlModel> {
     private final static Logger logger = LogManager.getLogger(CGModelTransformers.class);
     private final List<CGModelTransformer> transformers = new ArrayList<>();
 
@@ -86,4 +91,12 @@ public class CGModelTransformers implements Consumer<List<Map<String, ?>>>, Supp
     public List<CGModelTransformer> get() {
         return this.transformers;
     }
+
+    @Override
+    public CqlModel apply(CqlModel cqlModel) {
+        for (CGModelTransformer transformer : transformers) {
+            cqlModel=transformer.apply(cqlModel);
+        }
+        return cqlModel;
+    }
 }

adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/ComputedTableStats.java

@@ -57,4 +57,12 @@ public class ComputedTableStats {
     public double getOpShareOfTotalOps() {
         return opShareOfTotalOps;
     }
+
+    public double getWeightedReadsOfTotal() {
+        return this.readShareOfTotalOps;
+    }
+
+    public double getWeightedWritesOfTotal() {
+        return this.writeShareOfTotalWrites;
+    }
 }

adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf

@@ -44,10 +44,6 @@ model_transformers:
 #  # Removes Keyspace DDL statements
 #  - class: CGKeySpaceDDLRemover
 
-#  This is now a generator behavior that is done automatically
-#  # Adds IF NOT EXIST to all DDL
-#  - class: CGIfNotExistsInjector
-
   # Replaces UDTs with blobs until we have full UDT generation capability
   - class: CGUdtReplacer
 
@@ -125,7 +121,7 @@ blockplan:
   drop-keyspaces: drop-keyspaces
   # not needed when tags=block:'drop.*'
   # drop: drop-types, drop-tables, drop-keyspaces
-  rampup: insert
+  rampup: insert-seq
   main-insert: insert-seq
   main-select: select-seq
   main-scan: scan-10-seq

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java

@@ -181,6 +181,7 @@ public class NBCLI implements Function<String[], Integer> {
             if (app!=null) {
                 String[] appargs = Arrays.copyOfRange(args, 1, args.length);
                 logger.info("invoking bundled app '" + args[0] + "' (" + app.getClass().getSimpleName() + ").");
+                globalOptions.setWantsStackTraces(true);
                 int result = app.appMain(appargs);
                 return result;
             }