IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

cqlgen fixes

Browse Source
This commit is contained in:
Jonathan Shook 2022-07-26 02:12:07 -05:00
parent ab03cfc54d
commit f3c0d001aa
6 changed files with 90 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java
View File

@ -18,6 +18,8 @@ package io.nosqlbench.cqlgen.core;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import io.nosqlbench.api.spi.BundledApp;
import io.nosqlbench.cqlgen.binders.Binding;
import io.nosqlbench.cqlgen.binders.BindingsAccumulator;
@ -52,7 +54,7 @@ import java.util.stream.Collectors;
*
* @see <a href="https://cassandra.apache.org/doc/trunk/cassandra/cql/index.html">Apache Cassandra CQL Docs</a>
*/
@Service(value= BundledApp.class,selector = "cqlgen")
@Service(value = BundledApp.class, selector = "cqlgen")
public class CGWorkloadExporter implements BundledApp {
private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.class);
private CGColumnRebinder binder;
@ -122,67 +124,63 @@ public class CGWorkloadExporter implements BundledApp {
}
Yaml yaml = new Yaml();
Path cfgpath = Path.of("cqlgen.conf");
CGWorkloadExporter exporter;
if (Files.exists(cfgpath)) {
try {
CGModelTransformers modelTransformers = new CGModelTransformers();
CGTextTransformers textTransformers = new CGTextTransformers();
String configfile = Files.readString(cfgpath);
Map cfgmap = yaml.loadAs(configfile, Map.class);
if (cfgmap.containsKey("model_transformers")) {
modelTransformers.accept((List<Map<String, ?>>) cfgmap.get("model_transformers"));
}
Object txtr = cfgmap.get("text_transformers");
if (txtr != null) {
textTransformers.accept((List<Map<String, ?>>) cfgmap.get("text_transformers"));
}
String ddl;
try {
ddl = Files.readString(srcpath);
logger.info("read " + ddl.length() + " character DDL file, parsing");
if (textTransformers != null) {
ddl = textTransformers.process(ddl);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
String defaultNamingTemplate = cfgmap.get("naming_template").toString();
setNamingTemplate(defaultNamingTemplate);
String partition_multipler = cfgmap.get("partition_multiplier").toString();
setPartitionMultiplier(Double.parseDouble(partition_multipler));
configureTimeouts(cfgmap.get("timeouts"));
configureBlocks(cfgmap.get("blockplan"));
configureQuantizerDigits(cfgmap.get("quantizer_digits"));
this.model = CqlModelParser.parse(ddl, srcpath);
String workload = getWorkloadAsYaml();
try {
Files.writeString(
target,
workload,
StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING
);
} catch (IOException e) {
e.printStackTrace();
}
} catch (IOException e) {
throw new RuntimeException(e);
}
Content<?> cqlgencfg = NBIO.local().prefix("cqlgen").name("cqlgen").extension("conf").one();
if (cqlgencfg == null) {
throw new RuntimeException("Unable to load cqlgen.conf");
}
Map cfgmap = yaml.loadAs(cqlgencfg.getInputStream(), Map.class);
CGModelTransformers modelTransformers = new CGModelTransformers();
CGTextTransformers textTransformers = new CGTextTransformers();
if (cfgmap.containsKey("model_transformers")) {
modelTransformers.accept((List<Map<String, ?>>) cfgmap.get("model_transformers"));
}
// Object txtr = cfgmap.get("text_transformers");
// if (txtr != null) {
// textTransformers.accept((List<Map<String, ?>>) cfgmap.get("text_transformers"));
// }
String ddl;
try {
ddl = Files.readString(srcpath);
logger.info("read " + ddl.length() + " character DDL file, parsing");
if (textTransformers != null) {
ddl = textTransformers.process(ddl);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
String defaultNamingTemplate = cfgmap.get("naming_template").toString();
setNamingTemplate(defaultNamingTemplate);
String partition_multipler = cfgmap.get("partition_multiplier").toString();
setPartitionMultiplier(Double.parseDouble(partition_multipler));
configureTimeouts(cfgmap.get("timeouts"));
configureBlocks(cfgmap.get("blockplan"));
configureQuantizerDigits(cfgmap.get("quantizer_digits"));
this.model = CqlModelParser.parse(ddl, srcpath);
this.model = modelTransformers.apply(this.model);
String workload = getWorkloadAsYaml();
try {
Files.writeString(
target,
workload,
StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING
);
} catch (IOException e) {
e.printStackTrace();
}
return 0;
}
@ -470,17 +468,18 @@ public class CGWorkloadExporter implements BundledApp {
}
private int totalRatioFor(CqlTable table) {
if (table.getTableAttributes() == null || table.getTableAttributes().size() == 0) {
if (table.hasStats()) {
return readRatioFor(table) + writeRatioFor(table);
} else {
return 1;
}
return readRatioFor(table) + writeRatioFor(table);
}
private int readRatioFor(CqlTable table) {
if (table.getTableAttributes() == null || table.getTableAttributes().size() == 0) {
return 1;
}
double weighted_reads = Double.parseDouble(table.getTableAttributes().getAttribute("weighted_reads"));
double weighted_reads = table.getComputedStats().getWeightedReadsOfTotal();
return (int) (weighted_reads * DEFAULT_RESOLUTION);
}
@ -488,7 +487,7 @@ public class CGWorkloadExporter implements BundledApp {
if (table.getTableAttributes() == null || table.getTableAttributes().size() == 0) {
return 1;
}
double weighted_writes = Double.parseDouble(table.getTableAttributes().getAttribute("weighted_writes"));
double weighted_writes = table.getComputedStats().getWeightedWritesOfTotal();
return (int) (weighted_writes * DEFAULT_RESOLUTION);
}

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTable.java
View File

@ -201,4 +201,8 @@ public class CqlTable implements NBNamedElement, Labeled {
public void setComputedStats(ComputedTableStats stats) {
this.computedTableStats = stats;
}
public boolean hasStats() {
return this.computedTableStats!=null;
}
}

15
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGModelTransformers.java
View File

@ -18,6 +18,7 @@ package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
import io.nosqlbench.cqlgen.model.CqlModel;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -27,9 +28,13 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
public class CGModelTransformers implements Consumer<List<Map<String, ?>>>, Supplier<List<CGModelTransformer>> {
public class CGModelTransformers implements
Consumer<List<Map<String, ?>>>,
Supplier<List<CGModelTransformer>>,
Function<CqlModel,CqlModel> {
private final static Logger logger = LogManager.getLogger(CGModelTransformers.class);
private final List<CGModelTransformer> transformers = new ArrayList<>();
@ -86,4 +91,12 @@ public class CGModelTransformers implements Consumer<List<Map<String, ?>>>, Supp
public List<CGModelTransformer> get() {
return this.transformers;
}
@Override
public CqlModel apply(CqlModel cqlModel) {
for (CGModelTransformer transformer : transformers) {
cqlModel=transformer.apply(cqlModel);
}
return cqlModel;
}
}

8
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/ComputedTableStats.java
View File

@ -57,4 +57,12 @@ public class ComputedTableStats {
public double getOpShareOfTotalOps() {
return opShareOfTotalOps;
}
public double getWeightedReadsOfTotal() {
return this.readShareOfTotalOps;
}
public double getWeightedWritesOfTotal() {
return this.writeShareOfTotalWrites;
}
}

6
adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf
View File

@ -44,10 +44,6 @@ model_transformers:
# # Removes Keyspace DDL statements
# - class: CGKeySpaceDDLRemover
# This is now a generator behavior that is done automatically
# # Adds IF NOT EXIST to all DDL
# - class: CGIfNotExistsInjector
# Replaces UDTs with blobs until we have full UDT generation capability
- class: CGUdtReplacer
@ -125,7 +121,7 @@ blockplan:
drop-keyspaces: drop-keyspaces
# not needed when tags=block:'drop.*'
# drop: drop-types, drop-tables, drop-keyspaces
rampup: insert
rampup: insert-seq
main-insert: insert-seq
main-select: select-seq
main-scan: scan-10-seq

1
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -181,6 +181,7 @@ public class NBCLI implements Function<String[], Integer> {
if (app!=null) {
String[] appargs = Arrays.copyOfRange(args, 1, args.length);
logger.info("invoking bundled app '" + args[0] + "' (" + app.getClass().getSimpleName() + ").");
globalOptions.setWantsStackTraces(true);
int result = app.appMain(appargs);
return result;
}