IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

cqlgen improvements and improved structure

Browse Source
This commit is contained in:
Jonathan Shook
2022-07-26 01:14:58 -05:00
parent 89b7cce900
commit c98d5ffc66
47 changed files with 555 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/BindingsLibrary.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/BindingsLibrary.java
View File

@@ -14,12 +14,17 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.binders;
package io.nosqlbench.cqlgen.api;
import io.nosqlbench.cqlgen.binders.Binding;
import io.nosqlbench.cqlgen.model.CqlColumnDef;
import java.util.Optional;
/**
* A bindings library is simply a service point for a specific way
* to map a column definition to a binding function.
*/
public interface BindingsLibrary {
Optional<Binding> resolveBindingsFor(CqlColumnDef def);
}

7
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGModelTransformer.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGModelTransformer.java
View File

@@ -14,12 +14,17 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.transformers;
package io.nosqlbench.cqlgen.api;
import io.nosqlbench.cqlgen.model.CqlModel;
import java.util.function.Function;
/**
* Most of the functionality of {@link CqlModel} preparation is handled with transformers.
* The type and order of transformers is important, as one transformer may be responsible
* for preparing the model for one or more downstream transformers.
*/
public interface CGModelTransformer extends Function<CqlModel,CqlModel> {
@Override
CqlModel apply(CqlModel model);

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGTextTransformer.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGTextTransformer.java
View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter;
package io.nosqlbench.cqlgen.api;
import java.util.function.Function;

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGTransformerConfigurable.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGTransformerConfigurable.java
View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.transformers;
package io.nosqlbench.cqlgen.api;
import java.util.function.Consumer;

51
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CqlModelInfo.java Normal file
View File

@@ -0,0 +1,51 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.api;
import io.nosqlbench.cqlgen.core.CGSchemaStats;
import io.nosqlbench.cqlgen.model.CqlKeyspace;
import io.nosqlbench.cqlgen.model.CqlTable;
import io.nosqlbench.cqlgen.model.CqlType;
import java.util.List;
import java.util.Map;
import java.util.Set;
public interface CqlModelInfo {
CGSchemaStats getStats();
boolean hasStats();
Map<String, CqlKeyspace> getKeyspacesByName();
List<CqlKeyspace> getKeyspaceDefs();
Map<String, Map<String, CqlTable>> getTableDefsByKeyspaceThenTable();
List<CqlTable> getTablesForKeyspace(String ksname);
List<CqlTable> getTableDefs();
Set<String> getAllKnownKeyspaceNames();
List<CqlType> getTypeDefs();
String getSummaryLine();
Map<String, Map<String, CqlType>> getTypesByKeyspaceThenName();
}

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/Binding.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/Binding.java
View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.binders;
package io.nosqlbench.cqlgen.binders;
public class Binding {
String name;

3
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/BindingsAccumulator.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/BindingsAccumulator.java
View File

@@ -14,8 +14,9 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.binders;
package io.nosqlbench.cqlgen.binders;
import io.nosqlbench.cqlgen.api.BindingsLibrary;
import io.nosqlbench.cqlgen.model.CqlColumnDef;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/NamingFolio.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/NamingFolio.java
View File

@@ -14,12 +14,12 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.binders;
package io.nosqlbench.cqlgen.binders;
import io.nosqlbench.cqlgen.model.CqlColumnDef;
import io.nosqlbench.cqlgen.model.CqlModel;
import io.nosqlbench.cqlgen.model.CqlTable;
import io.nosqlbench.cqlgen.exporter.CGElementNamer;
import io.nosqlbench.cqlgen.core.CGElementNamer;
import io.nosqlbench.api.labels.Labeled;
import java.util.*;

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/NamingStyle.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/NamingStyle.java
View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.binders;
package io.nosqlbench.cqlgen.binders;
public enum NamingStyle {
/**

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/UnresolvedBindingException.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/UnresolvedBindingException.java
View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.binders;
package io.nosqlbench.cqlgen.binders;
import io.nosqlbench.cqlgen.model.CqlColumnDef;

22
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingBuilder.java Normal file
View File

@@ -0,0 +1,22 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.bindspecs;
public interface BindingBuilder {
BindingBuilder prependRaw(String prefunc);
BindingSpec build();
}

41
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingSpec.java Normal file
View File

@@ -0,0 +1,41 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.bindspecs;
import io.nosqlbench.api.labels.Labeled;
public interface BindingSpec {
/**
* Is this binding intended to be for a limited population?
* If so, the value will be the maximum cardinality of values which the binding
* is allowed to produce.
* @return The effective cardinality, which could be {@link Double#POSITIVE_INFINITY}
*/
default double getCardinality() {
return Double.POSITIVE_INFINITY;
}
/**
* The fully qualified name of the entity for which the binding values pertain.
* This is
* @return
*/
Labeled getTarget();
String getTypedef();
}

49
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingSpecImpl.java Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.bindspecs;
import io.nosqlbench.api.labels.Labeled;
public class BindingSpecImpl implements BindingSpec {
private Labeled target;
private double cardinality;
private String typedef;
public BindingSpecImpl(Labeled target) {
this.target = target;
}
@Override
public Labeled getTarget() {
return target;
}
@Override
public String getTypedef() {
return typedef;
}
@Override
public double getCardinality() {
return BindingSpec.super.getCardinality();
}
public void setTarget(Labeled target) {
this.target = target;
}
}

7
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGColumnRebinder.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGColumnRebinder.java
View File

@@ -14,11 +14,11 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter;
package io.nosqlbench.cqlgen.core;
import io.nosqlbench.cqlgen.model.CqlColumnDef;
import io.nosqlbench.cqlgen.exporter.binders.Binding;
import io.nosqlbench.cqlgen.exporter.binders.BindingsAccumulator;
import io.nosqlbench.cqlgen.binders.Binding;
import io.nosqlbench.cqlgen.binders.BindingsAccumulator;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -43,6 +43,7 @@ public class CGColumnRebinder {
}
}
private Binding dividedBinding(CqlColumnDef column) {
CGTableStats stats = column.getTable().getTableAttributes();
if (stats == null) {

6
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGDefaultCqlBindings.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java
View File

@@ -14,11 +14,11 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter;
package io.nosqlbench.cqlgen.core;
import io.nosqlbench.cqlgen.model.CqlColumnDef;
import io.nosqlbench.cqlgen.exporter.binders.Binding;
import io.nosqlbench.cqlgen.exporter.binders.BindingsLibrary;
import io.nosqlbench.cqlgen.binders.Binding;
import io.nosqlbench.cqlgen.api.BindingsLibrary;
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.api.content.Content;

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGElementNamer.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGElementNamer.java
View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter;
package io.nosqlbench.cqlgen.core;
import io.nosqlbench.api.labels.Labeled;

7
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/traverser/CqlDDlDirectoryTraverser.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGExporterConfig.java
View File

@@ -14,11 +14,10 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.traverser;
package io.nosqlbench.cqlgen.core;
import java.nio.file.Path;
public class CGExporterConfig {
public CGExporterConfig(String[] args) {
public class CqlDDlDirectoryTraverser {
public void buildWorkloads(Path sourcePath, Path targetPath) {
}
}

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGKeyspaceStats.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGKeyspaceStats.java
View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter;
package io.nosqlbench.cqlgen.core;
import java.util.HashMap;
import java.util.Map;

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGLiteralFormat.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGLiteralFormat.java
View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter;
package io.nosqlbench.cqlgen.core;
import io.nosqlbench.cqlgen.model.CqlColumnDef;

3
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGSchemaStats.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGSchemaStats.java
View File

@@ -14,12 +14,13 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter;
package io.nosqlbench.cqlgen.core;
import java.util.HashMap;
import java.util.Map;
public class CGSchemaStats {
Map<String, CGKeyspaceStats> keyspaces = new HashMap<String, CGKeyspaceStats>();
public Map<String, CGKeyspaceStats> getKeyspaces() {

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGTableStats.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGTableStats.java
View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter;
package io.nosqlbench.cqlgen.core;
import java.util.HashMap;
import java.util.Map;

7
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGTextTransformers.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGTextTransformers.java
View File

@@ -14,10 +14,11 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter;
package io.nosqlbench.cqlgen.core;
import io.nosqlbench.cqlgen.exporter.transformers.CGNameObfuscator;
import io.nosqlbench.cqlgen.exporter.transformers.CGTransformerConfigurable;
import io.nosqlbench.cqlgen.api.CGTextTransformer;
import io.nosqlbench.cqlgen.transformers.CGNameObfuscator;
import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

99
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGWorkloadExporter.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java
View File

@@ -14,17 +14,19 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter;
package io.nosqlbench.cqlgen.core;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.cqlgen.exporter.binders.BindingsAccumulator;
import io.nosqlbench.cqlgen.exporter.binders.BindingsLibrary;
import io.nosqlbench.cqlgen.exporter.transformers.CGModelTransformers;
import io.nosqlbench.cqlgen.parser.CqlModelParser;
import io.nosqlbench.cqlgen.exporter.binders.Binding;
import io.nosqlbench.cqlgen.exporter.binders.NamingFolio;
import io.nosqlbench.api.spi.BundledApp;
import io.nosqlbench.cqlgen.binders.Binding;
import io.nosqlbench.cqlgen.binders.BindingsAccumulator;
import io.nosqlbench.cqlgen.api.BindingsLibrary;
import io.nosqlbench.cqlgen.binders.NamingFolio;
import io.nosqlbench.cqlgen.transformers.CGModelTransformers;
import io.nosqlbench.cqlgen.model.*;
import io.nosqlbench.cqlgen.parser.CqlModelParser;
import io.nosqlbench.nb.annotations.Service;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.snakeyaml.engine.v2.api.Dump;
@@ -40,7 +42,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
@@ -51,11 +52,12 @@ import java.util.stream.Collectors;
*
* @see <a href="https://cassandra.apache.org/doc/trunk/cassandra/cql/index.html">Apache Cassandra CQL Docs</a>
*/
public class CGWorkloadExporter {
@Service(value= BundledApp.class,selector = "cqlgen")
public class CGWorkloadExporter implements BundledApp {
private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.class);
private CGColumnRebinder binder;
private NamingFolio namer;
private final CqlModel model;
private CqlModel model;
private final int DEFAULT_RESOLUTION = 10000;
@@ -79,38 +81,16 @@ public class CGWorkloadExporter {
"update", 10.0
));
public CGWorkloadExporter(CqlModel model, CGModelTransformers transformers) {
this.model = model;
for (Function<CqlModel, CqlModel> transformer : transformers.get()) {
CqlModel modified = transformer.apply(this.model);
model = modified;
}
}
public CGWorkloadExporter(String ddl, Path srcpath, CGModelTransformers transformers) {
this(CqlModelParser.parse(ddl, srcpath), transformers);
}
public CGWorkloadExporter(String ddl, CGModelTransformers transformers) {
this(ddl, null, transformers);
}
public CGWorkloadExporter(Path path, CGModelTransformers transformers) {
this(CqlModelParser.parse(path), transformers);
}
private String loadFile(Path path) {
try {
String ddl = Files.readString(path);
logger.info("read " + ddl.length() + " character DDL file");
return ddl;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private CGExporterConfig config;
public static void main(String[] args) {
new CGWorkloadExporter().appMain(args);
}
@Override
public int appMain(String[] args) {
this.config = new CGExporterConfig(args);
logger.info("running CQL workload exporter with args:" + Arrays.toString(args));
if (args.length == 0) {
@@ -145,6 +125,7 @@ public class CGWorkloadExporter {
Path cfgpath = Path.of("cqlgen.conf");
CGWorkloadExporter exporter;
if (Files.exists(cfgpath)) {
try {
CGModelTransformers modelTransformers = new CGModelTransformers();
@@ -172,20 +153,21 @@ public class CGWorkloadExporter {
throw new RuntimeException(e);
}
exporter = new CGWorkloadExporter(ddl, srcpath, modelTransformers);
String defaultNamingTemplate = cfgmap.get("naming_template").toString();
exporter.setNamingTemplate(defaultNamingTemplate);
setNamingTemplate(defaultNamingTemplate);
String partition_multipler = cfgmap.get("partition_multiplier").toString();
exporter.setPartitionMultiplier(Double.parseDouble(partition_multipler));
setPartitionMultiplier(Double.parseDouble(partition_multipler));
exporter.configureTimeouts(cfgmap.get("timeouts"));
configureTimeouts(cfgmap.get("timeouts"));
exporter.configureBlocks(cfgmap.get("blockplan"));
exporter.configureQuantizerDigits(cfgmap.get("quantizer_digits"));
configureBlocks(cfgmap.get("blockplan"));
configureQuantizerDigits(cfgmap.get("quantizer_digits"));
String workload = exporter.getWorkloadAsYaml();
this.model = CqlModelParser.parse(ddl, srcpath);
String workload = getWorkloadAsYaml();
try {
Files.writeString(
target,
@@ -201,8 +183,21 @@ public class CGWorkloadExporter {
throw new RuntimeException(e);
}
}
return 0;
}
private String loadFile(Path path) {
try {
String ddl = Files.readString(path);
logger.info("read " + ddl.length() + " character DDL file");
return ddl;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private void configureQuantizerDigits(Object quantizer_digits) {
if (quantizer_digits != null) {
this.quantizerDigits = Integer.parseInt(quantizer_digits.toString());
@@ -237,10 +232,10 @@ public class CGWorkloadExporter {
case "drop-tables" -> genDropTablesBlock(model, blockname);
case "drop-keyspaces" -> genDropKeyspacesOpTemplates(model, blockname);
case "truncate-tables" -> genTruncateTablesOpTemplates(model, blockname);
case "insert" -> genInsertOpTemplates(model, blockname);
case "select" -> genSelectOpTemplates(model, blockname);
case "scan-10" -> genScanOpTemplates(model, blockname);
case "update" -> genUpdateOpTemplates(model, blockname);
case "insert-seq" -> genInsertOpTemplates(model, blockname);
case "select-seq" -> genSelectOpTemplates(model, blockname);
case "scan-10-seq" -> genScanOpTemplates(model, blockname);
case "update-seq" -> genUpdateOpTemplates(model, blockname);
default -> throw new RuntimeException("Unable to create block entries for " + component + ".");
};
block.putAll(additions);
@@ -446,7 +441,7 @@ public class CGWorkloadExporter {
.replaceAll("BINDINGS",
String.join(", ",
table.getColumnDefinitions().stream()
.map(binder::forColumn)
.map(c -> binder.forColumn(c))
.map(c -> "{" + c.getName() + "}").toList()));
}

52
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGIfNotExistsInjector.java
View File

@@ -1,52 +0,0 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.transformers;
import io.nosqlbench.cqlgen.model.CqlModel;
/**
* @deprecated Superseded by direct rendering from AST in generator
*/
public class CGIfNotExistsInjector implements CGModelTransformer {
@Override
public CqlModel apply(CqlModel model) {
// for (CqlKeyspace keyspace : model.getKeyspaceDefs()) {
// keyspace.setRefDdl(keyspace.getRefddl().replaceAll(
// "(?m)(?s)(?i)(\\s*CREATE (TABLE|KEYSPACE|TYPE) +)(?!IF NOT EXISTS)",
// "$1IF NOT EXISTS "
// ));
// }
// for (CqlTable table : model.getTableDefs()) {
// String refddl = table.getRefDdl();
// String replaced = refddl.replaceAll(
// "(?m)(?s)(?i)(\\s*CREATE (TABLE|KEYSPACE|TYPE) +)(?!IF NOT EXISTS)",
// "$1IF NOT EXISTS "
// );
//
// table.setRefDdl(replaced);
// }
// for (CqlType type : model.getTypes()) {
// type.setRefddl(type.getRefDdl().replaceAll(
// "(?m)(?s)(?i)(\\s*CREATE (TABLE|KEYSPACE|TYPE) +)(?!IF NOT EXISTS)",
// "$1IF NOT EXISTS "
// ));
//
// }
return model;
}
}

20
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/ComputedSchemaStats.java Normal file
View File

@@ -0,0 +1,20 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.model;
public class ComputedSchemaStats {
}

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlKeyspace.java
View File

@@ -18,7 +18,7 @@ package io.nosqlbench.cqlgen.model;
import io.nosqlbench.api.config.NBNamedElement;
import io.nosqlbench.api.labels.Labeled;
import io.nosqlbench.cqlgen.exporter.CGKeyspaceStats;
import io.nosqlbench.cqlgen.core.CGKeyspaceStats;
import java.util.Map;

105
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModel.java
View File

@@ -16,9 +16,10 @@
package io.nosqlbench.cqlgen.model;
import io.nosqlbench.cqlgen.exporter.CGKeyspaceStats;
import io.nosqlbench.cqlgen.exporter.CGSchemaStats;
import io.nosqlbench.cqlgen.exporter.CGTableStats;
import io.nosqlbench.cqlgen.api.CqlModelInfo;
import io.nosqlbench.cqlgen.core.CGKeyspaceStats;
import io.nosqlbench.cqlgen.core.CGSchemaStats;
import io.nosqlbench.cqlgen.core.CGTableStats;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -26,30 +27,49 @@ import java.util.*;
import java.util.function.Supplier;
/**
* <p>
* This model contains definition level details for schema elements which are parsed from the
* Antlr4 CQL grammar.
* Because keyspace, table, column, and type elements are handled sometimes in different ways,
* these are stored in separate data structures.
* When you see a *refddl or similar field, this is a copy of the text image from the original
* parsed syntax. These are used for populating schema blocks without doing a full parse.
* If you update either the refddl or the actual AST level elements for any of the types in this
* model, you are required to update the other version along with it, using string substitution
* if necessary.
* Key elements include:
* <UL>
* <LI>keyspace definitions, organized by keyspace name</LI>
* <li>type definitions, organized by keyspace name</li>
* <li>table definitions with included column definitions, organized by keyspace name</li>
* </UL>
* </p>
*
* <p>Because keyspace, table, and type elements are handled sometimes in different ways,
* these are stored in separate data structures, mapped by the logical keyspace name. This means
* that you will see table definitions for named keyspaces even if those named keyspaces are not represented
* in the keyspace definitions. This allows for sub-selecting of rendered elements by logical
* name without requiring a fully-interconnected keyspace->table->column object graph.
* </p>
*/
public class CqlModel {
public class CqlModel implements CqlModelInfo {
private final static Logger logger = LogManager.getLogger(CqlModel.class);
private final Supplier<List<String>> errors;
Map<String, CqlKeyspace> keyspaceDefs = new LinkedHashMap<>();
Map<String, Map<String, CqlTable>> tableDefs = new LinkedHashMap<>();
Map<String, Map<String, CqlType>> typeDefs = new LinkedHashMap<>();
private final Map<String, CqlKeyspace> keyspaceDefs = new LinkedHashMap<>();
private final Map<String, Map<String, CqlTable>> tableDefs = new LinkedHashMap<>();
private final Map<String, Map<String, CqlType>> typeDefs = new LinkedHashMap<>();
CGSchemaStats schemaStats = null;
private CGSchemaStats schemaStats = null;
private ComputedSchemaStats computedSchemaStats;
public CGSchemaStats getKeyspaceAttributes() {
@Override
public CGSchemaStats getStats() {
return schemaStats;
}
@Override
public boolean hasStats() {
return schemaStats!=null;
}
public ComputedSchemaStats getComputedStats() {
return computedSchemaStats;
}
public void setKeyspaceAttributes(CGSchemaStats schemaStats) {
this.schemaStats = schemaStats;
for (String statsKeyspacename : schemaStats.getKeyspaces().keySet()) {
@@ -64,13 +84,13 @@ public class CqlModel {
for (String statsTableName : keyspaceStats.getKeyspaceTables().keySet()) {
CGTableStats tableStats = keyspaceStats.getKeyspaceTables().get(statsTableName);
Map<String, CqlTable> modelTables = tableDefs.get(statsKeyspacename);
if (modelTables!=null) {
if (modelTables != null) {
CqlTable modelTable = modelTables.get(statsTableName);
if (modelTable!=null) {
logger.debug("setting table stats for '" + statsKeyspacename+"."+statsTableName+"'");
if (modelTable != null) {
logger.debug("setting table stats for '" + statsKeyspacename + "." + statsTableName + "'");
modelTable.setTableAttributes(tableStats);
} else {
logger.debug(" skipping table stats for '" + statsKeyspacename + "."+statsTableName+"'");
logger.debug(" skipping table stats for '" + statsKeyspacename + "." + statsTableName + "'");
}
} else {
logger.debug(" SKIPPING stats for all tables in keyspace '" + statsKeyspacename + "'");
@@ -83,9 +103,6 @@ public class CqlModel {
transient CqlTable table;
transient CqlType udt;
public boolean hasStats() {
return schemaStats!=null;
}
public CqlModel(Supplier<List<String>> errorSource) {
this.errors = errorSource;
@@ -99,10 +116,10 @@ public class CqlModel {
keyspace = new CqlKeyspace();
}
public void saveKeyspace(String text,String refddl) {
public void saveKeyspace(String text, String refddl) {
keyspace.setKeyspaceName(text);
this.keyspaceDefs.put(text, keyspace);
keyspace=null;
keyspace = null;
}
public void newTable() {
@@ -112,7 +129,7 @@ public class CqlModel {
public void saveTable(String keyspace, String text) {
table.setKeyspace(keyspace);
table.setName(text);
this.tableDefs.computeIfAbsent(keyspace, ks->new LinkedHashMap<>()).put(text, table);
this.tableDefs.computeIfAbsent(keyspace, ks -> new LinkedHashMap<>()).put(text, table);
table = null;
}
@@ -123,28 +140,33 @@ public class CqlModel {
}
}
@Override
public Map<String, CqlKeyspace> getKeyspacesByName() {
return keyspaceDefs;
}
@Override
public List<CqlKeyspace> getKeyspaceDefs() {
return new ArrayList<>(this.keyspaceDefs.values());
}
@Override
public Map<String, Map<String, CqlTable>> getTableDefsByKeyspaceThenTable() {
return tableDefs;
}
@Override
public List<CqlTable> getTablesForKeyspace(String ksname) {
Map<String, CqlTable> tables = this.tableDefs.get(ksname);
if (tables!=null) {
if (tables != null) {
return new ArrayList<>(tables.values());
}
return List.of();
}
@Override
public List<CqlTable> getTableDefs() {
return tableDefs.values().stream().flatMap(m->m.values().stream()).toList();
return tableDefs.values().stream().flatMap(m -> m.values().stream()).toList();
}
@Override
@@ -155,7 +177,7 @@ public class CqlModel {
sb.append("keyspace '").append(keyspace.getName()).append("':\n");
sb.append(keyspace).append("\n");
tableDefs.getOrDefault(ks,Map.of()).values().stream()
tableDefs.getOrDefault(ks, Map.of()).values().stream()
.forEach(table -> {
sb.append("table '").append(table.getName()).append("':\n");
sb.append(table);
@@ -167,8 +189,10 @@ public class CqlModel {
/**
* Get all the keyspace names which have been referenced in any way, whether or not
* this was in a keyspace definition or some other DDL like table or udt names.
*
* @return A list of all known keyspace names
*/
@Override
public Set<String> getAllKnownKeyspaceNames() {
Set<String> ksnames = new LinkedHashSet<>();
ksnames.addAll(this.keyspaceDefs.keySet());
@@ -196,10 +220,11 @@ public class CqlModel {
udt.setKeyspace(keyspace);
udt.setName(name);
Map<String, CqlType> ksTypes = this.typeDefs.computeIfAbsent(keyspace, ks -> new LinkedHashMap<>());
ksTypes.put(udt.getName(),udt);
udt=null;
ksTypes.put(udt.getName(), udt);
udt = null;
}
@Override
public List<CqlType> getTypeDefs() {
ArrayList<CqlType> list = new ArrayList<>();
for (Map<String, CqlType> cqlTypesByKeyspace : typeDefs.values()) {
@@ -222,8 +247,9 @@ public class CqlModel {
this.typeDefs.remove(name);
}
@Override
public String getSummaryLine() {
return "keyspaces: " + keyspaceDefs.size() + ", tables: " + getTableDefs().size() +
return "keyspaces: " + keyspaceDefs.size() + ", tables: " + getTableDefs().size() +
", columns: " + getTableDefs().stream().mapToInt(t -> t.getColumnDefinitions().size()).sum() +
", types: " + getTypeDefs().size();
}
@@ -236,7 +262,7 @@ public class CqlModel {
}
if (this.tableDefs.containsKey(keyspaceName)) {
Map<String, CqlTable> tablesForKeyspace = this.tableDefs.remove(keyspaceName);
if (tablesForKeyspace!=null) {
if (tablesForKeyspace != null) {
for (CqlTable table : tablesForKeyspace.values()) {
table.setKeyspace(newKeyspaceName);
}
@@ -245,12 +271,12 @@ public class CqlModel {
}
if (this.typeDefs.containsKey(keyspaceName)) {
Map<String, CqlType> typesForKeyspace = this.typeDefs.remove(keyspaceName);
if (typesForKeyspace!=null) {
if (typesForKeyspace != null) {
for (CqlType cqltype : typesForKeyspace.values()) {
cqltype.setKeyspace(newKeyspaceName);
}
}
this.typeDefs.put(newKeyspaceName,typesForKeyspace);
this.typeDefs.put(newKeyspaceName, typesForKeyspace);
}
}
@@ -258,14 +284,14 @@ public class CqlModel {
Map<String, CqlTable> tablesInKs = tableDefs.get(extant.getKeySpace());
CqlTable table = tablesInKs.get(extant.getName());
table.setName(newTableName);
tablesInKs.put(table.getName(),table);
tablesInKs.put(table.getName(), table);
}
public void renameType(String keyspaceName, String typeName, String newTypeName) {
Map<String,CqlType> typesInKeyspace = typeDefs.get(keyspaceName);
Map<String, CqlType> typesInKeyspace = typeDefs.get(keyspaceName);
CqlType cqlType = typesInKeyspace.remove(typeName);
cqlType.setName(newTypeName);
typesInKeyspace.put(newTypeName,cqlType);
typesInKeyspace.put(newTypeName, cqlType);
}
public void setTableCompactStorage(boolean isCompactStorage) {
@@ -280,6 +306,7 @@ public class CqlModel {
keyspace.setReplicationData(repldata);
}
@Override
public Map<String, Map<String, CqlType>> getTypesByKeyspaceThenName() {
return typeDefs;
}
@@ -289,6 +316,6 @@ public class CqlModel {
}
public boolean isEmpty() {
return this.keyspaceDefs.size()==0 && this.tableDefs.size()==0 && this.typeDefs.size()==0;
return this.keyspaceDefs.size() == 0 && this.tableDefs.size() == 0 && this.typeDefs.size() == 0;
}
}

12
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTable.java
View File

@@ -18,7 +18,8 @@ package io.nosqlbench.cqlgen.model;
import io.nosqlbench.api.config.NBNamedElement;
import io.nosqlbench.api.labels.Labeled;
import io.nosqlbench.cqlgen.exporter.CGTableStats;
import io.nosqlbench.cqlgen.core.CGTableStats;
import io.nosqlbench.cqlgen.transformers.ComputedTableStats;
import java.util.*;
import java.util.function.Function;
@@ -33,6 +34,7 @@ public class CqlTable implements NBNamedElement, Labeled {
List<String> clusteringOrders = new ArrayList<>();
List<CqlColumnDef> coldefs = new ArrayList<>();
private boolean compactStorage;
private ComputedTableStats computedTableStats;
public CqlTable() {
}
@@ -191,4 +193,12 @@ public class CqlTable implements NBNamedElement, Labeled {
public boolean isLastClusteringColumn(int position) {
return clustering.length > 0 && position == clustering[clustering.length - 1];
}
public ComputedTableStats getComputedStats() {
return this.computedTableStats;
}
public void setComputedStats(ComputedTableStats stats) {
this.computedTableStats = stats;
}
}

2
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGCachingNameRemapper.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGCachingNameRemapper.java
View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.transformers;
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.api.config.NBNamedElement;
import io.nosqlbench.api.labels.Labeled;

6
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGGenStatsInjector.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGGenStatsInjector.java
View File

@@ -14,9 +14,11 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.transformers;
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.exporter.CGSchemaStats;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
import io.nosqlbench.cqlgen.core.CGSchemaStats;
import io.nosqlbench.cqlgen.model.CqlModel;
import java.io.IOException;

3
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGKeySpaceDDLRemover.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeySpaceDDLRemover.java
View File

@@ -14,8 +14,9 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.transformers;
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.model.CqlKeyspace;
import io.nosqlbench.cqlgen.model.CqlModel;

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGKeyspaceFilter.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeyspaceFilter.java
View File

@@ -14,8 +14,10 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.transformers;
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
import io.nosqlbench.cqlgen.model.CqlModel;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGModelTransformers.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGModelTransformers.java
View File

@@ -14,8 +14,10 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.transformers;
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGNameObfuscator.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGNameObfuscator.java
View File

@@ -20,8 +20,10 @@
* once an element is named, use the same name throughout
* prefix each element type with a code for the type
*/
package io.nosqlbench.cqlgen.exporter.transformers;
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
import io.nosqlbench.cqlgen.model.CqlModel;
import io.nosqlbench.cqlgen.model.CqlTable;
import io.nosqlbench.cqlgen.model.CqlType;

32
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGRatioCalculator.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioCalculator.java
View File

@@ -14,9 +14,10 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.transformers;
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.exporter.CGTableStats;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.core.CGTableStats;
import io.nosqlbench.cqlgen.model.CqlModel;
import io.nosqlbench.cqlgen.model.CqlTable;
@@ -28,29 +29,30 @@ public class CGRatioCalculator implements CGModelTransformer {
// TODO: True this up
return model;
}
double totalReads = 0.0d;
double totalWrites = 0.0d;
double totalSpace = 0.0d;
double totalOps=0.0d;
double totalOps = 0.0d;
for (CqlTable table : model.getTableDefs()) {
CGTableStats tableAttributes = table.getTableAttributes();
if (tableAttributes==null) {
if (tableAttributes == null) {
continue;
}
String local_read_count = tableAttributes.getAttribute("Local read count");
double reads = Double.parseDouble(local_read_count);
totalReads+=reads;
totalOps+=reads;
totalReads += reads;
totalOps += reads;
String local_write_count = table.getTableAttributes().getAttribute("Local write count");
double writes = Double.parseDouble(local_write_count);
totalWrites += writes;
totalOps+=writes;
totalOps += writes;
String space_used_total = table.getTableAttributes().getAttribute("Space used (total)");
double space = Double.parseDouble(space_used_total);
totalSpace+=space;
totalSpace += space;
}
for (CqlTable table : model.getTableDefs()) {
@@ -59,11 +61,17 @@ public class CGRatioCalculator implements CGModelTransformer {
double totalTableReads = reads / totalOps;
double totalTableWrites = writes / totalOps;
table.getTableAttributes().setAttribute("weighted_reads", String.valueOf(totalTableReads));
table.getTableAttributes().setAttribute("weighted_writes", String.valueOf(totalTableWrites));
table.getTableAttributes().setAttribute("weighted_ops", String.valueOf(totalTableReads+totalTableWrites));
double tableSpaceUsed = Double.parseDouble(table.getTableAttributes().getAttribute("Space used (total)"));
table.getTableAttributes().setAttribute("weighted_space", String.valueOf(tableSpaceUsed / totalSpace));
double weighted_space = tableSpaceUsed / totalSpace;
double op_share_of_total_ops = totalTableReads + totalTableWrites;
ComputedTableStats computedTableStats = new ComputedTableStats()
.setReadShareOfTotalOps(reads / totalOps)
.setReadShareOfTotalReads(reads / totalReads)
.setWriteShareOfTotalOps(writes / totalOps)
.setWriteShareOfTotalWrites(writes / totalWrites)
.setOpShareOfTotalOps(op_share_of_total_ops)
.setSpaceUsedOfTotalSpace(weighted_space);
table.setComputedStats(computedTableStats);
}
return model;

73
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioSuffixer.java Normal file
View File

@@ -0,0 +1,73 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.api.config.standard.*;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.core.CGSchemaStats;
import io.nosqlbench.cqlgen.model.CqlModel;
import io.nosqlbench.cqlgen.model.CqlTable;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class CGRatioSuffixer implements CGModelTransformer, NBConfigurable {
private double resolution;
private String format;
@Override
public CqlModel apply(CqlModel model) {
CGSchemaStats schemastats = model.getStats();
if (schemastats == null) {
return model;
}
for (CqlTable tableDef : model.getTableDefs()) {
double opshare = tableDef.getComputedStats().getOpShareOfTotalOps();
String newname = String.format(this.format, tableDef.getName(), opshare);
model.renameTable(tableDef,newname);
}
return model;
}
@Override
public void applyConfig(NBConfiguration cfg) {
this.format = cfg.get("format", String.class);
if (!format.contains("NAME")) {
throw new RuntimeException("format config param for the CGRatioSuffixer must contain 'NAME', but it is '" + format + "'");
}
Pattern pattern = Pattern.compile("%(2\\$)?(?<resolution>\\d+)d");
Matcher matcher = pattern.matcher(format);
if (!matcher.matches()) {
throw new RuntimeException("Could not find the required decimal format specifier for the format config parameter of " + CGRatioSuffixer.class);
}
this.resolution = Double.parseDouble(matcher.group("resolution"));
}
@Override
public NBConfigModel getConfigModel() {
return ConfigModel.of(CGRatioSuffixer.class)
.add(Param.defaultTo("format","%1$s_%2$5d").setDescription(
"The format specifier as in Java String.format, with a required string format for the first arg, and a required decimal format for the second."
))
.asReadOnly();
}
}

5
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGRegexReplacer.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRegexReplacer.java
View File

@@ -14,9 +14,10 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.transformers;
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.exporter.CGTextTransformer;
import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
import io.nosqlbench.cqlgen.api.CGTextTransformer;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGReplicationSettingInjector.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGReplicationSettingInjector.java
View File

@@ -14,8 +14,10 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.transformers;
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
import io.nosqlbench.cqlgen.model.CqlKeyspace;
import io.nosqlbench.cqlgen.model.CqlModel;

3
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGUdtReplacer.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGUdtReplacer.java
View File

@@ -14,8 +14,9 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.transformers;
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.model.CqlColumnDef;
import io.nosqlbench.cqlgen.model.CqlModel;
import io.nosqlbench.cqlgen.model.CqlTable;

60
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/ComputedTableStats.java Normal file
View File

@@ -0,0 +1,60 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.cqlgen.transformers;
public class ComputedTableStats {
private double readShareOfTotalOps;
private double readShareOfTotalReads;
private double writeShareOfTotalOps;
private double writeShareOfTotalWrites;
private double opShareOfTotalOps;
private double spaceUsedOfTotalSpace;
public ComputedTableStats setReadShareOfTotalOps(double v) {
this.readShareOfTotalOps = v;
return this;
}
public ComputedTableStats setReadShareOfTotalReads(double v) {
this.readShareOfTotalReads =v;
return this;
}
public ComputedTableStats setWriteShareOfTotalOps(double v) {
this.writeShareOfTotalOps = v;
return this;
}
public ComputedTableStats setWriteShareOfTotalWrites(double v) {
this.writeShareOfTotalWrites = v;
return this;
}
public ComputedTableStats setOpShareOfTotalOps(double op_share_total) {
this.opShareOfTotalOps = op_share_total;
return this;
}
public ComputedTableStats setSpaceUsedOfTotalSpace(double weightedSpace) {
this.spaceUsedOfTotalSpace = weightedSpace;
return this;
}
public double getOpShareOfTotalOps() {
return opShareOfTotalOps;
}
}

8
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CqlSchemaStatsParser.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CqlSchemaStatsParser.java
View File

@@ -14,11 +14,11 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.transformers;
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.exporter.CGKeyspaceStats;
import io.nosqlbench.cqlgen.exporter.CGSchemaStats;
import io.nosqlbench.cqlgen.exporter.CGTableStats;
import io.nosqlbench.cqlgen.core.CGKeyspaceStats;
import io.nosqlbench.cqlgen.core.CGSchemaStats;
import io.nosqlbench.cqlgen.core.CGTableStats;
import org.apache.commons.math4.util.Pair;
import java.io.BufferedReader;

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/UnusedTableRemover.java → adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/UnusedTableRemover.java
View File

@@ -14,8 +14,10 @@
* limitations under the License.
*/
package io.nosqlbench.cqlgen.exporter.transformers;
package io.nosqlbench.cqlgen.transformers;
import io.nosqlbench.cqlgen.api.CGModelTransformer;
import io.nosqlbench.cqlgen.api.CGTransformerConfigurable;
import io.nosqlbench.cqlgen.model.CqlModel;
import io.nosqlbench.cqlgen.model.CqlTable;
import org.apache.logging.log4j.LogManager;

32
adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf
View File

@@ -8,13 +8,13 @@
# outfile: _replaced.cql
# replacers:
# - - '^(\s*?)\b(options|role|roles|permissions|permission|date|key|timestamp|type|keys)\b(\s+[a-zA-Z][a-zA-Z<>,_ -]*?,?)$'
# - '$1PREFIX$2SUFFIX$3'
# - '$1"$2"$3'
# - - '^(.*?PRIMARY KEY.*?)\b(options|role|roles|permissions|permission|date|key|timestamp|type|keys)\b(.*?)$'
# - '$1PREFIX$2SUFFIX$3'
# - '$1"$2"$3'
# - - '^(.*?CLUSTERING ORDER BY.+?)\b(options|role|roles|permissions|permission|date|key|timestamp|type|keys)\b(.*?)$'
# - '$1PREFIX$2SUFFIX$3'
# - '$1"$2"$3'
# - - '^(\s*?CREATE TABLE.+?)\b(options|role|roles|permissions|permission|date|key|timestamp|type|keys)\b(.*?)$'
# - '$1PREFIX$2SUFFIX$3'
# - '$1"$2"$3'
#
model_transformers:
@@ -27,7 +27,7 @@ model_transformers:
- exclude: dsefs_.*
- exclude: cfs_.*
- exclude: cfs
- exclude: HiveMetaStore
- exclude: Hive.*
- exclude: spark_system
- include: .*
@@ -64,13 +64,13 @@ model_transformers:
# as usual.
- class: CGRatioCalculator
# if this is set, and the fractional rate of operations against a table
# counting reads and writes is less than the percent threshold, then
# the table will be excluded from all op template generation.
- class: UnusedTableRemover
config:
# this is as a fractional number, so 0.1 is the same as 10%
minimum_threshold: 0.001
# # if this is set, and the fractional rate of operations against a table
# # counting reads and writes is less than the percent threshold, then
# # the table will be excluded from all op template generation.
# - class: UnusedTableRemover
# config:
# # this is as a fractional number, so 0.1 is the same as 10%
# minimum_threshold: 0.001
# replaces names of keyspaces, tables, and columns with generated values
- class: CGNameObfuscator
@@ -126,10 +126,10 @@ blockplan:
# not needed when tags=block:'drop.*'
# drop: drop-types, drop-tables, drop-keyspaces
rampup: insert
main-insert: insert
main-select: select
main-scan: scan-10
main-update: update
main-insert: insert-seq
main-select: select-seq
main-scan: scan-10-seq
main-update: update-seq
# not needed when tags=block:'main.*'
# main: insert, select, scan-10, update

17
adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/cql/parser/CqlParserHarnessTest.java
View File

@@ -16,16 +16,11 @@
package io.nosqlbench.converters.cql.cql.parser;
import io.nosqlbench.cqlgen.exporter.CGWorkloadExporter;
import io.nosqlbench.cqlgen.exporter.transformers.CGModelTransformers;
import io.nosqlbench.cqlgen.core.CGWorkloadExporter;
import io.nosqlbench.cqlgen.parser.CqlModelParser;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
public class CqlParserHarnessTest {
private final static String ksddl = """
@@ -47,19 +42,13 @@ public class CqlParserHarnessTest {
@Test
public void testAllTypes() {
CGWorkloadExporter exporter = new CGWorkloadExporter(Path.of("src/test/resources/testschemas/cql_alltypes.cql"), new CGModelTransformers());
CGWorkloadExporter exporter = new CGWorkloadExporter();
exporter.appMain(new String[]{"src/test/resources/testschemas/cql_alltypes.cql"});
exporter.setNamingTemplate("[OPTYPE-][COLUMN-][TYPEDEF-][TABLE!]-[KEYSPACE]");
var data = exporter.getWorkloadAsYaml();
}
@Disabled
@Test
public void testGenBasicWorkload() {
CGWorkloadExporter exporter = new CGWorkloadExporter(ddl, new CGModelTransformers());
assertThatThrownBy(() -> exporter.getWorkloadAsYaml()).isInstanceOf(RuntimeException.class);
}
@Test
public void testCqlParserHarnessCombined() {
CqlModelParser.parse(ddl, null);

2
adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/CGElementNamerTest.java
View File

@@ -17,7 +17,7 @@
package io.nosqlbench.converters.cql.exporters;
import io.nosqlbench.api.labels.Labeled;
import io.nosqlbench.cqlgen.exporter.CGElementNamer;
import io.nosqlbench.cqlgen.core.CGElementNamer;
import org.junit.jupiter.api.Test;
import java.util.Map;

2
adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/CGWorkloadExporterTest.java
View File

@@ -16,7 +16,7 @@
package io.nosqlbench.converters.cql.exporters;
import io.nosqlbench.cqlgen.exporter.CGColumnRebinder;
import io.nosqlbench.cqlgen.core.CGColumnRebinder;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;

2
adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/NamingFolioTest.java
View File

@@ -16,7 +16,7 @@
package io.nosqlbench.converters.cql.exporters;
import io.nosqlbench.cqlgen.exporter.binders.NamingFolio;
import io.nosqlbench.cqlgen.binders.NamingFolio;
import org.junit.jupiter.api.Test;
import java.util.Map;