From c98d5ffc66af39b7f8e485154b295865915403da Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 26 Jul 2022 01:14:58 -0500 Subject: [PATCH] cqlgen improvements and improved structure --- .../binders => api}/BindingsLibrary.java | 7 +- .../CGModelTransformer.java | 7 +- .../{exporter => api}/CGTextTransformer.java | 2 +- .../CGTransformerConfigurable.java | 2 +- .../nosqlbench/cqlgen/api/CqlModelInfo.java | 51 +++++++++ .../{exporter => }/binders/Binding.java | 2 +- .../binders/BindingsAccumulator.java | 3 +- .../{exporter => }/binders/NamingFolio.java | 4 +- .../{exporter => }/binders/NamingStyle.java | 2 +- .../binders/UnresolvedBindingException.java | 2 +- .../cqlgen/bindspecs/BindingBuilder.java | 22 ++++ .../cqlgen/bindspecs/BindingSpec.java | 41 +++++++ .../cqlgen/bindspecs/BindingSpecImpl.java | 49 ++++++++ .../{exporter => core}/CGColumnRebinder.java | 7 +- .../CGDefaultCqlBindings.java | 6 +- .../{exporter => core}/CGElementNamer.java | 2 +- .../CGExporterConfig.java} | 7 +- .../{exporter => core}/CGKeyspaceStats.java | 2 +- .../{exporter => core}/CGLiteralFormat.java | 2 +- .../{exporter => core}/CGSchemaStats.java | 3 +- .../{exporter => core}/CGTableStats.java | 2 +- .../CGTextTransformers.java | 7 +- .../CGWorkloadExporter.java | 99 ++++++++--------- .../transformers/CGIfNotExistsInjector.java | 52 --------- .../cqlgen/model/ComputedSchemaStats.java | 20 ++++ .../nosqlbench/cqlgen/model/CqlKeyspace.java | 2 +- .../io/nosqlbench/cqlgen/model/CqlModel.java | 105 +++++++++++------- .../io/nosqlbench/cqlgen/model/CqlTable.java | 12 +- .../transformers/CGCachingNameRemapper.java | 2 +- .../transformers/CGGenStatsInjector.java | 6 +- .../transformers/CGKeySpaceDDLRemover.java | 3 +- .../transformers/CGKeyspaceFilter.java | 4 +- .../transformers/CGModelTransformers.java | 4 +- .../transformers/CGNameObfuscator.java | 4 +- .../transformers/CGRatioCalculator.java | 32 ++++-- .../cqlgen/transformers/CGRatioSuffixer.java | 73 ++++++++++++ .../transformers/CGRegexReplacer.java | 5 +- .../CGReplicationSettingInjector.java | 4 +- .../transformers/CGUdtReplacer.java | 3 +- .../transformers/ComputedTableStats.java | 60 ++++++++++ .../transformers/CqlSchemaStatsParser.java | 8 +- .../transformers/UnusedTableRemover.java | 4 +- .../src/main/resources/cqlgen/cqlgen.conf | 32 +++--- .../cql/cql/parser/CqlParserHarnessTest.java | 17 +-- .../cql/exporters/CGElementNamerTest.java | 2 +- .../cql/exporters/CGWorkloadExporterTest.java | 2 +- .../cql/exporters/NamingFolioTest.java | 2 +- 47 files changed, 555 insertions(+), 234 deletions(-) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter/binders => api}/BindingsLibrary.java (78%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter/transformers => api}/CGModelTransformer.java (73%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => api}/CGTextTransformer.java (95%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter/transformers => api}/CGTransformerConfigurable.java (93%) create mode 100644 adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CqlModelInfo.java rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/binders/Binding.java (96%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/binders/BindingsAccumulator.java (98%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/binders/NamingFolio.java (97%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/binders/NamingStyle.java (96%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/binders/UnresolvedBindingException.java (95%) create mode 100644 adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingBuilder.java create mode 100644 adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingSpec.java create mode 100644 adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingSpecImpl.java rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => core}/CGColumnRebinder.java (94%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => core}/CGDefaultCqlBindings.java (94%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => core}/CGElementNamer.java (99%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{traverser/CqlDDlDirectoryTraverser.java => core/CGExporterConfig.java} (77%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => core}/CGKeyspaceStats.java (98%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => core}/CGLiteralFormat.java (97%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => core}/CGSchemaStats.java (96%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => core}/CGTableStats.java (97%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => core}/CGTextTransformers.java (94%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => core}/CGWorkloadExporter.java (93%) delete mode 100644 adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGIfNotExistsInjector.java create mode 100644 adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/ComputedSchemaStats.java rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/transformers/CGCachingNameRemapper.java (97%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/transformers/CGGenStatsInjector.java (90%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/transformers/CGKeySpaceDDLRemover.java (91%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/transformers/CGKeyspaceFilter.java (96%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/transformers/CGModelTransformers.java (95%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/transformers/CGNameObfuscator.java (95%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/transformers/CGRatioCalculator.java (69%) create mode 100644 adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioSuffixer.java rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/transformers/CGRegexReplacer.java (96%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/transformers/CGReplicationSettingInjector.java (90%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/transformers/CGUdtReplacer.java (93%) create mode 100644 adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/ComputedTableStats.java rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/transformers/CqlSchemaStatsParser.java (94%) rename adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/{exporter => }/transformers/UnusedTableRemover.java (94%) diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/BindingsLibrary.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/BindingsLibrary.java similarity index 78% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/BindingsLibrary.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/BindingsLibrary.java index 2d6577f9d..6f2d25c10 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/BindingsLibrary.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/BindingsLibrary.java @@ -14,12 +14,17 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.binders; +package io.nosqlbench.cqlgen.api; +import io.nosqlbench.cqlgen.binders.Binding; import io.nosqlbench.cqlgen.model.CqlColumnDef; import java.util.Optional; +/** + * A bindings library is simply a service point for a specific way + * to map a column definition to a binding function. + */ public interface BindingsLibrary { Optional resolveBindingsFor(CqlColumnDef def); } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGModelTransformer.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGModelTransformer.java similarity index 73% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGModelTransformer.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGModelTransformer.java index 03bd3bae0..753bc39e4 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGModelTransformer.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGModelTransformer.java @@ -14,12 +14,17 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.transformers; +package io.nosqlbench.cqlgen.api; import io.nosqlbench.cqlgen.model.CqlModel; import java.util.function.Function; +/** + * Most of the functionality of {@link CqlModel} preparation is handled with transformers. + * The type and order of transformers is important, as one transformer may be responsible + * for preparing the model for one or more downstream transformers. + */ public interface CGModelTransformer extends Function { @Override CqlModel apply(CqlModel model); diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGTextTransformer.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGTextTransformer.java similarity index 95% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGTextTransformer.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGTextTransformer.java index f463170ea..74603c446 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGTextTransformer.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGTextTransformer.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter; +package io.nosqlbench.cqlgen.api; import java.util.function.Function; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGTransformerConfigurable.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGTransformerConfigurable.java similarity index 93% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGTransformerConfigurable.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGTransformerConfigurable.java index 96eb61fc1..1c16368cb 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGTransformerConfigurable.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CGTransformerConfigurable.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.transformers; +package io.nosqlbench.cqlgen.api; import java.util.function.Consumer; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CqlModelInfo.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CqlModelInfo.java new file mode 100644 index 000000000..dbc172b6f --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/api/CqlModelInfo.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.cqlgen.api; + +import io.nosqlbench.cqlgen.core.CGSchemaStats; +import io.nosqlbench.cqlgen.model.CqlKeyspace; +import io.nosqlbench.cqlgen.model.CqlTable; +import io.nosqlbench.cqlgen.model.CqlType; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public interface CqlModelInfo { + + CGSchemaStats getStats(); + + boolean hasStats(); + + Map getKeyspacesByName(); + + List getKeyspaceDefs(); + + Map> getTableDefsByKeyspaceThenTable(); + + List getTablesForKeyspace(String ksname); + + List getTableDefs(); + + Set getAllKnownKeyspaceNames(); + + List getTypeDefs(); + + String getSummaryLine(); + + Map> getTypesByKeyspaceThenName(); +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/Binding.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/Binding.java similarity index 96% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/Binding.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/Binding.java index 75e6b961a..213dad584 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/Binding.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/Binding.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.binders; +package io.nosqlbench.cqlgen.binders; public class Binding { String name; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/BindingsAccumulator.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/BindingsAccumulator.java similarity index 98% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/BindingsAccumulator.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/BindingsAccumulator.java index e8f6a0c83..d02ae1f25 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/BindingsAccumulator.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/BindingsAccumulator.java @@ -14,8 +14,9 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.binders; +package io.nosqlbench.cqlgen.binders; +import io.nosqlbench.cqlgen.api.BindingsLibrary; import io.nosqlbench.cqlgen.model.CqlColumnDef; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/NamingFolio.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/NamingFolio.java similarity index 97% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/NamingFolio.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/NamingFolio.java index f7d19e4f8..3a10615be 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/NamingFolio.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/NamingFolio.java @@ -14,12 +14,12 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.binders; +package io.nosqlbench.cqlgen.binders; import io.nosqlbench.cqlgen.model.CqlColumnDef; import io.nosqlbench.cqlgen.model.CqlModel; import io.nosqlbench.cqlgen.model.CqlTable; -import io.nosqlbench.cqlgen.exporter.CGElementNamer; +import io.nosqlbench.cqlgen.core.CGElementNamer; import io.nosqlbench.api.labels.Labeled; import java.util.*; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/NamingStyle.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/NamingStyle.java similarity index 96% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/NamingStyle.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/NamingStyle.java index 49886eda6..ae17f6271 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/NamingStyle.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/NamingStyle.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.binders; +package io.nosqlbench.cqlgen.binders; public enum NamingStyle { /** diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/UnresolvedBindingException.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/UnresolvedBindingException.java similarity index 95% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/UnresolvedBindingException.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/UnresolvedBindingException.java index 6a5dce36b..360d02035 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/binders/UnresolvedBindingException.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/binders/UnresolvedBindingException.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.binders; +package io.nosqlbench.cqlgen.binders; import io.nosqlbench.cqlgen.model.CqlColumnDef; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingBuilder.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingBuilder.java new file mode 100644 index 000000000..94ec2c3f5 --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingBuilder.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.cqlgen.bindspecs; + +public interface BindingBuilder { + BindingBuilder prependRaw(String prefunc); + BindingSpec build(); +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingSpec.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingSpec.java new file mode 100644 index 000000000..7c11bc7f7 --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingSpec.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.cqlgen.bindspecs; + +import io.nosqlbench.api.labels.Labeled; + +public interface BindingSpec { + + /** + * Is this binding intended to be for a limited population? + * If so, the value will be the maximum cardinality of values which the binding + * is allowed to produce. + * @return The effective cardinality, which could be {@link Double#POSITIVE_INFINITY} + */ + default double getCardinality() { + return Double.POSITIVE_INFINITY; + } + + /** + * The fully qualified name of the entity for which the binding values pertain. + * This is + * @return + */ + Labeled getTarget(); + + String getTypedef(); +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingSpecImpl.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingSpecImpl.java new file mode 100644 index 000000000..3b9f3a40b --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/bindspecs/BindingSpecImpl.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.cqlgen.bindspecs; + +import io.nosqlbench.api.labels.Labeled; + +public class BindingSpecImpl implements BindingSpec { + private Labeled target; + private double cardinality; + private String typedef; + + public BindingSpecImpl(Labeled target) { + this.target = target; + } + + @Override + public Labeled getTarget() { + return target; + } + + @Override + public String getTypedef() { + return typedef; + } + + @Override + public double getCardinality() { + return BindingSpec.super.getCardinality(); + } + + public void setTarget(Labeled target) { + this.target = target; + } + +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGColumnRebinder.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGColumnRebinder.java similarity index 94% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGColumnRebinder.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGColumnRebinder.java index 3cb2113e0..facc133e0 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGColumnRebinder.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGColumnRebinder.java @@ -14,11 +14,11 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter; +package io.nosqlbench.cqlgen.core; import io.nosqlbench.cqlgen.model.CqlColumnDef; -import io.nosqlbench.cqlgen.exporter.binders.Binding; -import io.nosqlbench.cqlgen.exporter.binders.BindingsAccumulator; +import io.nosqlbench.cqlgen.binders.Binding; +import io.nosqlbench.cqlgen.binders.BindingsAccumulator; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -43,6 +43,7 @@ public class CGColumnRebinder { } } + private Binding dividedBinding(CqlColumnDef column) { CGTableStats stats = column.getTable().getTableAttributes(); if (stats == null) { diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGDefaultCqlBindings.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java similarity index 94% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGDefaultCqlBindings.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java index bafda8f9a..705852c07 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGDefaultCqlBindings.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java @@ -14,11 +14,11 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter; +package io.nosqlbench.cqlgen.core; import io.nosqlbench.cqlgen.model.CqlColumnDef; -import io.nosqlbench.cqlgen.exporter.binders.Binding; -import io.nosqlbench.cqlgen.exporter.binders.BindingsLibrary; +import io.nosqlbench.cqlgen.binders.Binding; +import io.nosqlbench.cqlgen.api.BindingsLibrary; import io.nosqlbench.engine.api.activityconfig.StatementsLoader; import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList; import io.nosqlbench.api.content.Content; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGElementNamer.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGElementNamer.java similarity index 99% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGElementNamer.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGElementNamer.java index 0a411d185..43b4f356b 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGElementNamer.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGElementNamer.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter; +package io.nosqlbench.cqlgen.core; import io.nosqlbench.api.labels.Labeled; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/traverser/CqlDDlDirectoryTraverser.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGExporterConfig.java similarity index 77% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/traverser/CqlDDlDirectoryTraverser.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGExporterConfig.java index 3934a15e3..2c460270b 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/traverser/CqlDDlDirectoryTraverser.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGExporterConfig.java @@ -14,11 +14,10 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.traverser; +package io.nosqlbench.cqlgen.core; -import java.nio.file.Path; +public class CGExporterConfig { + public CGExporterConfig(String[] args) { -public class CqlDDlDirectoryTraverser { - public void buildWorkloads(Path sourcePath, Path targetPath) { } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGKeyspaceStats.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGKeyspaceStats.java similarity index 98% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGKeyspaceStats.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGKeyspaceStats.java index 77606711a..d7b0a8fea 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGKeyspaceStats.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGKeyspaceStats.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter; +package io.nosqlbench.cqlgen.core; import java.util.HashMap; import java.util.Map; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGLiteralFormat.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGLiteralFormat.java similarity index 97% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGLiteralFormat.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGLiteralFormat.java index c9c253895..2cdb54d16 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGLiteralFormat.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGLiteralFormat.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter; +package io.nosqlbench.cqlgen.core; import io.nosqlbench.cqlgen.model.CqlColumnDef; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGSchemaStats.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGSchemaStats.java similarity index 96% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGSchemaStats.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGSchemaStats.java index 718287b8a..70880cca2 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGSchemaStats.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGSchemaStats.java @@ -14,12 +14,13 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter; +package io.nosqlbench.cqlgen.core; import java.util.HashMap; import java.util.Map; public class CGSchemaStats { + Map keyspaces = new HashMap(); public Map getKeyspaces() { diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGTableStats.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGTableStats.java similarity index 97% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGTableStats.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGTableStats.java index bc7d37790..97c64a6ca 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGTableStats.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGTableStats.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter; +package io.nosqlbench.cqlgen.core; import java.util.HashMap; import java.util.Map; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGTextTransformers.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGTextTransformers.java similarity index 94% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGTextTransformers.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGTextTransformers.java index 356e66d19..9f09ef378 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGTextTransformers.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGTextTransformers.java @@ -14,10 +14,11 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter; +package io.nosqlbench.cqlgen.core; -import io.nosqlbench.cqlgen.exporter.transformers.CGNameObfuscator; -import io.nosqlbench.cqlgen.exporter.transformers.CGTransformerConfigurable; +import io.nosqlbench.cqlgen.api.CGTextTransformer; +import io.nosqlbench.cqlgen.transformers.CGNameObfuscator; +import io.nosqlbench.cqlgen.api.CGTransformerConfigurable; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGWorkloadExporter.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java similarity index 93% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGWorkloadExporter.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java index 55c9a98f3..181e07fe4 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/CGWorkloadExporter.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java @@ -14,17 +14,19 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter; +package io.nosqlbench.cqlgen.core; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import io.nosqlbench.cqlgen.exporter.binders.BindingsAccumulator; -import io.nosqlbench.cqlgen.exporter.binders.BindingsLibrary; -import io.nosqlbench.cqlgen.exporter.transformers.CGModelTransformers; -import io.nosqlbench.cqlgen.parser.CqlModelParser; -import io.nosqlbench.cqlgen.exporter.binders.Binding; -import io.nosqlbench.cqlgen.exporter.binders.NamingFolio; +import io.nosqlbench.api.spi.BundledApp; +import io.nosqlbench.cqlgen.binders.Binding; +import io.nosqlbench.cqlgen.binders.BindingsAccumulator; +import io.nosqlbench.cqlgen.api.BindingsLibrary; +import io.nosqlbench.cqlgen.binders.NamingFolio; +import io.nosqlbench.cqlgen.transformers.CGModelTransformers; import io.nosqlbench.cqlgen.model.*; +import io.nosqlbench.cqlgen.parser.CqlModelParser; +import io.nosqlbench.nb.annotations.Service; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.snakeyaml.engine.v2.api.Dump; @@ -40,7 +42,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.*; -import java.util.function.Function; import java.util.stream.Collectors; /** @@ -51,11 +52,12 @@ import java.util.stream.Collectors; * * @see Apache Cassandra CQL Docs */ -public class CGWorkloadExporter { +@Service(value= BundledApp.class,selector = "cqlgen") +public class CGWorkloadExporter implements BundledApp { private final static Logger logger = LogManager.getLogger(CGWorkloadExporter.class); private CGColumnRebinder binder; private NamingFolio namer; - private final CqlModel model; + private CqlModel model; private final int DEFAULT_RESOLUTION = 10000; @@ -79,38 +81,16 @@ public class CGWorkloadExporter { "update", 10.0 )); - public CGWorkloadExporter(CqlModel model, CGModelTransformers transformers) { - this.model = model; - for (Function transformer : transformers.get()) { - CqlModel modified = transformer.apply(this.model); - model = modified; - } - } - - public CGWorkloadExporter(String ddl, Path srcpath, CGModelTransformers transformers) { - this(CqlModelParser.parse(ddl, srcpath), transformers); - } - - public CGWorkloadExporter(String ddl, CGModelTransformers transformers) { - this(ddl, null, transformers); - } - - public CGWorkloadExporter(Path path, CGModelTransformers transformers) { - this(CqlModelParser.parse(path), transformers); - } - - private String loadFile(Path path) { - try { - String ddl = Files.readString(path); - logger.info("read " + ddl.length() + " character DDL file"); - return ddl; - } catch (IOException e) { - throw new RuntimeException(e); - } - - } + private CGExporterConfig config; public static void main(String[] args) { + new CGWorkloadExporter().appMain(args); + } + + @Override + public int appMain(String[] args) { + this.config = new CGExporterConfig(args); + logger.info("running CQL workload exporter with args:" + Arrays.toString(args)); if (args.length == 0) { @@ -145,6 +125,7 @@ public class CGWorkloadExporter { Path cfgpath = Path.of("cqlgen.conf"); CGWorkloadExporter exporter; + if (Files.exists(cfgpath)) { try { CGModelTransformers modelTransformers = new CGModelTransformers(); @@ -172,20 +153,21 @@ public class CGWorkloadExporter { throw new RuntimeException(e); } - exporter = new CGWorkloadExporter(ddl, srcpath, modelTransformers); String defaultNamingTemplate = cfgmap.get("naming_template").toString(); - exporter.setNamingTemplate(defaultNamingTemplate); + setNamingTemplate(defaultNamingTemplate); String partition_multipler = cfgmap.get("partition_multiplier").toString(); - exporter.setPartitionMultiplier(Double.parseDouble(partition_multipler)); + setPartitionMultiplier(Double.parseDouble(partition_multipler)); - exporter.configureTimeouts(cfgmap.get("timeouts")); + configureTimeouts(cfgmap.get("timeouts")); - exporter.configureBlocks(cfgmap.get("blockplan")); - exporter.configureQuantizerDigits(cfgmap.get("quantizer_digits")); + configureBlocks(cfgmap.get("blockplan")); + configureQuantizerDigits(cfgmap.get("quantizer_digits")); - String workload = exporter.getWorkloadAsYaml(); + this.model = CqlModelParser.parse(ddl, srcpath); + + String workload = getWorkloadAsYaml(); try { Files.writeString( target, @@ -201,8 +183,21 @@ public class CGWorkloadExporter { throw new RuntimeException(e); } } + return 0; } + private String loadFile(Path path) { + try { + String ddl = Files.readString(path); + logger.info("read " + ddl.length() + " character DDL file"); + return ddl; + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + private void configureQuantizerDigits(Object quantizer_digits) { if (quantizer_digits != null) { this.quantizerDigits = Integer.parseInt(quantizer_digits.toString()); @@ -237,10 +232,10 @@ public class CGWorkloadExporter { case "drop-tables" -> genDropTablesBlock(model, blockname); case "drop-keyspaces" -> genDropKeyspacesOpTemplates(model, blockname); case "truncate-tables" -> genTruncateTablesOpTemplates(model, blockname); - case "insert" -> genInsertOpTemplates(model, blockname); - case "select" -> genSelectOpTemplates(model, blockname); - case "scan-10" -> genScanOpTemplates(model, blockname); - case "update" -> genUpdateOpTemplates(model, blockname); + case "insert-seq" -> genInsertOpTemplates(model, blockname); + case "select-seq" -> genSelectOpTemplates(model, blockname); + case "scan-10-seq" -> genScanOpTemplates(model, blockname); + case "update-seq" -> genUpdateOpTemplates(model, blockname); default -> throw new RuntimeException("Unable to create block entries for " + component + "."); }; block.putAll(additions); @@ -446,7 +441,7 @@ public class CGWorkloadExporter { .replaceAll("BINDINGS", String.join(", ", table.getColumnDefinitions().stream() - .map(binder::forColumn) + .map(c -> binder.forColumn(c)) .map(c -> "{" + c.getName() + "}").toList())); } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGIfNotExistsInjector.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGIfNotExistsInjector.java deleted file mode 100644 index 1310b4997..000000000 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGIfNotExistsInjector.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2022 nosqlbench - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.nosqlbench.cqlgen.exporter.transformers; - -import io.nosqlbench.cqlgen.model.CqlModel; - -/** - * @deprecated Superseded by direct rendering from AST in generator - */ -public class CGIfNotExistsInjector implements CGModelTransformer { - - @Override - public CqlModel apply(CqlModel model) { -// for (CqlKeyspace keyspace : model.getKeyspaceDefs()) { -// keyspace.setRefDdl(keyspace.getRefddl().replaceAll( -// "(?m)(?s)(?i)(\\s*CREATE (TABLE|KEYSPACE|TYPE) +)(?!IF NOT EXISTS)", -// "$1IF NOT EXISTS " -// )); -// } -// for (CqlTable table : model.getTableDefs()) { -// String refddl = table.getRefDdl(); -// String replaced = refddl.replaceAll( -// "(?m)(?s)(?i)(\\s*CREATE (TABLE|KEYSPACE|TYPE) +)(?!IF NOT EXISTS)", -// "$1IF NOT EXISTS " -// ); -// -// table.setRefDdl(replaced); -// } -// for (CqlType type : model.getTypes()) { -// type.setRefddl(type.getRefDdl().replaceAll( -// "(?m)(?s)(?i)(\\s*CREATE (TABLE|KEYSPACE|TYPE) +)(?!IF NOT EXISTS)", -// "$1IF NOT EXISTS " -// )); -// -// } - return model; - } -} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/ComputedSchemaStats.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/ComputedSchemaStats.java new file mode 100644 index 000000000..660de966d --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/ComputedSchemaStats.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.cqlgen.model; + +public class ComputedSchemaStats { +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlKeyspace.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlKeyspace.java index 6192bec3d..6a70af512 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlKeyspace.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlKeyspace.java @@ -18,7 +18,7 @@ package io.nosqlbench.cqlgen.model; import io.nosqlbench.api.config.NBNamedElement; import io.nosqlbench.api.labels.Labeled; -import io.nosqlbench.cqlgen.exporter.CGKeyspaceStats; +import io.nosqlbench.cqlgen.core.CGKeyspaceStats; import java.util.Map; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModel.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModel.java index f39d5edb0..3f62f509f 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModel.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlModel.java @@ -16,9 +16,10 @@ package io.nosqlbench.cqlgen.model; -import io.nosqlbench.cqlgen.exporter.CGKeyspaceStats; -import io.nosqlbench.cqlgen.exporter.CGSchemaStats; -import io.nosqlbench.cqlgen.exporter.CGTableStats; +import io.nosqlbench.cqlgen.api.CqlModelInfo; +import io.nosqlbench.cqlgen.core.CGKeyspaceStats; +import io.nosqlbench.cqlgen.core.CGSchemaStats; +import io.nosqlbench.cqlgen.core.CGTableStats; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -26,30 +27,49 @@ import java.util.*; import java.util.function.Supplier; /** + *

* This model contains definition level details for schema elements which are parsed from the * Antlr4 CQL grammar. - * Because keyspace, table, column, and type elements are handled sometimes in different ways, - * these are stored in separate data structures. - * When you see a *refddl or similar field, this is a copy of the text image from the original - * parsed syntax. These are used for populating schema blocks without doing a full parse. - * If you update either the refddl or the actual AST level elements for any of the types in this - * model, you are required to update the other version along with it, using string substitution - * if necessary. + * Key elements include: + *

    + *
  • keyspace definitions, organized by keyspace name
    • + *
  • type definitions, organized by keyspace name
    • + *
  • table definitions with included column definitions, organized by keyspace name
    • + *
+ *

+ * + *

Because keyspace, table, and type elements are handled sometimes in different ways, + * these are stored in separate data structures, mapped by the logical keyspace name. This means + * that you will see table definitions for named keyspaces even if those named keyspaces are not represented + * in the keyspace definitions. This allows for sub-selecting of rendered elements by logical + * name without requiring a fully-interconnected keyspace->table->column object graph. + *

*/ -public class CqlModel { +public class CqlModel implements CqlModelInfo { private final static Logger logger = LogManager.getLogger(CqlModel.class); private final Supplier> errors; - Map keyspaceDefs = new LinkedHashMap<>(); - Map> tableDefs = new LinkedHashMap<>(); - Map> typeDefs = new LinkedHashMap<>(); + private final Map keyspaceDefs = new LinkedHashMap<>(); + private final Map> tableDefs = new LinkedHashMap<>(); + private final Map> typeDefs = new LinkedHashMap<>(); - CGSchemaStats schemaStats = null; + private CGSchemaStats schemaStats = null; + private ComputedSchemaStats computedSchemaStats; - public CGSchemaStats getKeyspaceAttributes() { + @Override + public CGSchemaStats getStats() { return schemaStats; } + @Override + public boolean hasStats() { + return schemaStats!=null; + } + + public ComputedSchemaStats getComputedStats() { + return computedSchemaStats; + } + public void setKeyspaceAttributes(CGSchemaStats schemaStats) { this.schemaStats = schemaStats; for (String statsKeyspacename : schemaStats.getKeyspaces().keySet()) { @@ -64,13 +84,13 @@ public class CqlModel { for (String statsTableName : keyspaceStats.getKeyspaceTables().keySet()) { CGTableStats tableStats = keyspaceStats.getKeyspaceTables().get(statsTableName); Map modelTables = tableDefs.get(statsKeyspacename); - if (modelTables!=null) { + if (modelTables != null) { CqlTable modelTable = modelTables.get(statsTableName); - if (modelTable!=null) { - logger.debug("setting table stats for '" + statsKeyspacename+"."+statsTableName+"'"); + if (modelTable != null) { + logger.debug("setting table stats for '" + statsKeyspacename + "." + statsTableName + "'"); modelTable.setTableAttributes(tableStats); } else { - logger.debug(" skipping table stats for '" + statsKeyspacename + "."+statsTableName+"'"); + logger.debug(" skipping table stats for '" + statsKeyspacename + "." + statsTableName + "'"); } } else { logger.debug(" SKIPPING stats for all tables in keyspace '" + statsKeyspacename + "'"); @@ -83,9 +103,6 @@ public class CqlModel { transient CqlTable table; transient CqlType udt; - public boolean hasStats() { - return schemaStats!=null; - } public CqlModel(Supplier> errorSource) { this.errors = errorSource; @@ -99,10 +116,10 @@ public class CqlModel { keyspace = new CqlKeyspace(); } - public void saveKeyspace(String text,String refddl) { + public void saveKeyspace(String text, String refddl) { keyspace.setKeyspaceName(text); this.keyspaceDefs.put(text, keyspace); - keyspace=null; + keyspace = null; } public void newTable() { @@ -112,7 +129,7 @@ public class CqlModel { public void saveTable(String keyspace, String text) { table.setKeyspace(keyspace); table.setName(text); - this.tableDefs.computeIfAbsent(keyspace, ks->new LinkedHashMap<>()).put(text, table); + this.tableDefs.computeIfAbsent(keyspace, ks -> new LinkedHashMap<>()).put(text, table); table = null; } @@ -123,28 +140,33 @@ public class CqlModel { } } + @Override public Map getKeyspacesByName() { return keyspaceDefs; } + @Override public List getKeyspaceDefs() { return new ArrayList<>(this.keyspaceDefs.values()); } + @Override public Map> getTableDefsByKeyspaceThenTable() { return tableDefs; } + @Override public List getTablesForKeyspace(String ksname) { Map tables = this.tableDefs.get(ksname); - if (tables!=null) { + if (tables != null) { return new ArrayList<>(tables.values()); } return List.of(); } + @Override public List getTableDefs() { - return tableDefs.values().stream().flatMap(m->m.values().stream()).toList(); + return tableDefs.values().stream().flatMap(m -> m.values().stream()).toList(); } @Override @@ -155,7 +177,7 @@ public class CqlModel { sb.append("keyspace '").append(keyspace.getName()).append("':\n"); sb.append(keyspace).append("\n"); - tableDefs.getOrDefault(ks,Map.of()).values().stream() + tableDefs.getOrDefault(ks, Map.of()).values().stream() .forEach(table -> { sb.append("table '").append(table.getName()).append("':\n"); sb.append(table); @@ -167,8 +189,10 @@ public class CqlModel { /** * Get all the keyspace names which have been referenced in any way, whether or not * this was in a keyspace definition or some other DDL like table or udt names. + * * @return A list of all known keyspace names */ + @Override public Set getAllKnownKeyspaceNames() { Set ksnames = new LinkedHashSet<>(); ksnames.addAll(this.keyspaceDefs.keySet()); @@ -196,10 +220,11 @@ public class CqlModel { udt.setKeyspace(keyspace); udt.setName(name); Map ksTypes = this.typeDefs.computeIfAbsent(keyspace, ks -> new LinkedHashMap<>()); - ksTypes.put(udt.getName(),udt); - udt=null; + ksTypes.put(udt.getName(), udt); + udt = null; } + @Override public List getTypeDefs() { ArrayList list = new ArrayList<>(); for (Map cqlTypesByKeyspace : typeDefs.values()) { @@ -222,8 +247,9 @@ public class CqlModel { this.typeDefs.remove(name); } + @Override public String getSummaryLine() { - return "keyspaces: " + keyspaceDefs.size() + ", tables: " + getTableDefs().size() + + return "keyspaces: " + keyspaceDefs.size() + ", tables: " + getTableDefs().size() + ", columns: " + getTableDefs().stream().mapToInt(t -> t.getColumnDefinitions().size()).sum() + ", types: " + getTypeDefs().size(); } @@ -236,7 +262,7 @@ public class CqlModel { } if (this.tableDefs.containsKey(keyspaceName)) { Map tablesForKeyspace = this.tableDefs.remove(keyspaceName); - if (tablesForKeyspace!=null) { + if (tablesForKeyspace != null) { for (CqlTable table : tablesForKeyspace.values()) { table.setKeyspace(newKeyspaceName); } @@ -245,12 +271,12 @@ public class CqlModel { } if (this.typeDefs.containsKey(keyspaceName)) { Map typesForKeyspace = this.typeDefs.remove(keyspaceName); - if (typesForKeyspace!=null) { + if (typesForKeyspace != null) { for (CqlType cqltype : typesForKeyspace.values()) { cqltype.setKeyspace(newKeyspaceName); } } - this.typeDefs.put(newKeyspaceName,typesForKeyspace); + this.typeDefs.put(newKeyspaceName, typesForKeyspace); } } @@ -258,14 +284,14 @@ public class CqlModel { Map tablesInKs = tableDefs.get(extant.getKeySpace()); CqlTable table = tablesInKs.get(extant.getName()); table.setName(newTableName); - tablesInKs.put(table.getName(),table); + tablesInKs.put(table.getName(), table); } public void renameType(String keyspaceName, String typeName, String newTypeName) { - Map typesInKeyspace = typeDefs.get(keyspaceName); + Map typesInKeyspace = typeDefs.get(keyspaceName); CqlType cqlType = typesInKeyspace.remove(typeName); cqlType.setName(newTypeName); - typesInKeyspace.put(newTypeName,cqlType); + typesInKeyspace.put(newTypeName, cqlType); } public void setTableCompactStorage(boolean isCompactStorage) { @@ -280,6 +306,7 @@ public class CqlModel { keyspace.setReplicationData(repldata); } + @Override public Map> getTypesByKeyspaceThenName() { return typeDefs; } @@ -289,6 +316,6 @@ public class CqlModel { } public boolean isEmpty() { - return this.keyspaceDefs.size()==0 && this.tableDefs.size()==0 && this.typeDefs.size()==0; + return this.keyspaceDefs.size() == 0 && this.tableDefs.size() == 0 && this.typeDefs.size() == 0; } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTable.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTable.java index 27ccf1b9b..1fe4f038c 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTable.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/model/CqlTable.java @@ -18,7 +18,8 @@ package io.nosqlbench.cqlgen.model; import io.nosqlbench.api.config.NBNamedElement; import io.nosqlbench.api.labels.Labeled; -import io.nosqlbench.cqlgen.exporter.CGTableStats; +import io.nosqlbench.cqlgen.core.CGTableStats; +import io.nosqlbench.cqlgen.transformers.ComputedTableStats; import java.util.*; import java.util.function.Function; @@ -33,6 +34,7 @@ public class CqlTable implements NBNamedElement, Labeled { List clusteringOrders = new ArrayList<>(); List coldefs = new ArrayList<>(); private boolean compactStorage; + private ComputedTableStats computedTableStats; public CqlTable() { } @@ -191,4 +193,12 @@ public class CqlTable implements NBNamedElement, Labeled { public boolean isLastClusteringColumn(int position) { return clustering.length > 0 && position == clustering[clustering.length - 1]; } + + public ComputedTableStats getComputedStats() { + return this.computedTableStats; + } + + public void setComputedStats(ComputedTableStats stats) { + this.computedTableStats = stats; + } } diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGCachingNameRemapper.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGCachingNameRemapper.java similarity index 97% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGCachingNameRemapper.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGCachingNameRemapper.java index 9f9d19642..467292a11 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGCachingNameRemapper.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGCachingNameRemapper.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.transformers; +package io.nosqlbench.cqlgen.transformers; import io.nosqlbench.api.config.NBNamedElement; import io.nosqlbench.api.labels.Labeled; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGGenStatsInjector.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGGenStatsInjector.java similarity index 90% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGGenStatsInjector.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGGenStatsInjector.java index bdcfb808e..e81a37503 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGGenStatsInjector.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGGenStatsInjector.java @@ -14,9 +14,11 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.transformers; +package io.nosqlbench.cqlgen.transformers; -import io.nosqlbench.cqlgen.exporter.CGSchemaStats; +import io.nosqlbench.cqlgen.api.CGModelTransformer; +import io.nosqlbench.cqlgen.api.CGTransformerConfigurable; +import io.nosqlbench.cqlgen.core.CGSchemaStats; import io.nosqlbench.cqlgen.model.CqlModel; import java.io.IOException; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGKeySpaceDDLRemover.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeySpaceDDLRemover.java similarity index 91% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGKeySpaceDDLRemover.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeySpaceDDLRemover.java index b51d39ed6..f600f608d 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGKeySpaceDDLRemover.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeySpaceDDLRemover.java @@ -14,8 +14,9 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.transformers; +package io.nosqlbench.cqlgen.transformers; +import io.nosqlbench.cqlgen.api.CGModelTransformer; import io.nosqlbench.cqlgen.model.CqlKeyspace; import io.nosqlbench.cqlgen.model.CqlModel; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGKeyspaceFilter.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeyspaceFilter.java similarity index 96% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGKeyspaceFilter.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeyspaceFilter.java index 0ef1a8c89..0f98510b8 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGKeyspaceFilter.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGKeyspaceFilter.java @@ -14,8 +14,10 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.transformers; +package io.nosqlbench.cqlgen.transformers; +import io.nosqlbench.cqlgen.api.CGModelTransformer; +import io.nosqlbench.cqlgen.api.CGTransformerConfigurable; import io.nosqlbench.cqlgen.model.CqlModel; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGModelTransformers.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGModelTransformers.java similarity index 95% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGModelTransformers.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGModelTransformers.java index ef1c3b4b7..0d897653b 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGModelTransformers.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGModelTransformers.java @@ -14,8 +14,10 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.transformers; +package io.nosqlbench.cqlgen.transformers; +import io.nosqlbench.cqlgen.api.CGModelTransformer; +import io.nosqlbench.cqlgen.api.CGTransformerConfigurable; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGNameObfuscator.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGNameObfuscator.java similarity index 95% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGNameObfuscator.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGNameObfuscator.java index 2f4f24737..cfe7e5648 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGNameObfuscator.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGNameObfuscator.java @@ -20,8 +20,10 @@ * once an element is named, use the same name throughout * prefix each element type with a code for the type */ -package io.nosqlbench.cqlgen.exporter.transformers; +package io.nosqlbench.cqlgen.transformers; +import io.nosqlbench.cqlgen.api.CGModelTransformer; +import io.nosqlbench.cqlgen.api.CGTransformerConfigurable; import io.nosqlbench.cqlgen.model.CqlModel; import io.nosqlbench.cqlgen.model.CqlTable; import io.nosqlbench.cqlgen.model.CqlType; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGRatioCalculator.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioCalculator.java similarity index 69% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGRatioCalculator.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioCalculator.java index 5f62e5ed6..4d4f45916 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGRatioCalculator.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioCalculator.java @@ -14,9 +14,10 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.transformers; +package io.nosqlbench.cqlgen.transformers; -import io.nosqlbench.cqlgen.exporter.CGTableStats; +import io.nosqlbench.cqlgen.api.CGModelTransformer; +import io.nosqlbench.cqlgen.core.CGTableStats; import io.nosqlbench.cqlgen.model.CqlModel; import io.nosqlbench.cqlgen.model.CqlTable; @@ -28,29 +29,30 @@ public class CGRatioCalculator implements CGModelTransformer { // TODO: True this up return model; } + double totalReads = 0.0d; double totalWrites = 0.0d; double totalSpace = 0.0d; - double totalOps=0.0d; + double totalOps = 0.0d; for (CqlTable table : model.getTableDefs()) { CGTableStats tableAttributes = table.getTableAttributes(); - if (tableAttributes==null) { + if (tableAttributes == null) { continue; } String local_read_count = tableAttributes.getAttribute("Local read count"); double reads = Double.parseDouble(local_read_count); - totalReads+=reads; - totalOps+=reads; + totalReads += reads; + totalOps += reads; String local_write_count = table.getTableAttributes().getAttribute("Local write count"); double writes = Double.parseDouble(local_write_count); totalWrites += writes; - totalOps+=writes; + totalOps += writes; String space_used_total = table.getTableAttributes().getAttribute("Space used (total)"); double space = Double.parseDouble(space_used_total); - totalSpace+=space; + totalSpace += space; } for (CqlTable table : model.getTableDefs()) { @@ -59,11 +61,17 @@ public class CGRatioCalculator implements CGModelTransformer { double totalTableReads = reads / totalOps; double totalTableWrites = writes / totalOps; - table.getTableAttributes().setAttribute("weighted_reads", String.valueOf(totalTableReads)); - table.getTableAttributes().setAttribute("weighted_writes", String.valueOf(totalTableWrites)); - table.getTableAttributes().setAttribute("weighted_ops", String.valueOf(totalTableReads+totalTableWrites)); double tableSpaceUsed = Double.parseDouble(table.getTableAttributes().getAttribute("Space used (total)")); - table.getTableAttributes().setAttribute("weighted_space", String.valueOf(tableSpaceUsed / totalSpace)); + double weighted_space = tableSpaceUsed / totalSpace; + double op_share_of_total_ops = totalTableReads + totalTableWrites; + ComputedTableStats computedTableStats = new ComputedTableStats() + .setReadShareOfTotalOps(reads / totalOps) + .setReadShareOfTotalReads(reads / totalReads) + .setWriteShareOfTotalOps(writes / totalOps) + .setWriteShareOfTotalWrites(writes / totalWrites) + .setOpShareOfTotalOps(op_share_of_total_ops) + .setSpaceUsedOfTotalSpace(weighted_space); + table.setComputedStats(computedTableStats); } return model; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioSuffixer.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioSuffixer.java new file mode 100644 index 000000000..7c2f1bdce --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRatioSuffixer.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.cqlgen.transformers; + +import io.nosqlbench.api.config.standard.*; +import io.nosqlbench.cqlgen.api.CGModelTransformer; +import io.nosqlbench.cqlgen.core.CGSchemaStats; +import io.nosqlbench.cqlgen.model.CqlModel; +import io.nosqlbench.cqlgen.model.CqlTable; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class CGRatioSuffixer implements CGModelTransformer, NBConfigurable { + + private double resolution; + private String format; + + @Override + public CqlModel apply(CqlModel model) { + CGSchemaStats schemastats = model.getStats(); + if (schemastats == null) { + return model; + } + + for (CqlTable tableDef : model.getTableDefs()) { + double opshare = tableDef.getComputedStats().getOpShareOfTotalOps(); + String newname = String.format(this.format, tableDef.getName(), opshare); + model.renameTable(tableDef,newname); + } + + return model; + } + + + @Override + public void applyConfig(NBConfiguration cfg) { + this.format = cfg.get("format", String.class); + if (!format.contains("NAME")) { + throw new RuntimeException("format config param for the CGRatioSuffixer must contain 'NAME', but it is '" + format + "'"); + } + Pattern pattern = Pattern.compile("%(2\\$)?(?\\d+)d"); + Matcher matcher = pattern.matcher(format); + if (!matcher.matches()) { + throw new RuntimeException("Could not find the required decimal format specifier for the format config parameter of " + CGRatioSuffixer.class); + } + this.resolution = Double.parseDouble(matcher.group("resolution")); + + } + + @Override + public NBConfigModel getConfigModel() { + return ConfigModel.of(CGRatioSuffixer.class) + .add(Param.defaultTo("format","%1$s_%2$5d").setDescription( + "The format specifier as in Java String.format, with a required string format for the first arg, and a required decimal format for the second." + )) + .asReadOnly(); + } +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGRegexReplacer.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRegexReplacer.java similarity index 96% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGRegexReplacer.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRegexReplacer.java index d992d9219..0d414bcba 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGRegexReplacer.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGRegexReplacer.java @@ -14,9 +14,10 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.transformers; +package io.nosqlbench.cqlgen.transformers; -import io.nosqlbench.cqlgen.exporter.CGTextTransformer; +import io.nosqlbench.cqlgen.api.CGTransformerConfigurable; +import io.nosqlbench.cqlgen.api.CGTextTransformer; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGReplicationSettingInjector.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGReplicationSettingInjector.java similarity index 90% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGReplicationSettingInjector.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGReplicationSettingInjector.java index d0462b1e3..cb1a8b658 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGReplicationSettingInjector.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGReplicationSettingInjector.java @@ -14,8 +14,10 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.transformers; +package io.nosqlbench.cqlgen.transformers; +import io.nosqlbench.cqlgen.api.CGModelTransformer; +import io.nosqlbench.cqlgen.api.CGTransformerConfigurable; import io.nosqlbench.cqlgen.model.CqlKeyspace; import io.nosqlbench.cqlgen.model.CqlModel; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGUdtReplacer.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGUdtReplacer.java similarity index 93% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGUdtReplacer.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGUdtReplacer.java index 6d5d99e51..4dbbebe04 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CGUdtReplacer.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CGUdtReplacer.java @@ -14,8 +14,9 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.transformers; +package io.nosqlbench.cqlgen.transformers; +import io.nosqlbench.cqlgen.api.CGModelTransformer; import io.nosqlbench.cqlgen.model.CqlColumnDef; import io.nosqlbench.cqlgen.model.CqlModel; import io.nosqlbench.cqlgen.model.CqlTable; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/ComputedTableStats.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/ComputedTableStats.java new file mode 100644 index 000000000..3e3d49098 --- /dev/null +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/ComputedTableStats.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2022 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.cqlgen.transformers; + +public class ComputedTableStats { + private double readShareOfTotalOps; + private double readShareOfTotalReads; + private double writeShareOfTotalOps; + private double writeShareOfTotalWrites; + private double opShareOfTotalOps; + private double spaceUsedOfTotalSpace; + + public ComputedTableStats setReadShareOfTotalOps(double v) { + this.readShareOfTotalOps = v; + return this; + } + + public ComputedTableStats setReadShareOfTotalReads(double v) { + this.readShareOfTotalReads =v; + return this; + } + + public ComputedTableStats setWriteShareOfTotalOps(double v) { + this.writeShareOfTotalOps = v; + return this; + } + + public ComputedTableStats setWriteShareOfTotalWrites(double v) { + this.writeShareOfTotalWrites = v; + return this; + } + + public ComputedTableStats setOpShareOfTotalOps(double op_share_total) { + this.opShareOfTotalOps = op_share_total; + return this; + } + + public ComputedTableStats setSpaceUsedOfTotalSpace(double weightedSpace) { + this.spaceUsedOfTotalSpace = weightedSpace; + return this; + } + + public double getOpShareOfTotalOps() { + return opShareOfTotalOps; + } +} diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CqlSchemaStatsParser.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CqlSchemaStatsParser.java similarity index 94% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CqlSchemaStatsParser.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CqlSchemaStatsParser.java index 60a00fec8..b4c824644 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/CqlSchemaStatsParser.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/CqlSchemaStatsParser.java @@ -14,11 +14,11 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.transformers; +package io.nosqlbench.cqlgen.transformers; -import io.nosqlbench.cqlgen.exporter.CGKeyspaceStats; -import io.nosqlbench.cqlgen.exporter.CGSchemaStats; -import io.nosqlbench.cqlgen.exporter.CGTableStats; +import io.nosqlbench.cqlgen.core.CGKeyspaceStats; +import io.nosqlbench.cqlgen.core.CGSchemaStats; +import io.nosqlbench.cqlgen.core.CGTableStats; import org.apache.commons.math4.util.Pair; import java.io.BufferedReader; diff --git a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/UnusedTableRemover.java b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/UnusedTableRemover.java similarity index 94% rename from adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/UnusedTableRemover.java rename to adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/UnusedTableRemover.java index 127b20106..2ee8cc55d 100644 --- a/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/exporter/transformers/UnusedTableRemover.java +++ b/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/transformers/UnusedTableRemover.java @@ -14,8 +14,10 @@ * limitations under the License. */ -package io.nosqlbench.cqlgen.exporter.transformers; +package io.nosqlbench.cqlgen.transformers; +import io.nosqlbench.cqlgen.api.CGModelTransformer; +import io.nosqlbench.cqlgen.api.CGTransformerConfigurable; import io.nosqlbench.cqlgen.model.CqlModel; import io.nosqlbench.cqlgen.model.CqlTable; import org.apache.logging.log4j.LogManager; diff --git a/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf b/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf index 8502bd011..b62763cdf 100644 --- a/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf +++ b/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf @@ -8,13 +8,13 @@ # outfile: _replaced.cql # replacers: # - - '^(\s*?)\b(options|role|roles|permissions|permission|date|key|timestamp|type|keys)\b(\s+[a-zA-Z][a-zA-Z<>,_ -]*?,?)$' -# - '$1PREFIX$2SUFFIX$3' +# - '$1"$2"$3' # - - '^(.*?PRIMARY KEY.*?)\b(options|role|roles|permissions|permission|date|key|timestamp|type|keys)\b(.*?)$' -# - '$1PREFIX$2SUFFIX$3' +# - '$1"$2"$3' # - - '^(.*?CLUSTERING ORDER BY.+?)\b(options|role|roles|permissions|permission|date|key|timestamp|type|keys)\b(.*?)$' -# - '$1PREFIX$2SUFFIX$3' +# - '$1"$2"$3' # - - '^(\s*?CREATE TABLE.+?)\b(options|role|roles|permissions|permission|date|key|timestamp|type|keys)\b(.*?)$' -# - '$1PREFIX$2SUFFIX$3' +# - '$1"$2"$3' # model_transformers: @@ -27,7 +27,7 @@ model_transformers: - exclude: dsefs_.* - exclude: cfs_.* - exclude: cfs - - exclude: HiveMetaStore + - exclude: Hive.* - exclude: spark_system - include: .* @@ -64,13 +64,13 @@ model_transformers: # as usual. - class: CGRatioCalculator - # if this is set, and the fractional rate of operations against a table - # counting reads and writes is less than the percent threshold, then - # the table will be excluded from all op template generation. - - class: UnusedTableRemover - config: - # this is as a fractional number, so 0.1 is the same as 10% - minimum_threshold: 0.001 +# # if this is set, and the fractional rate of operations against a table +# # counting reads and writes is less than the percent threshold, then +# # the table will be excluded from all op template generation. +# - class: UnusedTableRemover +# config: +# # this is as a fractional number, so 0.1 is the same as 10% +# minimum_threshold: 0.001 # replaces names of keyspaces, tables, and columns with generated values - class: CGNameObfuscator @@ -126,10 +126,10 @@ blockplan: # not needed when tags=block:'drop.*' # drop: drop-types, drop-tables, drop-keyspaces rampup: insert - main-insert: insert - main-select: select - main-scan: scan-10 - main-update: update + main-insert: insert-seq + main-select: select-seq + main-scan: scan-10-seq + main-update: update-seq # not needed when tags=block:'main.*' # main: insert, select, scan-10, update diff --git a/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/cql/parser/CqlParserHarnessTest.java b/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/cql/parser/CqlParserHarnessTest.java index a47b7a423..d5b9044f2 100644 --- a/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/cql/parser/CqlParserHarnessTest.java +++ b/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/cql/parser/CqlParserHarnessTest.java @@ -16,16 +16,11 @@ package io.nosqlbench.converters.cql.cql.parser; -import io.nosqlbench.cqlgen.exporter.CGWorkloadExporter; -import io.nosqlbench.cqlgen.exporter.transformers.CGModelTransformers; +import io.nosqlbench.cqlgen.core.CGWorkloadExporter; import io.nosqlbench.cqlgen.parser.CqlModelParser; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import java.nio.file.Path; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; - public class CqlParserHarnessTest { private final static String ksddl = """ @@ -47,19 +42,13 @@ public class CqlParserHarnessTest { @Test public void testAllTypes() { - CGWorkloadExporter exporter = new CGWorkloadExporter(Path.of("src/test/resources/testschemas/cql_alltypes.cql"), new CGModelTransformers()); + CGWorkloadExporter exporter = new CGWorkloadExporter(); + exporter.appMain(new String[]{"src/test/resources/testschemas/cql_alltypes.cql"}); exporter.setNamingTemplate("[OPTYPE-][COLUMN-][TYPEDEF-][TABLE!]-[KEYSPACE]"); var data = exporter.getWorkloadAsYaml(); } - @Disabled - @Test - public void testGenBasicWorkload() { - CGWorkloadExporter exporter = new CGWorkloadExporter(ddl, new CGModelTransformers()); - assertThatThrownBy(() -> exporter.getWorkloadAsYaml()).isInstanceOf(RuntimeException.class); - } - @Test public void testCqlParserHarnessCombined() { CqlModelParser.parse(ddl, null); diff --git a/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/CGElementNamerTest.java b/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/CGElementNamerTest.java index 549da0761..01b9141e8 100644 --- a/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/CGElementNamerTest.java +++ b/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/CGElementNamerTest.java @@ -17,7 +17,7 @@ package io.nosqlbench.converters.cql.exporters; import io.nosqlbench.api.labels.Labeled; -import io.nosqlbench.cqlgen.exporter.CGElementNamer; +import io.nosqlbench.cqlgen.core.CGElementNamer; import org.junit.jupiter.api.Test; import java.util.Map; diff --git a/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/CGWorkloadExporterTest.java b/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/CGWorkloadExporterTest.java index 20bd1dd14..afacfeea0 100644 --- a/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/CGWorkloadExporterTest.java +++ b/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/CGWorkloadExporterTest.java @@ -16,7 +16,7 @@ package io.nosqlbench.converters.cql.exporters; -import io.nosqlbench.cqlgen.exporter.CGColumnRebinder; +import io.nosqlbench.cqlgen.core.CGColumnRebinder; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; diff --git a/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/NamingFolioTest.java b/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/NamingFolioTest.java index 8c37ae8d6..8bfdce609 100644 --- a/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/NamingFolioTest.java +++ b/adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/exporters/NamingFolioTest.java @@ -16,7 +16,7 @@ package io.nosqlbench.converters.cql.exporters; -import io.nosqlbench.cqlgen.exporter.binders.NamingFolio; +import io.nosqlbench.cqlgen.binders.NamingFolio; import org.junit.jupiter.api.Test; import java.util.Map;