checkpoint:

* fixed binding null issue with warning for none found
* configurable naming support in obfuscator
* partial implementation of text transformer
This commit is contained in:
Jonathan Shook
2022-07-17 17:06:10 -05:00
parent 562978c9cc
commit fbd8947e7a
17 changed files with 280 additions and 46 deletions

View File

@@ -26,11 +26,11 @@ public class CqlColumnDef implements NBNamedElement, Labeled {
private String keyspace;
private String name;
private String type;
private int position;
private ColType coltype;
private final int position;
public CqlColumnDef(CqlTable table, int position, String colname, String typedef) {
this.table = table;
this.position=position;
this.type = typedef;
this.name = colname;
}

View File

@@ -0,0 +1,25 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters;
import java.util.function.Function;
public interface CGTextTransformer extends Function<String,String> {
@Override
String apply(String input);
}

View File

@@ -0,0 +1,93 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters;
import io.nosqlbench.converters.cql.exporters.transformers.CGNameObfuscator;
import io.nosqlbench.converters.cql.exporters.transformers.CGTransformerConfigurable;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Supplier;
public class CGTextTransformers implements Consumer<List<Map<String, ?>>>, Supplier<List<CGTextTransformer>> {
private final static Logger logger = LogManager.getLogger(CGTextTransformers.class);
private final List<CGTextTransformer> transformers = new ArrayList<>();
@Override
public List<CGTextTransformer> get() {
return transformers;
}
@Override
public void accept(List<Map<String, ?>> configs) {
List<CGTextTransformer> transformers = new ArrayList<>();
for (Map<String, ?> cfgmap : configs) {
// Instantiate Transformer
String classname = cfgmap.get("class").toString();
if (!classname.contains(".")) {
String newname = CGNameObfuscator.class.getPackageName() + "." + classname;
logger.info("qualified transformer '" + classname + "' as '" + newname + "'");
classname = newname;
}
Class<?> txclass = null;
CGTextTransformer transformer = null;
try {
txclass = Class.forName(classname);
Constructor<?> ctor = txclass.getConstructor();
Object instance = ctor.newInstance();
if (instance instanceof CGTextTransformer t) {
transformer = t;
} else {
throw new RuntimeException("Object " + instance.getClass().getName() + " is not a " + CGTextTransformer.class.getName());
}
} catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException e) {
throw new RuntimeException(e);
}
// Configure Transformer IFF ...
if (transformer instanceof CGTransformerConfigurable configurable) {
Object cfgvalues = cfgmap.get("config");
if (cfgvalues instanceof Map txconfigmap) {
configurable.accept((txconfigmap));
logger.info("configured transformer with " + txconfigmap);
}
}
transformers.add(transformer);
}
this.transformers.addAll(transformers);
}
public String process(String content) {
for (CGTextTransformer transformer : transformers) {
content = transformer.apply(content);
}
return content;
}
}

View File

@@ -20,7 +20,7 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.converters.cql.cqlast.*;
import io.nosqlbench.converters.cql.exporters.binders.*;
import io.nosqlbench.converters.cql.exporters.transformers.CGTransformersInit;
import io.nosqlbench.converters.cql.exporters.transformers.CGModelTransformers;
import io.nosqlbench.converters.cql.parser.CqlModelParser;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -75,7 +75,7 @@ public class CGWorkloadExporter {
private boolean elideUnusedTables;
private Map<String, List<String>> blockplan = Map.of();
public CGWorkloadExporter(CqlModel model, CGTransformersInit transformers) {
public CGWorkloadExporter(CqlModel model, CGModelTransformers transformers) {
this.model = model;
for (Function<CqlModel, CqlModel> transformer : transformers.get()) {
CqlModel modified = transformer.apply(this.model);
@@ -83,18 +83,29 @@ public class CGWorkloadExporter {
}
}
public CGWorkloadExporter(String ddl, Path srcpath, CGTransformersInit transformers) {
public CGWorkloadExporter(String ddl, Path srcpath, CGModelTransformers transformers) {
this(CqlModelParser.parse(ddl, srcpath), transformers);
}
public CGWorkloadExporter(String ddl, CGTransformersInit transformers) {
public CGWorkloadExporter(String ddl, CGModelTransformers transformers) {
this(ddl, null, transformers);
}
public CGWorkloadExporter(Path path, CGTransformersInit transformers) {
public CGWorkloadExporter(Path path, CGModelTransformers transformers) {
this(CqlModelParser.parse(path), transformers);
}
private String loadFile(Path path) {
try {
String ddl = Files.readString(path);
logger.info("read " + ddl.length() + " character DDL file");
return ddl;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static void main(String[] args) {
logger.info("running CQL workload exporter with args:" + Arrays.toString(args));
@@ -132,14 +143,32 @@ public class CGWorkloadExporter {
CGWorkloadExporter exporter;
if (Files.exists(cfgpath)) {
try {
CGTransformersInit transformers = new CGTransformersInit();
CGModelTransformers modelTransformers = new CGModelTransformers();
CGTextTransformers textTransformers = new CGTextTransformers();
String configfile = Files.readString(cfgpath);
Map cfgmap = yaml.loadAs(configfile, Map.class);
if (cfgmap.containsKey("model_transformers")) {
transformers.accept((List<Map<String, ?>>) cfgmap.get("model_transformers"));
modelTransformers.accept((List<Map<String, ?>>) cfgmap.get("model_transformers"));
}
exporter = new CGWorkloadExporter(srcpath, transformers);
Object txtr = cfgmap.get("text_transformers");
if (txtr!=null) {
textTransformers.accept((List<Map<String,?>>) cfgmap.get("text_transformers"));
}
String ddl;
try {
ddl = Files.readString(srcpath);
logger.info("read " + ddl.length() + " character DDL file, parsing");
if (textTransformers!=null) {
ddl = textTransformers.process(ddl);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
exporter = new CGWorkloadExporter(ddl, srcpath, modelTransformers);
String defaultNamingTemplate = cfgmap.get("naming_template").toString();
exporter.setNamingTemplate(defaultNamingTemplate);
@@ -286,13 +315,14 @@ public class CGWorkloadExporter {
private LinkedHashMap<String, Object> genScenarios(CqlModel model) {
return new LinkedHashMap<>() {{
put("default",
new LinkedHashMap<>() {{
put("schema", "run driver=cql tags=block:schema-.* threads===UNDEF cycles===UNDEF");
put("rampup", "run driver=cql tags=block:rampup-.* threads=auto cycles===TEMPLATE(rampup-cycles,10000)");
put("main", "run driver=cql tags=block:main-.* threads=auto cycles===TEMPLATE(main-cycles,10000)");
}});
put("truncate", "run driver=cql tags=block:truncate-.* threads===UNDEF cycles===UNDEF");
put("schema-keyspaces", "run driver=cql tags=block:schema-keyspaces threads===UNDEF cycles===UNDEF");
put("schema-types", "run driver=cql tags=block:schema-types threads===UNDEF cycles===UNDEF");
@@ -324,7 +354,7 @@ public class CGWorkloadExporter {
private String genScanSyntax(CqlTable table) {
return """
select * from KEYSPACE.TABLE
select * from KEYSPACE.TABLE
where PREDICATE
LIMIT;
"""
@@ -429,7 +459,7 @@ public class CGWorkloadExporter {
return bindings.forColumn(columnDef);
}
modulo = quantizeModuloByMagnitude(modulo,1);
logger.info("Set partition modulo for " + tableDef.getFullName() + " to " + modulo);
logger.debug("Set partition modulo for " + tableDef.getFullName() + " to " + modulo);
Binding binding = bindings.forColumn(columnDef, "Mod(" + modulo + "L); ");
return binding;
}
@@ -477,7 +507,6 @@ public class CGWorkloadExporter {
return 1;
}
return readRatioFor(table) + writeRatioFor(table);
}
private int readRatioFor(CqlTable table) {
@@ -624,7 +653,7 @@ public class CGWorkloadExporter {
ops.put(
namer.nameFor(table, "optype", "drop", "blockname", blockname),
Map.of(
"simple", "drop table " + table.getFullName() + ";",
"simple", "drop table if exists " + table.getFullName() + ";",
"timeout", timeouts.get("drop")
)
);
@@ -640,7 +669,7 @@ public class CGWorkloadExporter {
ops.put(
namer.nameFor(type, "optype", "drop-type", "blockname", blockname),
Map.of(
"simple", "drop type " + type.getKeyspace() + "." + type.getName() + ";",
"simple", "drop type if exists " + type.getKeyspace() + "." + type.getName() + ";",
"timeout", timeouts.get("drop")
)
);
@@ -656,7 +685,7 @@ public class CGWorkloadExporter {
ops.put(
namer.nameFor(type, "optype", "drop-keyspace", "blockname", blockname),
Map.of(
"simple", "drop keyspace " + type.getKeyspace() + ";",
"simple", "drop keyspace if exists " + type.getKeyspace() + ";",
"timeout", timeouts.get("drop")
)
);
@@ -677,7 +706,6 @@ public class CGWorkloadExporter {
"simple", "truncate " + table.getFullName() + ";",
"timeout", timeouts.get("truncate")
)
);
}
return truncateblock;

View File

@@ -79,6 +79,9 @@ public class BindingsAccumulator {
for (BindingsLibrary library : libraries) {
Optional<Binding> binding = library.resolveBindingsFor(def);
if (binding.isPresent()) {
if (binding.get().getRecipe()==null) {
throw new RuntimeException("Binding returned from library " + library + "' was null, for def '" + def + "'");
}
return binding;
}
}

View File

@@ -14,6 +14,8 @@ bindings:
frozen<list<int>>: ListSizedHashed(HashRange(3,7),ToInt()));
list<text>: ListStepped(NumberNameToString(),NumberNameToString())
map<text,text>: MapSized(3, Combinations('A-Z;0-9'), NumberNameToString(), ToString());
map<int,int>: MapSized(3, ToInt(), ToInt());
counter: HashRange(1,3);
set<text>: SetSized(HashRange(3,4),NumberNameToString()));
smallint: ToShort();
time: StartingEpochMillis('2022-01-01 00:00:00'); ToLocalTime();

View File

@@ -1,9 +1,15 @@
# unimplemented
#text_transformers:
text_transformers:
# - class: CGRegexReplacer
# config:
# replacers:
# - /a/b/
# - - '(\s*)(options|role|roles|permissions|permission|date|key|timestamp|type|keys) ([a-zA-Z][a-zA-Z<>_-]*)(,?)'
# - '$1base$2 $3$4'
# - - '(.*PRIMARY KEY .*?)\b(options|role|roles|permissions|permission|date|key|timestamp|type|keys)\b(.*)'
# - '$1base$2$3'
# - - '(.*CLUSTERING ORDER BY.+?)\b(options|role|roles|permissions|permission|date|key|timestamp|type|keys)\b(.*)'
# - '$1base$2$3'
# - - '(.*CREATE TABLE system_auth\.)(options|role|roles|permissions|permission|date|key|timestamp|type|keys)\b(.*)'
# - '$1base$2$3'
model_transformers:
@@ -12,9 +18,9 @@ model_transformers:
config:
include:
- dba_info
- vzw_order
- mcs_or_prod
- prod_parallel_test
- vzw_order
- vzw_common
- vzw_soe
@@ -48,6 +54,8 @@ model_transformers:
# replaces names of keyspaces, tables, and columns with generated values
- class: CGNameObfuscator
config:
namer: Combinations('0-9;0-9;0-9;0-9;0-9');
naming_template: "[OPTYPE-][KEYSPACE-][TYPE-][NAME]"

View File

@@ -26,7 +26,7 @@ import java.util.function.Function;
import java.util.function.LongFunction;
public class CGCachingNameRemapper {
private final LongFunction<String> namefunc;
private LongFunction<String> namefunc;
private final Map<String,String> remapped = new HashMap<>();
private long index=0;
@@ -57,4 +57,8 @@ public class CGCachingNameRemapper {
public Function<String, String> mapperForType(Labeled cqlTable, String prefix) {
return in -> this.nameForType(cqlTable.getClass().getSimpleName(),in, prefix);
}
public void setNamingFunction(LongFunction<String> namerFunc) {
this.namefunc = namerFunc;
}
}

View File

@@ -23,7 +23,7 @@ import java.io.IOException;
import java.nio.file.Path;
import java.util.Map;
public class CGGenStatsInjector implements CGModelTransformer, CGTransformerConfigType {
public class CGGenStatsInjector implements CGModelTransformer, CGTransformerConfigurable {
private CGSchemaStats schemaStats = null;
public CGGenStatsInjector() {

View File

@@ -25,7 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfigType {
public class CGKeyspaceFilter implements CGModelTransformer, CGTransformerConfigurable {
private List<Pattern> patterns = List.of(Pattern.compile(".*"));
private final static Logger logger = LogManager.getLogger(CGKeyspaceFilter.class);

View File

@@ -27,11 +27,11 @@ import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Supplier;
public class CGTransformersInit implements Consumer<List<Map<String, ?>>>, Supplier<List<CGModelTransformer>> {
private final static Logger logger = LogManager.getLogger(CGTransformersInit.class);
private List<CGModelTransformer> transformers = new ArrayList<>();
public class CGModelTransformers implements Consumer<List<Map<String, ?>>>, Supplier<List<CGModelTransformer>> {
private final static Logger logger = LogManager.getLogger(CGModelTransformers.class);
private final List<CGModelTransformer> transformers = new ArrayList<>();
public CGTransformersInit() {
public CGModelTransformers() {
}
@Override
@@ -65,7 +65,7 @@ public class CGTransformersInit implements Consumer<List<Map<String, ?>>>, Suppl
// Configure Transformer IFF ...
if (transformer instanceof CGTransformerConfigType configurable) {
if (transformer instanceof CGTransformerConfigurable configurable) {
Object cfgvalues = cfgmap.get("config");
if (cfgvalues instanceof Map txconfigmap) {
configurable.accept((txconfigmap));

View File

@@ -25,14 +25,19 @@ package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlTable;
import io.nosqlbench.converters.cql.cqlast.CqlType;
import io.nosqlbench.virtdata.core.bindings.DataMapper;
import io.nosqlbench.virtdata.core.bindings.VirtData;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public class CGNameObfuscator implements CGModelTransformer {
import java.util.Map;
import java.util.Optional;
import java.util.function.LongFunction;
public class CGNameObfuscator implements CGModelTransformer, CGTransformerConfigurable {
private final static Logger logger = LogManager.getLogger(CGNameObfuscator.class);
private final CGCachingNameRemapper remapper = new CGCachingNameRemapper();
private Object keyspaceName;
@Override
public CqlModel apply(CqlModel model) {
@@ -56,8 +61,17 @@ public class CGNameObfuscator implements CGModelTransformer {
type.renameColumns(remapper.mapperForType(type, "typ"));
}
return model;
}
@Override
public void accept(Map<String, ?> stringMap) {
Object namer = stringMap.get("namer");
Optional<DataMapper<String>> optionalMapper = VirtData.getOptionalMapper(namer.toString());
LongFunction<String> namerFunc = optionalMapper.orElseThrow(
() -> new RuntimeException("Unable to resolve obfuscator namer '" + namer + "'")
);
remapper.setNamingFunction(namerFunc);
}
}

View File

@@ -0,0 +1,61 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters.transformers;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class CGRegexReplacer implements Function<String,String>, CGTransformerConfigurable {
@Override
public String apply(String s) {
return null;
}
@Override
public void accept(Map<String, ?> stringMap) {
List<List<String>> replacers = (List<List<String>>) stringMap.get("replacers");
}
private final static class Replacer implements Function<String,String>{
private final Pattern pattern;
private final String replacement;
Replacer(String from, String to) {
this.pattern = Pattern.compile(from);
this.replacement = to;
}
@Override
public String apply(String s) {
Matcher matcher = pattern.matcher(s);
StringBuilder sb = new StringBuilder();
while (matcher.find()) {
matcher.appendReplacement(sb,replacement);
}
matcher.appendTail(sb);
return sb.toString();
}
}
}

View File

@@ -21,7 +21,7 @@ import io.nosqlbench.converters.cql.cqlast.CqlModel;
import java.util.Map;
public class CGReplicationSettingInjector implements CGModelTransformer, CGTransformerConfigType {
public class CGReplicationSettingInjector implements CGModelTransformer, CGTransformerConfigurable {
private String replicationFields;
@Override

View File

@@ -19,5 +19,5 @@ package io.nosqlbench.converters.cql.exporters.transformers;
import java.util.Map;
import java.util.function.Consumer;
public interface CGTransformerConfigType extends Consumer<Map<String, ?>> {
public interface CGTransformerConfigurable extends Consumer<Map<String, ?>> {
}

View File

@@ -42,7 +42,7 @@ public class CqlModelParser {
try {
String ddl = Files.readString(path);
logger.info("read " + ddl.length() + " character DDL file, parsing");
CqlModel parsed = parse(ddl, path);
CqlModel parsed = parse(ddl, null);
logger.info("parsed cql model: " + parsed.getSummaryLine());
return parsed;
@@ -61,7 +61,6 @@ public class CqlModelParser {
}
public static CqlModel parse(String input, Path origin) {
try {
CodePointCharStream cstream = CharStreams.fromString(input);
CGErrorListener errorListener = new CGErrorListener(origin);
@@ -91,7 +90,4 @@ public class CqlModelParser {
}
}
public static void parse(String ddl) {
parse(ddl,null);
}
}

View File

@@ -17,7 +17,7 @@
package io.nosqlbench.converters.cql.cql.parser;
import io.nosqlbench.converters.cql.exporters.CGWorkloadExporter;
import io.nosqlbench.converters.cql.exporters.transformers.CGTransformersInit;
import io.nosqlbench.converters.cql.exporters.transformers.CGModelTransformers;
import io.nosqlbench.converters.cql.parser.CqlModelParser;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
@@ -47,7 +47,7 @@ public class CqlParserHarnessTest {
@Test
public void testAllTypes() {
CGWorkloadExporter exporter = new CGWorkloadExporter(Path.of("src/test/resources/testschemas/cql_alltypes.cql"), new CGTransformersInit());
CGWorkloadExporter exporter = new CGWorkloadExporter(Path.of("src/test/resources/testschemas/cql_alltypes.cql"), new CGModelTransformers());
exporter.setNamingTemplate("[OPTYPE-][COLUMN-][TYPEDEF-][TABLE!]-[KEYSPACE]");
var data = exporter.getWorkloadAsYaml();
@@ -56,13 +56,13 @@ public class CqlParserHarnessTest {
@Disabled
@Test
public void testGenBasicWorkload() {
CGWorkloadExporter exporter = new CGWorkloadExporter(ddl, new CGTransformersInit());
CGWorkloadExporter exporter = new CGWorkloadExporter(ddl, new CGModelTransformers());
assertThatThrownBy(() -> exporter.getWorkloadAsYaml()).isInstanceOf(RuntimeException.class);
}
@Test
public void testCqlParserHarnessCombined() {
CqlModelParser.parse(ddl);
CqlModelParser.parse(ddl, null);
}
@Disabled
@@ -75,7 +75,7 @@ public class CqlParserHarnessTest {
'class' : 'SimpleStrategy',\s
'replication_factor' : 1\s
};
""");
""", null);
}
@Test
@@ -88,7 +88,7 @@ public class CqlParserHarnessTest {
race_position int,\s
cyclist_name FROZEN<fullname>,\s
PRIMARY KEY (race_name, race_position));
""");
""", null);
}