IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

first cut of obfuscator

Browse Source
This commit is contained in:
Jonathan Shook 2022-07-15 11:26:33 -05:00
parent 4e08b65f2d
commit 2af5f2c39a
12 changed files with 182 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlColumnDef.java
View File

@ -16,16 +16,17 @@
package io.nosqlbench.converters.cql.cqlast;
import io.nosqlbench.api.config.NBNamedElement;
import io.nosqlbench.api.labels.Labeled;
import java.util.Map;
public class CqlColumnDef implements Labeled {
public class CqlColumnDef implements NBNamedElement, Labeled {
private String refDefinitionDdl;
private String refTypeDefddl;
private String table;
private String keyspace;
private final String name;
private String name;
private String type;
public CqlColumnDef(String colname, String typedef, String refColumnDdl) {
@ -95,4 +96,8 @@ public class CqlColumnDef implements Labeled {
public boolean isCounter() {
return getTrimmedTypedef().equalsIgnoreCase("counter");
}
public void setName(String name) {
this.name = name;
}
}

5
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlKeyspace.java
View File

@ -16,12 +16,13 @@
package io.nosqlbench.converters.cql.cqlast;
import io.nosqlbench.api.config.NBNamedElement;
import io.nosqlbench.api.labels.Labeled;
import java.util.HashMap;
import java.util.Map;
public class CqlKeyspace implements Labeled {
public class CqlKeyspace implements NBNamedElement, Labeled {
String keyspaceName= "";
String refddl;
private String refReplDdl;
@ -44,7 +45,7 @@ public class CqlKeyspace implements Labeled {
this.keyspaceName=name;
}
public String getKeyspaceName() {
public String getName() {
return this.keyspaceName;
}

24
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModel.java
View File

@ -58,7 +58,7 @@ public class CqlModel {
public void saveTable(String keyspace, String text, String refddl) {
table.setKeyspace(keyspace);
table.setTable(text);
table.setName(text);
table.setRefDdl(refddl);
this.tables.computeIfAbsent(keyspace, ks->new LinkedHashMap<>()).put(text, table);
table = null;
@ -71,15 +71,26 @@ public class CqlModel {
}
}
public Map<String, CqlKeyspace> getKeyspaces() {
public Map<String, CqlKeyspace> getKeyspacesByName() {
return keyspaces;
}
public Map<String, Map<String, CqlTable>> getTablesByKeyspace() {
public List<CqlKeyspace> getKeyspaces() {
return new ArrayList<>(this.keyspaces.values());
}
public Map<String, Map<String, CqlTable>> getTablesByNameByKeyspace() {
return tables;
}
public List<CqlTable> getAllTables() {
public List<CqlTable> getTablesForKeyspace(String ksname) {
Map<String, CqlTable> tables = this.tables.get(ksname);
if (tables!=null) {
return new ArrayList<>(tables.values());
}
}
public List<CqlTable> getTables() {
return tables.values().stream().flatMap(m->m.values().stream()).toList();
}
@ -88,12 +99,12 @@ public class CqlModel {
StringBuilder sb = new StringBuilder();
for (String ks : keyspaces.keySet()) {
CqlKeyspace keyspace = keyspaces.get(ks);
sb.append("keyspace '").append(keyspace.getKeyspaceName()).append("':\n");
sb.append("keyspace '").append(keyspace.getName()).append("':\n");
sb.append(keyspace).append("\n");
tables.getOrDefault(ks,Map.of()).values().stream()
.forEach(table -> {
sb.append("table '").append(table.getTableName()).append("':\n");
sb.append("table '").append(table.getName()).append("':\n");
sb.append(table);
});
}
@ -149,6 +160,5 @@ public class CqlModel {
}
}
return list;
}
}

41
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlTable.java
View File

@ -16,18 +16,24 @@
package io.nosqlbench.converters.cql.cqlast;
import io.nosqlbench.api.config.NBNamedElement;
import io.nosqlbench.api.labels.Labeled;
import java.util.*;
import java.util.stream.Collectors;
public class CqlTable implements Labeled {
String table = "";
public class CqlTable implements NBNamedElement, Labeled {
String name = "";
String keyspace = "";
List<CqlColumnDef> coldefs = new ArrayList<>();
Map<String, String> tableAttributes = new HashMap<String, String>();
List<String> partitionKeys = new ArrayList<>();
List<String> clusteringColumns = new ArrayList<>();
private String refddl;
public CqlTable() {
}
public Map<String, String> getTableAttributes() {
return tableAttributes;
}
@ -36,20 +42,12 @@ public class CqlTable implements Labeled {
this.tableAttributes = tableAttributes;
}
Map<String,String> tableAttributes = new HashMap<String,String>();
List<String> partitionKeys = new ArrayList<>();
List<String> clusteringColumns = new ArrayList<>();
public CqlTable() {
}
public void addcolumnDef(CqlColumnDef cqlField) {
this.coldefs.add(cqlField);
}
public void setTable(String tableName) {
this.table = tableName;
public void setName(String tableName) {
this.name = tableName;
for (CqlColumnDef coldef : coldefs) {
coldef.setTable(tableName);
}
@ -61,10 +59,10 @@ public class CqlTable implements Labeled {
@Override
public String toString() {
return "cql table: '" + this.table + "':\n"
return "cql table: '" + this.name + "':\n"
+ this.coldefs.stream()
.map(Object::toString)
.map(s -> " " +s)
.map(s -> " " + s)
.collect(Collectors.joining("\n"));
}
@ -72,12 +70,12 @@ public class CqlTable implements Labeled {
return this.coldefs;
}
public String getTableName() {
return this.table;
public String getName() {
return this.name;
}
public void setKeyspace(String keyspace) {
this.keyspace=keyspace;
this.keyspace = keyspace;
for (CqlColumnDef coldef : coldefs) {
coldef.setKeyspace(keyspace);
}
@ -88,7 +86,7 @@ public class CqlTable implements Labeled {
}
public void setRefDdl(String refddl) {
this.refddl=refddl;
this.refddl = refddl;
}
public String getRefddl() {
@ -104,7 +102,7 @@ public class CqlTable implements Labeled {
public Map<String, String> getLabels() {
return Map.of(
"keyspace", this.keyspace,
"table", this.table
"table", this.name
);
}
@ -131,7 +129,7 @@ public class CqlTable implements Labeled {
.findFirst();
if (!def.isPresent()) {
throw new RuntimeException("Unable to find column definition in table '" +
this.getTableName() + "' for column '" + colname + "'");
this.getName() + "' for column '" + colname + "'");
}
return def.orElseThrow();
}
@ -142,4 +140,5 @@ public class CqlTable implements Labeled {
.filter(n -> !clusteringColumns.contains(n.getName()))
.toList();
}
}

4
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlType.java
View File

@ -16,10 +16,12 @@
package io.nosqlbench.converters.cql.cqlast;
import io.nosqlbench.api.config.NBNamedElement;
import java.util.LinkedHashMap;
import java.util.Map;
public class CqlType {
public class CqlType implements NBNamedElement {
private String keyspace;
private String name;
private String refddl;

27
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlWorkloadExporter.java
View File

@ -25,6 +25,7 @@ import io.nosqlbench.converters.cql.cqlast.CqlTable;
import io.nosqlbench.converters.cql.exporters.binders.*;
import io.nosqlbench.converters.cql.exporters.transformers.CqlModelFixup;
import io.nosqlbench.converters.cql.exporters.transformers.RatioCalculator;
import io.nosqlbench.converters.cql.exporters.transformers.StatsEnhancer;
import io.nosqlbench.converters.cql.parser.CqlModelParser;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -190,7 +191,7 @@ public class CqlWorkloadExporter {
private Map<String, Object> genMainBlock(CqlModel model) {
Map<String, Object> mainOpTemplates = new LinkedHashMap<>();
for (CqlTable table : model.getAllTables()) {
for (CqlTable table : model.getTables()) {
if (!isCounterTable(table)) {
@ -206,7 +207,7 @@ public class CqlWorkloadExporter {
throw new RuntimeException("Unable to generate main insert template for table '" + table + "'");
}
} else {
logger.info("skipped insert for counter table '" + table.getTableName() + "'");
logger.info("skipped insert for counter table '" + table.getName() + "'");
}
Optional<String> updateTemplate = this.genUpdateTemplate(table);
@ -264,7 +265,7 @@ public class CqlWorkloadExporter {
Map<String, String> rampupOpTemplates = new LinkedHashMap<>();
for (CqlTable table : model.getAllTables()) {
for (CqlTable table : model.getTables()) {
if (!isCounterTable(table)) {
Optional<String> insert = genInsertTemplate(table);
if (insert.isPresent()) {
@ -281,7 +282,7 @@ public class CqlWorkloadExporter {
private Optional<String> genSelectTemplate(CqlTable table) {
try {
return Optional.of("select * from " + table.getKeySpace() + "." + table.getTableName() +
return Optional.of("select * from " + table.getKeySpace() + "." + table.getName() +
"\n WHERE " + genPredicateTemplate(table));
} catch (UnresolvedBindingException ube) {
logger.error(ube);
@ -325,7 +326,7 @@ public class CqlWorkloadExporter {
where PREDICATES;
"""
.replaceAll("KEYSPACE", table.getKeySpace())
.replaceAll("TABLE", table.getTableName())
.replaceAll("TABLE", table.getName())
.replaceAll("PREDICATES", genPredicateTemplate(table))
.replaceAll("ASSIGNMENTS", genAssignments(table)));
@ -355,7 +356,7 @@ public class CqlWorkloadExporter {
try {
List<CqlColumnDef> cdefs = table.getColumnDefinitions();
return Optional.of("insert into " +
table.getKeySpace() + "." + table.getTableName() + "\n" +
table.getKeySpace() + "." + table.getName() + "\n" +
" ( " + cdefs.stream().map(CqlColumnDef::getName)
.collect(Collectors.joining(" , ")) +
" )\n values\n (" +
@ -411,10 +412,10 @@ public class CqlWorkloadExporter {
Map<String, Object> ops = new LinkedHashMap<>();
truncateblock.put("ops", ops);
for (CqlTable table : model.getAllTables()) {
for (CqlTable table : model.getTables()) {
ops.put(
namer.nameFor(table, "optype", "truncate"),
"truncate " + table.getKeySpace() + "." + table.getTableName() + ";"
"truncate " + table.getKeySpace() + "." + table.getName() + ";"
);
}
return truncateblock;
@ -424,12 +425,12 @@ public class CqlWorkloadExporter {
Map<String, Object> schemablock = new LinkedHashMap<>();
Map<String, Object> ops = new LinkedHashMap<>();
for (CqlKeyspace ks : model.getKeyspaces().values()) {
ops.put("create-keyspace-" + ks.getKeyspaceName(), ks.getRefDdlWithReplFields(DEFAULT_REPLICATION));
for (CqlKeyspace ks : model.getKeyspacesByName().values()) {
ops.put("create-keyspace-" + ks.getName(), ks.getRefDdlWithReplFields(DEFAULT_REPLICATION));
}
for (String ksname : model.getTablesByKeyspace().keySet()) {
for (CqlTable cqltable : model.getTablesByKeyspace().get(ksname).values()) {
ops.put("create-table-" + ksname + "." + cqltable.getTableName(), cqltable.getRefDdl());
for (String ksname : model.getTablesByNameByKeyspace().keySet()) {
for (CqlTable cqltable : model.getTablesByNameByKeyspace().get(ksname).values()) {
ops.put("create-table-" + ksname + "." + cqltable.getName(), cqltable.getRefDdl());
}
}

2
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/binders/NamingFolio.java
View File

@ -77,7 +77,7 @@ public class NamingFolio {
public void populate(CqlModel model) {
for (CqlTable table : model.getAllTables()) {
for (CqlTable table : model.getTables()) {
for (CqlColumnDef coldef : table.getColumnDefinitions()) {
addFieldRef(coldef.getLabels());
}

2
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/CqlModelFixup.java
View File

@ -28,7 +28,7 @@ public class CqlModelFixup implements Function<CqlModel,CqlModel> {
@Override
public CqlModel apply(CqlModel model) {
List<String> toReplace = model.getTypes().stream().map(t -> t.getKeyspace() + "." + t.getName()).toList();
for (CqlTable table : model.getAllTables()) {
for (CqlTable table : model.getTables()) {
for (CqlColumnDef coldef : table.getColumnDefinitions()) {
String coldefDdl = coldef.getRefddl();
for (String searchFor : toReplace) {

46
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/NameRemapper.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.api.config.NBNamedElement;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_string.Combinations;
import java.util.HashMap;
import java.util.Map;
import java.util.function.LongFunction;
public class NameRemapper {
private final LongFunction<String> namefunc;
private final Map<String,String> remapped = new HashMap<>();
private long index=0;
public NameRemapper() {
this.namefunc = new Combinations("a-z;a-z;a-z;a-z;a-z;a-z;");
}
public NameRemapper(LongFunction<String> function) {
this.namefunc = function;
}
public synchronized String nameFor(NBNamedElement element) {
String canonical = element.getClass().getSimpleName()+"--"+element.getName();
if (!remapped.containsKey(canonical)) {
String newname = namefunc.apply(index++);
remapped.put(canonical,newname);
}
return remapped.get(canonical);
}
}

61
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/Obfuscator.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Create random but stable names for all elements.
* Use a deterministic method for creating names.
* once an element is named, use the same name throughout
* prefix each element type with a code for the type
*/
package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.converters.cql.cqlast.CqlColumnDef;
import io.nosqlbench.converters.cql.cqlast.CqlKeyspace;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlTable;
import java.util.function.Function;
public class Obfuscator implements Function<CqlModel,CqlModel> {
private final NameRemapper remapper = new NameRemapper();
@Override
public CqlModel apply(CqlModel model) {
for (CqlKeyspace keyspace : model.getKeyspaces()) {
String ksname = keyspace.getName();
String ksnewname = remapper.nameFor(keyspace);
keyspace.setKeyspaceName(ksnewname);
keyspace.setRefDdl();
for (CqlTable cqlTable : model.getTablesForKeyspace(keyspace.getName())) {
String tablename = cqlTable.getName();
String tbnewname = remapper.nameFor(cqlTable);
cqlTable.setName(tbnewname);
for (CqlColumnDef coldef : cqlTable.getColumnDefinitions()) {
String colname = coldef.getName();
String replacement = remapper.nameFor(coldef);
coldef.setName(replacement);
}
}
}
}
}

4
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/RatioCalculator.java
View File

@ -33,7 +33,7 @@ public class RatioCalculator implements Function<CqlModel,CqlModel> {
double totalSpace = 0.0d;
double totalOps=0.0d;
for (CqlTable table : model.getAllTables()) {
for (CqlTable table : model.getTables()) {
String local_read_count = table.getTableAttributes().get("Local read count");
double reads = Double.parseDouble(local_read_count);
totalReads+=reads;
@ -49,7 +49,7 @@ public class RatioCalculator implements Function<CqlModel,CqlModel> {
totalSpace+=space;
}
for (CqlTable table : model.getAllTables()) {
for (CqlTable table : model.getTables()) {
double reads = Double.parseDouble(table.getTableAttributes().get("Local read count"));
double writes = Double.parseDouble(table.getTableAttributes().get("Local write count"));

12
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/StatsEnhancer.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/transformers/StatsEnhancer.java
View File

@ -14,10 +14,12 @@
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters;
package io.nosqlbench.converters.cql.exporters.transformers;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlTable;
import io.nosqlbench.converters.cql.exporters.CqlKeyspaceStats;
import io.nosqlbench.converters.cql.exporters.CqlSchemaStats;
import java.util.Map;
import java.util.function.Function;
@ -34,13 +36,13 @@ public class StatsEnhancer implements Function<CqlModel, CqlModel> {
if (schemaStats != null) {
//TODO: rewrite this in something resembling an efficient way
CqlKeyspaceStats ksStats = null;
for (String ksName : model.getKeyspaces().keySet()) {
for (String ksName : model.getKeyspacesByName().keySet()) {
if ((ksStats = schemaStats.getKeyspace(ksName)) != null) {
model.getKeyspaces().get(ksName).setKeyspaceAttributes(ksStats.getKeyspaceAttributes());
Map<String, CqlTable> ksTables = model.getTablesByKeyspace().get(ksName);
model.getKeyspacesByName().get(ksName).setKeyspaceAttributes(ksStats.getKeyspaceAttributes());
Map<String, CqlTable> ksTables = model.getTablesByNameByKeyspace().get(ksName);
for (String tableName : ksTables.keySet()) {
if (ksStats.getKeyspaceTable(tableName) != null) {
model.getTablesByKeyspace().get(ksName).get(tableName)
model.getTablesByNameByKeyspace().get(ksName).get(tableName)
.setTableAttributes(ksStats.getKeyspaceTable(tableName).getAttributes());
}
}