IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

fully parsing the example file now

Browse Source
This commit is contained in:
Jonathan Shook 2022-07-11 18:44:38 -05:00
parent 454412aa8f
commit 2df9e6b044
7 changed files with 74 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CQBErrorListener.java
View File

@ -20,12 +20,15 @@ import org.antlr.v4.runtime.BaseErrorListener;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Recognizer;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.function.Supplier;
public class CQBErrorListener extends BaseErrorListener {
public class CQBErrorListener extends BaseErrorListener implements Supplier<List<String>> {
List<String> errors = new ArrayList<>();
private final Path origin;
@ -37,20 +40,35 @@ public class CQBErrorListener extends BaseErrorListener {
@Override
public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) {
String locref= getLocationRef(origin,line, charPositionInLine);
String errmsg = "Error in " + origin.toString()+ "\n:"+locref;
String errmsg = "Error in " + origin.toString()+ ":\n" +
e.toString() + ":\n"+locref;
errors.add(errmsg);
}
public String getLocationRef(Path srcpath, int line, int charPositionInLine) {
boolean inij = System.getProperty("sun.java.command","").toLowerCase(Locale.ROOT).contains("intellij");
Path vcwd = Path.of(".").toAbsolutePath().normalize();
vcwd = inij ? vcwd.getParent().normalize() : vcwd;
Path relpath = vcwd.relativize(srcpath.toAbsolutePath());
if (inij) {
relpath = Path.of("local/verizon/");
// relpath = Path.of(relpath.toString().replace("target/classes/","src/main/resources/"));
Path cwd = Path.of(".").toAbsolutePath().normalize();
Path vcwd = cwd;
while (vcwd!=null) {
if (Files.exists(Path.of(vcwd + File.separator + ".git"))) {
break;
}
vcwd = vcwd.getParent();
}
return "\t at (" + relpath + ":" + line+":"+charPositionInLine + ")";
boolean inij = System.getProperty("sun.java.command","").toLowerCase(Locale.ROOT).contains("intellij");
vcwd = inij ? cwd.getParent().normalize() : vcwd;
Path relpath = vcwd.relativize(srcpath.toAbsolutePath());
if (inij) {
relpath = Path.of(relpath.toString().replace("target/classes/","src/main/resources/"));
}
return "\tat (" + relpath + ":" + line+":"+charPositionInLine + ")";
}
@Override
public List<String> get() {
return this.errors;
}
}

26
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModel.java
View File

@ -16,32 +16,32 @@
package io.nosqlbench.converters.cql.cqlast;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
public class CqlModel {
private final Supplier<List<String>> errors;
Map<String, CqlKeyspace> keyspaces = new LinkedHashMap<>();
Map<String, Map<String, CqlTable>> tables = new LinkedHashMap<>();
public List<String> getErrors() {
return errors;
}
public void setErrors(List<String> errors) {
this.errors = errors;
}
List<String> errors = new ArrayList<>();
transient
CqlKeyspace keyspace = null;
transient
CqlTable table;
public CqlModel(Supplier<List<String>> errorSource) {
this.errors = errorSource;
}
public List<String> getErrors() {
return errors.get();
}
public void newKeyspace() {
keyspace = new CqlKeyspace();
}
@ -87,12 +87,12 @@ public class CqlModel {
for (String ks : keyspaces.keySet()) {
CqlKeyspace keyspace = keyspaces.get(ks);
sb.append("keyspace '").append(keyspace.getKeyspaceName()).append("':\n");
sb.append(keyspace.toString()).append("\n");
sb.append(keyspace).append("\n");
tables.getOrDefault(ks,Map.of()).values().stream()
.forEach(table -> {
sb.append("table '").append(table.getTableName()).append("':\n");
sb.append(table.toString());
sb.append(table);
});
}
return sb.toString();

3
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModelBuilder.java
View File

@ -28,10 +28,11 @@ import java.util.List;
public class CqlModelBuilder extends CqlParserBaseListener {
private final CQBErrorListener errorListener;
CqlModel model = new CqlModel();
private final CqlModel model;
public CqlModelBuilder(CQBErrorListener errorListener) {
this.errorListener = errorListener;
this.model = new CqlModel(errorListener);
}
@Override

12
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlWorkloadExporter.java
View File

@ -77,10 +77,10 @@ public class CqlWorkloadExporter {
}
Path srcpath = Path.of(args[0]);
if (!srcpath.toString().endsWith(".cql")) {
throw new RuntimeException("File '" + srcpath.toString() + "' must end in .cql");
throw new RuntimeException("File '" + srcpath + "' must end in .cql");
}
if (!Files.exists(srcpath)) {
throw new RuntimeException("File '" + srcpath.toString() + "' does not exist.");
throw new RuntimeException("File '" + srcpath + "' does not exist.");
}
Path target = Path.of(srcpath.toString().replace("\\.cql", "\\.yaml"));
@ -91,7 +91,7 @@ public class CqlWorkloadExporter {
throw new RuntimeException("Target file must end in .yaml");
}
if (Files.exists(target) && !target.toString().startsWith("_")) {
throw new RuntimeException("Target file '" + target.toString() + "' exists. Please remove it first or use a different target file name.");
throw new RuntimeException("Target file '" + target + "' exists. Please remove it first or use a different target file name.");
}
CqlWorkloadExporter exporter = new CqlWorkloadExporter(srcpath);
@ -128,7 +128,7 @@ public class CqlWorkloadExporter {
model.getAllTables()
.stream()
.collect(Collectors.toMap(
t -> "insert-" + t.getTableName(),
t -> "insert-" + t.getKeySpace()+"__"+t.getTableName(),
this::genUpsertTemplate)
)
);
@ -137,7 +137,7 @@ public class CqlWorkloadExporter {
model.getAllTables()
.stream()
.collect(Collectors.toMap(
t -> "select-" + t.getTableName(),
t -> "select-" + t.getKeySpace()+"__"+t.getTableName(),
this::genSelectTemplate)
));
@ -150,7 +150,7 @@ public class CqlWorkloadExporter {
Map<String, String> rampupOpTemplates = model.getAllTables()
.stream()
.collect(Collectors.toMap(
t -> "insert-" + t.getTableName(),
t -> "insert-" + t.getKeySpace()+"__"+t.getTableName(),
this::genUpsertTemplate)
);

3
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/grammars/CqlLexer.g4
View File

@ -87,7 +87,8 @@ K_KEYS: 'KEYS';
K_KEYSPACE: 'KEYSPACE';
K_KEYSPACES: 'KEYSPACES';
K_LANGUAGE: 'LANGUAGE';
K_LEVEL: 'LEVEL';
// Disabled because there was no definitive reference to this as a bare keyword in the specs
//K_LEVEL: 'LEVEL';
K_LIMIT: 'LIMIT';
K_LOCAL_ONE: 'LOCAL_ONE';
K_LOCAL_QUORUM: 'LOCAL_QUORUM';

29
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/grammars/CqlParser.g4
View File

@ -367,8 +367,9 @@ dropIndex
;
// TODO: update to https://docs.datastax.com/en/dse/6.8/cql/cql/cql_reference/cql_commands/cqlCreateTable.html
// TODO: Update optional sequence for compact storage [options] clustering order [options] as documented above
createTable
: kwCreate kwTable ifNotExist? (keyspace DOT)? table syntaxBracketLr columnDefinitionList syntaxBracketRr (withElement (kwAnd tableOptions)*)?
: kwCreate kwTable ifNotExist? (keyspace DOT)? table syntaxBracketLr columnDefinitionList syntaxBracketRr (kwWith tableOptions (kwAnd tableOptions)*)?
;
withElement
@ -386,15 +387,22 @@ tableOptions
tableOptionItem
: tableOptionName OPERATOR_EQ tableOptionValue
| tableOptionName OPERATOR_EQ optionHash
| kwCompactStorage
| clusteringOrder
;
tableOptionName
: OBJECT_NAME
;
kwCompactStorage
: K_COMPACT K_STORAGE
;
tableOptionValue
: stringLiteral
| floatLiteral
| booleanLiteral
;
optionHash
@ -447,7 +455,7 @@ compoundKey
;
compositeKey
: syntaxBracketLr partitionKeyList syntaxBracketRr (syntaxComma clusteringKeyList)
: syntaxBracketLr partitionKeyList syntaxBracketRr (syntaxComma clusteringKeyList)?
;
partitionKeyList
@ -778,11 +786,15 @@ column
;
dataType
: dataTypeName dataTypeDefinition?
: dataTypeName
| K_FROZEN syntaxBracketLa dataType syntaxBracketRa
| K_SET syntaxBracketLa dataType syntaxBracketRa
| K_LIST syntaxBracketLa dataType syntaxBracketRa
| K_MAP syntaxBracketLa dataType syntaxComma dataType syntaxBracketRa
;
dataTypeName
: OBJECT_NAME
: (keyspace DOT)? OBJECT_NAME
| K_TIMESTAMP
| K_SET
| K_ASCII
@ -794,7 +806,7 @@ dataTypeName
| K_DECIMAL
| K_DOUBLE
| K_FLOAT
| K_FROZEN
// | K_FROZEN
| K_INET
| K_INT
| K_LIST
@ -812,8 +824,11 @@ dataTypeName
;
dataTypeDefinition
: syntaxBracketLa dataTypeName (syntaxComma dataTypeName)* syntaxBracketRa
| syntaxBracketLa dataType syntaxBracketRa
: syntaxBracketLa dataType syntaxBracketRa
| syntaxBracketLa dataTypeName (syntaxBracketLa dataTypeName syntaxBracketRa) syntaxBracketRa
| syntaxBracketLa dataTypeName (syntaxComma dataTypeName)* syntaxBracketRa
| syntaxBracketLa dataTypeName (syntaxComma syntaxBracketLa dataTypeName syntaxBracketRa)* syntaxBracketRa
// | syntaxBracketLa dataTypeName (syntaxComma dataTypeDefinition)* syntaxBracketRa
;
orderDirection

2
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/parser/CqlModelParser.java
View File

@ -65,7 +65,7 @@ public class CqlModelParser {
CqlModel model = cqlModelBuilder.getModel();
if (model.getErrors().size()>0) {
System.out.println(model.getErrors());
throw new RuntimeException("Unable to render model for unparsable input with " + model.getErrors() + " errors");
throw new RuntimeException("Unable to render model for unparsable input with errors:\n" + model.getErrors());
} else {
return model;
}