IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-20 11:38:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

checkpoint update

Browse Source
This commit is contained in:
Jonathan Shook 2022-07-11 10:59:18 -05:00
parent e4f7e7e771
commit ebd3b775a5
24 changed files with 2733 additions and 2186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

546
adapter-cqld4/gen/io/nosqlbench/converters/cql/cql/grammars/CqlLexer.interp
View File

File diff suppressed because one or more lines are too long

1244
adapter-cqld4/gen/io/nosqlbench/converters/cql/cql/grammars/CqlLexer.java
View File

File diff suppressed because it is too large Load Diff

338
adapter-cqld4/gen/io/nosqlbench/converters/cql/cql/grammars/CqlLexer.tokens
View File

@ -1,338 +0,0 @@
LR_BRACKET=1
RR_BRACKET=2
LC_BRACKET=3
RC_BRACKET=4
LS_BRACKET=5
RS_BRACKET=6
COMMA=7
SEMI=8
COLON=9
DOT=10
STAR=11
DIVIDE=12
MODULE=13
PLUS=14
MINUSMINUS=15
MINUS=16
DQUOTE=17
SQUOTE=18
OPERATOR_EQ=19
OPERATOR_LT=20
OPERATOR_GT=21
OPERATOR_LTE=22
OPERATOR_GTE=23
K_ADD=24
K_AGGREGATE=25
K_ALL=26
K_ALLOW=27
K_ALTER=28
K_AND=29
K_ANY=30
K_APPLY=31
K_AS=32
K_ASC=33
K_AUTHORIZE=34
K_BATCH=35
K_BEGIN=36
K_BY=37
K_CALLED=38
K_CLUSTERING=39
K_COLUMNFAMILY=40
K_COMPACT=41
K_CONSISTENCY=42
K_CONTAINS=43
K_CREATE=44
K_CUSTOM=45
K_DELETE=46
K_DESC=47
K_DESCRIBE=48
K_DISTINCT=49
K_DROP=50
K_DURABLE_WRITES=51
K_EACH_QUORUM=52
K_ENTRIES=53
K_EXECUTE=54
K_EXISTS=55
K_FALSE=56
K_FILTERING=57
K_FINALFUNC=58
K_FROM=59
K_FULL=60
K_FUNCTION=61
K_FUNCTIONS=62
K_GRANT=63
K_IF=64
K_IN=65
K_INDEX=66
K_INFINITY=67
K_INITCOND=68
K_INPUT=69
K_INSERT=70
K_INTO=71
K_IS=72
K_JSON=73
K_KEY=74
K_KEYS=75
K_KEYSPACE=76
K_KEYSPACES=77
K_LANGUAGE=78
K_LEVEL=79
K_LIMIT=80
K_LOCAL_ONE=81
K_LOCAL_QUORUM=82
K_LOGGED=83
K_LOGIN=84
K_MATERIALIZED=85
K_MODIFY=86
K_NAN=87
K_NORECURSIVE=88
K_NOSUPERUSER=89
K_NOT=90
K_NULL=91
K_OF=92
K_ON=93
K_ONE=94
K_OPTIONS=95
K_OR=96
K_ORDER=97
K_PARTITION=98
K_PASSWORD=99
K_PER=100
K_PERMISSION=101
K_PERMISSIONS=102
K_PRIMARY=103
K_QUORUM=104
K_RENAME=105
K_REPLACE=106
K_REPLICATION=107
K_RETURNS=108
K_REVOKE=109
K_ROLE=110
K_ROLES=111
K_SCHEMA=112
K_SELECT=113
K_SET=114
K_SFUNC=115
K_STATIC=116
K_STORAGE=117
K_STYPE=118
K_SUPERUSER=119
K_TABLE=120
K_THREE=121
K_TIMESTAMP=122
K_TO=123
K_TOKEN=124
K_TRIGGER=125
K_TRUE=126
K_TRUNCATE=127
K_TTL=128
K_TWO=129
K_TYPE=130
K_UNLOGGED=131
K_UPDATE=132
K_USE=133
K_USER=134
K_USING=135
K_UUID=136
K_VALUES=137
K_VIEW=138
K_WHERE=139
K_WITH=140
K_WRITETIME=141
K_ASCII=142
K_BIGINT=143
K_BLOB=144
K_BOOLEAN=145
K_COUNTER=146
K_DATE=147
K_DECIMAL=148
K_DOUBLE=149
K_FLOAT=150
K_FROZEN=151
K_INET=152
K_INT=153
K_LIST=154
K_MAP=155
K_SMALLINT=156
K_TEXT=157
K_TIMEUUID=158
K_TIME=159
K_TINYINT=160
K_TUPLE=161
K_VARCHAR=162
K_VARINT=163
CODE_BLOCK=164
STRING_LITERAL=165
DECIMAL_LITERAL=166
FLOAT_LITERAL=167
HEXADECIMAL_LITERAL=168
REAL_LITERAL=169
OBJECT_NAME=170
UUID=171
SPACE=172
SPEC_MYSQL_COMMENT=173
COMMENT_INPUT=174
LINE_COMMENT=175
'('=1
')'=2
'{'=3
'}'=4
'['=5
']'=6
','=7
';'=8
':'=9
'.'=10
'*'=11
'/'=12
'%'=13
'+'=14
'--'=15
'-'=16
'"'=17
'\''=18
'='=19
'<'=20
'>'=21
'<='=22
'>='=23
'ADD'=24
'AGGREGATE'=25
'ALL'=26
'ALLOW'=27
'ALTER'=28
'AND'=29
'ANY'=30
'APPLY'=31
'AS'=32
'ASC'=33
'AUTHORIZE'=34
'BATCH'=35
'BEGIN'=36
'BY'=37
'CALLED'=38
'CLUSTERING'=39
'COLUMNFAMILY'=40
'COMPACT'=41
'CONSISTENCY'=42
'CONTAINS'=43
'CREATE'=44
'CUSTOM'=45
'DELETE'=46
'DESC'=47
'DESCRIBE'=48
'DISTINCT'=49
'DROP'=50
'DURABLE_WRITES'=51
'EACH_QUORUM'=52
'ENTRIES'=53
'EXECUTE'=54
'EXISTS'=55
'FALSE'=56
'FILTERING'=57
'FINALFUNC'=58
'FROM'=59
'FULL'=60
'FUNCTION'=61
'FUNCTIONS'=62
'GRANT'=63
'IF'=64
'IN'=65
'INDEX'=66
'INFINITY'=67
'INITCOND'=68
'INPUT'=69
'INSERT'=70
'INTO'=71
'IS'=72
'JSON'=73
'KEY'=74
'KEYS'=75
'KEYSPACE'=76
'KEYSPACES'=77
'LANGUAGE'=78
'LEVEL'=79
'LIMIT'=80
'LOCAL_ONE'=81
'LOCAL_QUORUM'=82
'LOGGED'=83
'LOGIN'=84
'MATERIALIZED'=85
'MODIFY'=86
'NAN'=87
'NORECURSIVE'=88
'NOSUPERUSER'=89
'NOT'=90
'NULL'=91
'OF'=92
'ON'=93
'ONE'=94
'OPTIONS'=95
'OR'=96
'ORDER'=97
'PARTITION'=98
'PASSWORD'=99
'PER'=100
'PERMISSION'=101
'PERMISSIONS'=102
'PRIMARY'=103
'QUORUM'=104
'RENAME'=105
'REPLACE'=106
'REPLICATION'=107
'RETURNS'=108
'REVOKE'=109
'ROLE'=110
'ROLES'=111
'SCHEMA'=112
'SELECT'=113
'SET'=114
'SFUNC'=115
'STATIC'=116
'STORAGE'=117
'STYPE'=118
'SUPERUSER'=119
'TABLE'=120
'THREE'=121
'TIMESTAMP'=122
'TO'=123
'TOKEN'=124
'TRIGGER'=125
'TRUE'=126
'TRUNCATE'=127
'TTL'=128
'TWO'=129
'TYPE'=130
'UNLOGGED'=131
'UPDATE'=132
'USE'=133
'USER'=134
'USING'=135
'UUID'=136
'VALUES'=137
'VIEW'=138
'WHERE'=139
'WITH'=140
'WRITETIME'=141
'ASCII'=142
'BIGINT'=143
'BLOB'=144
'BOOLEAN'=145
'COUNTER'=146
'DATE'=147
'DECIMAL'=148
'DOUBLE'=149
'FLOAT'=150
'FROZEN'=151
'INET'=152
'INT'=153
'LIST'=154
'MAP'=155
'SMALLINT'=156
'TEXT'=157
'TIMEUUID'=158
'TIME'=159
'TINYINT'=160
'TUPLE'=161
'VARCHAR'=162
'VARINT'=163

2
adapter-cqld4/pom.xml
View File

@ -104,7 +104,7 @@
<artifactId>antlr4-maven-plugin</artifactId>
<version>4.10.1</version>
<configuration>
<sourceDirectory>src/main/java/io/nosqlbench/converters/cql/cql/grammars</sourceDirectory>
<sourceDirectory>src/main/java/io/nosqlbench/converters/cql/grammars</sourceDirectory>
<arguments>
<argument>-package</argument>
<argument>io.nosqlbench.converters.cql.generated</argument>

17
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cql/exporters/CqlLiteralFormat.java
View File

@ -1,17 +0,0 @@
package io.nosqlbench.converters.cql.cql.exporters;
import java.util.function.Function;
public enum CqlLiteralFormat {
text(v -> "\""+v+"\""),
UNKNOWN(v -> v);
private final Function<String, String> literalFormat;
CqlLiteralFormat(Function<String,String> modifier) {
this.literalFormat = modifier;
}
public String format(String value) {
return this.literalFormat.apply(value);
}
}

8
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cql/traverser/CqlDDlDirectoryTraverser.java
View File

@ -1,8 +0,0 @@
package io.nosqlbench.converters.cql.cql.traverser;
import java.nio.file.Path;
public class CqlDDlDirectoryTraverser {
public void buildWorkloads(Path sourcePath, Path targetPath) {
}
}

2
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cql/cqlast/CqlColumnDef.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlColumnDef.java
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
package io.nosqlbench.converters.cql.cql.cqlast;
package io.nosqlbench.converters.cql.cqlast;
public class CqlColumnDef {
String name;

2
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cql/cqlast/CqlKeyspace.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlKeyspace.java
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
package io.nosqlbench.converters.cql.cql.cqlast;
package io.nosqlbench.converters.cql.cqlast;
public class CqlKeyspace {
String keyspaceName= "";

18
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cql/cqlast/CqlModel.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModel.java
View File

@ -1,4 +1,20 @@
package io.nosqlbench.converters.cql.cql.cqlast;
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.cqlast;
import java.util.LinkedHashMap;
import java.util.List;

2
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cql/cqlast/CqlModelBuilder.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModelBuilder.java
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
package io.nosqlbench.converters.cql.cql.cqlast;
package io.nosqlbench.converters.cql.cqlast;
import io.nosqlbench.converters.cql.generated.CqlParser;
import io.nosqlbench.converters.cql.generated.CqlParserBaseListener;

2
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cql/cqlast/CqlTable.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlTable.java
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
package io.nosqlbench.converters.cql.cql.cqlast;
package io.nosqlbench.converters.cql.cqlast;
import java.util.ArrayList;
import java.util.List;

59
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlLiteralFormat.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters;
import java.util.Locale;
import java.util.Optional;
import java.util.function.Function;
public enum CqlLiteralFormat {
TEXT(v -> "\""+v+"\""),
ASCII(v -> "\""+v+"\""),
VARCHAR(v -> "\""+v+"\""),
TINYINT,
SMALLINT,
INT,
BIGINT,
COUNTER,
BLOB,
BOOLEAN,
DECIMAL,
DOUBLE,
FLOAT,
UNKNOWN;
private final Function<String, String> literalFormat;
CqlLiteralFormat() {
this.literalFormat=v->v;
}
CqlLiteralFormat(Function<String,String> modifier) {
this.literalFormat = modifier;
}
public String format(String value) {
return this.literalFormat.apply(value);
}
public static Optional<CqlLiteralFormat> valueOfCqlType(String typename) {
for (CqlLiteralFormat value : CqlLiteralFormat.values()) {
if (typename.toUpperCase(Locale.ROOT).equals(value.toString().toUpperCase(Locale.ROOT))) {
return Optional.of(value);
}
}
return Optional.empty();
}
}

79
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cql/exporters/CqlWorkloadExporter.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlWorkloadExporter.java
View File

@ -1,12 +1,28 @@
package io.nosqlbench.converters.cql.cql.exporters;
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.exporters;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.converters.cql.cql.cqlast.CqlColumnDef;
import io.nosqlbench.converters.cql.cql.cqlast.CqlKeyspace;
import io.nosqlbench.converters.cql.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cql.cqlast.CqlTable;
import io.nosqlbench.converters.cql.cql.parser.CqlModelParser;
import io.nosqlbench.converters.cql.cqlast.CqlColumnDef;
import io.nosqlbench.converters.cql.cqlast.CqlKeyspace;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlTable;
import io.nosqlbench.converters.cql.parser.CqlModelParser;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.snakeyaml.engine.v2.api.Dump;
@ -16,7 +32,11 @@ import org.snakeyaml.engine.v2.common.NonPrintableStyle;
import org.snakeyaml.engine.v2.common.ScalarStyle;
import org.snakeyaml.engine.v2.representer.BaseRepresenter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -40,6 +60,43 @@ public class CqlWorkloadExporter {
this.model = CqlModelParser.parse(path);
}
public static void main(String[] args) {
if (args.length == 0) {
throw new RuntimeException("Usage example: PROG filepath.cql filepath.yaml");
}
Path srcpath = Path.of(args[0]);
if (!srcpath.toString().endsWith(".cql")) {
throw new RuntimeException("File '" + srcpath.toString() + "' must end in .cql");
}
if (!Files.exists(srcpath)) {
throw new RuntimeException("File '" + srcpath.toString() + "' does not exist.");
}
Path target = Path.of(srcpath.toString().replace("\\.cql", "\\.yaml"));
if (args.length == 2) {
target = Path.of(args[1]);
}
if (!target.toString().endsWith(".yaml")) {
throw new RuntimeException("Target file must end in .yaml");
}
if (Files.exists(target) && !target.toString().startsWith("_")) {
throw new RuntimeException("Target file '" + target.toString() + "' exists. Please remove it first or use a different target file name.");
}
CqlWorkloadExporter exporter = new CqlWorkloadExporter(srcpath);
String workload = exporter.getWorkloadAsYaml();
try {
Files.write(
target,
workload.getBytes(StandardCharsets.UTF_8),
StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING
);
} catch (IOException e) {
e.printStackTrace();
}
}
public Map<String, Object> getWorkload() {
Map<String, Object> workload = new LinkedHashMap<>();
@ -108,13 +165,13 @@ public class CqlWorkloadExporter {
private String genPredicatePart(CqlColumnDef def) {
String typeName = def.getType();
CqlLiteralFormat cqlLiteralFormat = null;
try {
cqlLiteralFormat = CqlLiteralFormat.valueOf(typeName);
} catch (IllegalArgumentException iae) {
cqlLiteralFormat = CqlLiteralFormat.UNKNOWN;
CqlLiteralFormat cqlLiteralFormat =
CqlLiteralFormat.valueOfCqlType(typeName).orElse(CqlLiteralFormat.UNKNOWN);
if (cqlLiteralFormat == CqlLiteralFormat.UNKNOWN) {
logger.warn("Unknown literal format for " + typeName);
}
return def.getName() + "=" + cqlLiteralFormat.format("{" + def.getName() + "}");
}

0
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cql/grammars/CqlLexer.g4 → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/grammars/CqlLexer.g4
View File

3
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cql/grammars/CqlParser.g4 → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/grammars/CqlParser.g4
View File

@ -375,7 +375,7 @@ withElement
;
clusteringOrder
: kwClustering kwOrder kwBy syntaxBracketLr column orderDirection? syntaxBracketRr
: kwClustering kwOrder kwBy syntaxBracketLr column orderDirection? ( syntaxComma column orderDirection? )* syntaxBracketRr
;
tableOptions
@ -812,6 +812,7 @@ dataTypeName
dataTypeDefinition
: syntaxBracketLa dataTypeName (syntaxComma dataTypeName)* syntaxBracketRa
| syntaxBracketLa dataType syntaxBracketRa
;
orderDirection

6
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cql/parser/CqlModelParser.java → adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/parser/CqlModelParser.java
View File

@ -14,10 +14,10 @@
* limitations under the License.
*/
package io.nosqlbench.converters.cql.cql.parser;
package io.nosqlbench.converters.cql.parser;
import io.nosqlbench.converters.cql.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cql.cqlast.CqlModelBuilder;
import io.nosqlbench.converters.cql.cqlast.CqlModel;
import io.nosqlbench.converters.cql.cqlast.CqlModelBuilder;
import io.nosqlbench.converters.cql.generated.CqlLexer;
import io.nosqlbench.converters.cql.generated.CqlParser;
import org.antlr.v4.runtime.CharStreams;

134
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/srcgrammars/v3/Cql.g Normal file
View File

@ -0,0 +1,134 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
grammar Cql;
options {
language = Java;
}
import Parser,Lexer;
@header {
package org.apache.cassandra.cql3;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.cassandra.auth.*;
import org.apache.cassandra.cql3.conditions.*;
import org.apache.cassandra.cql3.functions.*;
import org.apache.cassandra.cql3.restrictions.CustomIndexExpression;
import org.apache.cassandra.cql3.selection.*;
import org.apache.cassandra.cql3.statements.*;
import org.apache.cassandra.cql3.statements.schema.*;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.utils.Pair;
}
@members {
public void addErrorListener(ErrorListener listener)
{
gParser.addErrorListener(listener);
}
public void removeErrorListener(ErrorListener listener)
{
gParser.removeErrorListener(listener);
}
public void displayRecognitionError(String[] tokenNames, RecognitionException e)
{
gParser.displayRecognitionError(tokenNames, e);
}
protected void addRecognitionError(String msg)
{
gParser.addRecognitionError(msg);
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Recovery methods are overridden to avoid wasting work on recovering from errors when the result will be
// ignored anyway.
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@Override
protected Object recoverFromMismatchedToken(IntStream input, int ttype, BitSet follow) throws RecognitionException
{
throw new MismatchedTokenException(ttype, input);
}
@Override
public void recover(IntStream input, RecognitionException re)
{
// Do nothing.
}
}
@lexer::header {
package org.apache.cassandra.cql3;
}
@lexer::members {
List<Token> tokens = new ArrayList<Token>();
public void emit(Token token)
{
state.token = token;
tokens.add(token);
}
public Token nextToken()
{
super.nextToken();
if (tokens.size() == 0)
return new CommonToken(Token.EOF);
return tokens.remove(0);
}
private final List<ErrorListener> listeners = new ArrayList<ErrorListener>();
public void addErrorListener(ErrorListener listener)
{
this.listeners.add(listener);
}
public void removeErrorListener(ErrorListener listener)
{
this.listeners.remove(listener);
}
public void displayRecognitionError(String[] tokenNames, RecognitionException e)
{
for (int i = 0, m = listeners.size(); i < m; i++)
listeners.get(i).syntaxError(this, tokenNames, e);
}
}
query returns [CQLStatement.Raw stmnt]
: st=cqlStatement (';')* EOF { $stmnt = st; }
;

374
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/srcgrammars/v3/Lexer.g Normal file
View File

@ -0,0 +1,374 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
lexer grammar Lexer;
@lexer::members {
List<Token> tokens = new ArrayList<Token>();
public void emit(Token token)
{
state.token = token;
tokens.add(token);
}
public Token nextToken()
{
super.nextToken();
if (tokens.size() == 0)
return new CommonToken(Token.EOF);
return tokens.remove(0);
}
private final List<ErrorListener> listeners = new ArrayList<ErrorListener>();
public void addErrorListener(ErrorListener listener)
{
this.listeners.add(listener);
}
public void removeErrorListener(ErrorListener listener)
{
this.listeners.remove(listener);
}
public void displayRecognitionError(String[] tokenNames, RecognitionException e)
{
for (int i = 0, m = listeners.size(); i < m; i++)
listeners.get(i).syntaxError(this, tokenNames, e);
}
}
// Case-insensitive keywords
// When adding a new reserved keyword, add entry to o.a.c.cql3.ReservedKeywords and
// pylib/cqlshlib/cqlhandling.py::cql_keywords_reserved.
// When adding a new unreserved keyword, add entry to unreserved keywords in Parser.g.
K_SELECT: S E L E C T;
K_FROM: F R O M;
K_AS: A S;
K_WHERE: W H E R E;
K_AND: A N D;
K_KEY: K E Y;
K_KEYS: K E Y S;
K_ENTRIES: E N T R I E S;
K_FULL: F U L L;
K_INSERT: I N S E R T;
K_UPDATE: U P D A T E;
K_WITH: W I T H;
K_LIMIT: L I M I T;
K_PER: P E R;
K_PARTITION: P A R T I T I O N;
K_USING: U S I N G;
K_USE: U S E;
K_DISTINCT: D I S T I N C T;
K_COUNT: C O U N T;
K_SET: S E T;
K_BEGIN: B E G I N;
K_UNLOGGED: U N L O G G E D;
K_BATCH: B A T C H;
K_APPLY: A P P L Y;
K_TRUNCATE: T R U N C A T E;
K_DELETE: D E L E T E;
K_IN: I N;
K_CREATE: C R E A T E;
K_SCHEMA: S C H E M A;
K_KEYSPACE: ( K E Y S P A C E
| K_SCHEMA );
K_KEYSPACES: K E Y S P A C E S;
K_COLUMNFAMILY:( C O L U M N F A M I L Y
| T A B L E );
K_TABLES: ( C O L U M N F A M I L I E S
| T A B L E S );
K_MATERIALIZED:M A T E R I A L I Z E D;
K_VIEW: V I E W;
K_INDEX: I N D E X;
K_CUSTOM: C U S T O M;
K_ON: O N;
K_TO: T O;
K_DROP: D R O P;
K_PRIMARY: P R I M A R Y;
K_INTO: I N T O;
K_VALUES: V A L U E S;
K_TIMESTAMP: T I M E S T A M P;
K_TTL: T T L;
K_CAST: C A S T;
K_ALTER: A L T E R;
K_RENAME: R E N A M E;
K_ADD: A D D;
K_TYPE: T Y P E;
K_TYPES: T Y P E S;
K_COMPACT: C O M P A C T;
K_STORAGE: S T O R A G E;
K_ORDER: O R D E R;
K_BY: B Y;
K_ASC: A S C;
K_DESC: D E S C;
K_ALLOW: A L L O W;
K_FILTERING: F I L T E R I N G;
K_IF: I F;
K_IS: I S;
K_CONTAINS: C O N T A I N S;
K_GROUP: G R O U P;
K_CLUSTER: C L U S T E R;
K_INTERNALS: I N T E R N A L S;
K_ONLY: O N L Y;
K_GRANT: G R A N T;
K_ALL: A L L;
K_PERMISSION: P E R M I S S I O N;
K_PERMISSIONS: P E R M I S S I O N S;
K_OF: O F;
K_REVOKE: R E V O K E;
K_MODIFY: M O D I F Y;
K_AUTHORIZE: A U T H O R I Z E;
K_DESCRIBE: D E S C R I B E;
K_EXECUTE: E X E C U T E;
K_NORECURSIVE: N O R E C U R S I V E;
K_MBEAN: M B E A N;
K_MBEANS: M B E A N S;
K_USER: U S E R;
K_USERS: U S E R S;
K_ROLE: R O L E;
K_ROLES: R O L E S;
K_SUPERUSER: S U P E R U S E R;
K_NOSUPERUSER: N O S U P E R U S E R;
K_PASSWORD: P A S S W O R D;
K_HASHED: H A S H E D;
K_LOGIN: L O G I N;
K_NOLOGIN: N O L O G I N;
K_OPTIONS: O P T I O N S;
K_ACCESS: A C C E S S;
K_DATACENTERS: D A T A C E N T E R S;
K_CLUSTERING: C L U S T E R I N G;
K_ASCII: A S C I I;
K_BIGINT: B I G I N T;
K_BLOB: B L O B;
K_BOOLEAN: B O O L E A N;
K_COUNTER: C O U N T E R;
K_DECIMAL: D E C I M A L;
K_DOUBLE: D O U B L E;
K_DURATION: D U R A T I O N;
K_FLOAT: F L O A T;
K_INET: I N E T;
K_INT: I N T;
K_SMALLINT: S M A L L I N T;
K_TINYINT: T I N Y I N T;
K_TEXT: T E X T;
K_UUID: U U I D;
K_VARCHAR: V A R C H A R;
K_VARINT: V A R I N T;
K_TIMEUUID: T I M E U U I D;
K_TOKEN: T O K E N;
K_WRITETIME: W R I T E T I M E;
K_MAXWRITETIME:M A X W R I T E T I M E;
K_DATE: D A T E;
K_TIME: T I M E;
K_NULL: N U L L;
K_NOT: N O T;
K_EXISTS: E X I S T S;
K_MAP: M A P;
K_LIST: L I S T;
K_POSITIVE_NAN: N A N;
K_NEGATIVE_NAN: '-' N A N;
K_POSITIVE_INFINITY: I N F I N I T Y;
K_NEGATIVE_INFINITY: '-' I N F I N I T Y;
K_TUPLE: T U P L E;
K_TRIGGER: T R I G G E R;
K_STATIC: S T A T I C;
K_FROZEN: F R O Z E N;
K_FUNCTION: F U N C T I O N;
K_FUNCTIONS: F U N C T I O N S;
K_AGGREGATE: A G G R E G A T E;
K_AGGREGATES: A G G R E G A T E S;
K_SFUNC: S F U N C;
K_STYPE: S T Y P E;
K_FINALFUNC: F I N A L F U N C;
K_INITCOND: I N I T C O N D;
K_RETURNS: R E T U R N S;
K_CALLED: C A L L E D;
K_INPUT: I N P U T;
K_LANGUAGE: L A N G U A G E;
K_OR: O R;
K_REPLACE: R E P L A C E;
K_JSON: J S O N;
K_DEFAULT: D E F A U L T;
K_UNSET: U N S E T;
K_LIKE: L I K E;
// Case-insensitive alpha characters
fragment A: ('a'|'A');
fragment B: ('b'|'B');
fragment C: ('c'|'C');
fragment D: ('d'|'D');
fragment E: ('e'|'E');
fragment F: ('f'|'F');
fragment G: ('g'|'G');
fragment H: ('h'|'H');
fragment I: ('i'|'I');
fragment J: ('j'|'J');
fragment K: ('k'|'K');
fragment L: ('l'|'L');
fragment M: ('m'|'M');
fragment N: ('n'|'N');
fragment O: ('o'|'O');
fragment P: ('p'|'P');
fragment Q: ('q'|'Q');
fragment R: ('r'|'R');
fragment S: ('s'|'S');
fragment T: ('t'|'T');
fragment U: ('u'|'U');
fragment V: ('v'|'V');
fragment W: ('w'|'W');
fragment X: ('x'|'X');
fragment Y: ('y'|'Y');
fragment Z: ('z'|'Z');
STRING_LITERAL
@init{
StringBuilder txt = new StringBuilder(); // temporary to build pg-style-string
}
@after{ setText(txt.toString()); }
:
/* pg-style string literal */
(
'\$' '\$'
( /* collect all input until '$$' is reached again */
{ (input.size() - input.index() > 1)
&& !"$$".equals(input.substring(input.index(), input.index() + 1)) }?
=> c=. { txt.appendCodePoint(c); }
)*
'\$' '\$'
)
|
/* conventional quoted string literal */
(
'\'' (c=~('\'') { txt.appendCodePoint(c);} | '\'' '\'' { txt.appendCodePoint('\''); })* '\''
)
;
QUOTED_NAME
@init{ StringBuilder b = new StringBuilder(); }
@after{ setText(b.toString()); }
: '\"' (c=~('\"') { b.appendCodePoint(c); } | '\"' '\"' { b.appendCodePoint('\"'); })+ '\"'
;
EMPTY_QUOTED_NAME
: '\"' '\"'
;
fragment DIGIT
: '0'..'9'
;
fragment LETTER
: ('A'..'Z' | 'a'..'z')
;
fragment HEX
: ('A'..'F' | 'a'..'f' | '0'..'9')
;
fragment EXPONENT
: E ('+' | '-')? DIGIT+
;
fragment DURATION_UNIT
: Y
| M O
| W
| D
| H
| M
| S
| M S
| U S
| '\u00B5' S
| N S
;
INTEGER
: '-'? DIGIT+
;
QMARK
: '?'
;
RANGE
: '..'
;
/*
* Normally a lexer only emits one token at a time, but ours is tricked out
* to support multiple (see @lexer::members near the top of the grammar).
*/
FLOAT
: (INTEGER '.' RANGE) => INTEGER '.'
| (INTEGER RANGE) => INTEGER {$type = INTEGER;}
| INTEGER ('.' DIGIT*)? EXPONENT?
;
/*
* This has to be before IDENT so it takes precendence over it.
*/
BOOLEAN
: T R U E | F A L S E
;
DURATION
: '-'? DIGIT+ DURATION_UNIT (DIGIT+ DURATION_UNIT)*
| '-'? 'P' (DIGIT+ 'Y')? (DIGIT+ 'M')? (DIGIT+ 'D')? ('T' (DIGIT+ 'H')? (DIGIT+ 'M')? (DIGIT+ 'S')?)? // ISO 8601 "format with designators"
| '-'? 'P' DIGIT+ 'W'
| '-'? 'P' DIGIT DIGIT DIGIT DIGIT '-' DIGIT DIGIT '-' DIGIT DIGIT 'T' DIGIT DIGIT ':' DIGIT DIGIT ':' DIGIT DIGIT // ISO 8601 "alternative format"
;
IDENT
: LETTER (LETTER | DIGIT | '_')*
;
HEXNUMBER
: '0' X HEX*
;
UUID
: HEX HEX HEX HEX HEX HEX HEX HEX '-'
HEX HEX HEX HEX '-'
HEX HEX HEX HEX '-'
HEX HEX HEX HEX '-'
HEX HEX HEX HEX HEX HEX HEX HEX HEX HEX HEX HEX
;
WS
: (' ' | '\t' | '\n' | '\r')+ { $channel = HIDDEN; }
;
COMMENT
: ('--' | '//') .* ('\n'|'\r') { $channel = HIDDEN; }
;
MULTILINE_COMMENT
: '/*' .* '*/' { $channel = HIDDEN; }
;

1944
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/srcgrammars/v3/Parser.g Normal file
View File

File diff suppressed because it is too large Load Diff

24
adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/traverser/CqlDDlDirectoryTraverser.java Normal file
View File

@ -0,0 +1,24 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.converters.cql.traverser;
import java.nio.file.Path;
public class CqlDDlDirectoryTraverser {
public void buildWorkloads(Path sourcePath, Path targetPath) {
}
}

12
adapter-cqld4/src/test/java/io/nosqlbench/converters/cql/cql/parser/CqlParserHarnessTest.java
View File

@ -16,10 +16,13 @@
package io.nosqlbench.converters.cql.cql.parser;
import io.nosqlbench.converters.cql.cql.exporters.CqlWorkloadExporter;
import io.nosqlbench.converters.cql.exporters.CqlWorkloadExporter;
import io.nosqlbench.converters.cql.parser.CqlModelParser;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
public class CqlParserHarnessTest {
private final static String ksddl = """
@ -38,6 +41,13 @@ public class CqlParserHarnessTest {
""";
private final static String ddl = ksddl + tbddl;
@Test
public void testAllTypes() {
CqlWorkloadExporter exporter = new CqlWorkloadExporter(Path.of("src/test/resources/testschemas/cql_alltypes.cql"));
var data = exporter.getWorkloadAsYaml();
}
@Test
public void testGenBasicWorkload() {
CqlWorkloadExporter exporter = new CqlWorkloadExporter(ddl);

62
adapter-cqld4/src/test/resources/testschemas/cql_alltypes.cql Normal file
View File

@ -0,0 +1,62 @@
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-- CREATE KEYSPACE baselines
-- WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true;
CREATE TABLE baselines.alltypes
(
id text PRIMARY KEY,
a_bigint bigint,
a_blob blob,
a_boolean boolean,
a_date date,
a_decimal decimal,
a_double double,
a_duration duration,
a_float float,
a_frozen frozen<list< int >>,
a_list list<text>,
a_map map<text,text>,
a_set set<text>,
a_smallint smallint,
a_text text,
a_time time,
a_timestamp timestamp,
a_timeuuid timeuuid,
a_tinyint tinyint,
a_uuid uuid,
a_varchar text,
a_varint varint,
an_ascii ascii,
an_inet inet,
an_int int
) WITH additional_write_policy = '99PERCENTILE'
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = ''
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND default_time_to_live = 0
AND gc_grace_seconds = 864000
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND nodesync = {'enabled': 'true', 'incremental': 'true'}
AND read_repair = 'BLOCKING'
AND speculative_retry = '99PERCENTILE';

23
driver-kafka/pom.xml
View File

@ -64,17 +64,18 @@
</dependency>
</dependencies>
<repositories>
<repository>
<id>confluent</id>
<name>Confluent Maven Repo</name>
<layout>default</layout>
<url>https://packages.confluent.io/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
<!-- <repositories>-->
<!-- <repository>-->
<!-- <id>confluent</id>-->
<!-- <name>Confluent Maven Repo</name>-->
<!-- <layout>default</layout>-->
<!-- <url>https://packages.confluent.io/maven/</url>-->
<!-- <snapshots>-->
<!-- <enabled>false</enabled>-->
<!-- </snapshots>-->
<!-- </repository>-->
<!-- </repositories>-->
<!-- <profiles>-->
<!-- <profile>-->
<!-- <id>shade</id>-->

18
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -53,6 +53,8 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@ -224,6 +226,22 @@ public class NBCLI implements Function<String[], Integer> {
annotatorsConfig = "[{type:'log',level:'info'}]";
}
if (args.length > 0 && args[0].toLowerCase().equals("cqlgen")) {
String exporterImpl = "io.nosqlbench.converters.cql.exporters.CqlWorkloadExporter";
String[] exporterArgs = Arrays.copyOfRange(args, 1, args.length);
try {
Class<?> genclass = Class.forName(exporterImpl);
Method main = genclass.getMethod("main", new String[0].getClass());
Object result = main.invoke(null, new Object[]{exporterArgs});
} catch (ClassNotFoundException e) {
throw new RuntimeException("cql workload exporter implementation " + exporterImpl + " was not found in this runtime.");
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
System.out.println("Error in app: " + e.toString());
e.printStackTrace();
throw new RuntimeException("error while invoking " + exporterImpl + ": " + e.toString(),e);
}
return EXIT_OK;
}
if (args.length > 0 && args[0].toLowerCase().equals("export-docs")) {
BundledMarkdownExporter.main(Arrays.copyOfRange(args,1,args.length));
return EXIT_OK;