checkpoint

adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java

@@ -126,12 +126,28 @@ public class Cqld4Space {
 //        builder.withCompression(ProtocolOptions.Compression.NONE);
 //
         Optional<String> usernameOpt = cfg.getOptional("username");
+        Optional<String> userfileOpt = cfg.getOptional("userfile");
         Optional<String> passwordOpt = cfg.getOptional("password");
         Optional<String> passfileOpt = cfg.getOptional("passfile");
 
+
+        String username = null;
         if (usernameOpt.isPresent()) {
-            String username = usernameOpt.get();
-            String password;
+            username = usernameOpt.get();
+        } else if (userfileOpt.isPresent()) {
+            Path path = Paths.get(userfileOpt.get());
+            try {
+                username = Files.readAllLines(path).get(0);
+            } catch (IOException e) {
+                String error = "Error while reading username from file:" + usernameOpt.get();
+                logger.error(error, e);
+                throw new RuntimeException(e);
+            }
+        }
+
+        String password;
+        if (username != null) {
+
             if (passwordOpt.isPresent()) {
                 password = passwordOpt.get();
             } else if (passfileOpt.isPresent()) {
@@ -139,7 +155,7 @@ public class Cqld4Space {
                 try {
                     password = Files.readAllLines(path).get(0);
                 } catch (IOException e) {
-                    String error = "Error while reading password from file:" + passfileOpt;
+                    String error = "Error while reading password from file:" + passfileOpt.get();
                     logger.error(error, e);
                     throw new RuntimeException(e);
                 }
@@ -151,25 +167,34 @@ public class Cqld4Space {
             builder.withAuthCredentials(username, password);
         }
 
-        cfg.getOptional("whitelist").ifPresent(wl -> {
-            List<InetSocketAddress> addrs = Arrays
-                .stream(wl.split(","))
-                .map(this::toSocketAddr)
-                .toList();
-            builder.withNodeDistanceEvaluator(new NodeFilterToDistanceEvaluatorAdapter(n -> {
-                return (n.getBroadcastAddress().isPresent() && addrs.contains(n.getBroadcastAddress().get()))
-                ||(n.getBroadcastRpcAddress().isPresent() && addrs.contains(n.getBroadcastRpcAddress().get()))
-                    ||(n.getListenAddress().isPresent() && addrs.contains(n.getListenAddress().get()));
-            }));
-        });
 
-        cfg.getOptional("cloud_proxy_address").ifPresent(cpa -> {
-            String[] addr = cpa.split(":",2);
-            if (addr.length==1) {
-                throw new RuntimeException("cloud_proxy_address must be specified in host:port form.");
-            }
-            builder.withCloudProxyAddress(InetSocketAddress.createUnresolved(addr[0],Integer.parseInt(addr[1])));
-        });
+        cfg.getOptional("whitelist").
+
+            ifPresent(wl ->
+
+            {
+                List<InetSocketAddress> addrs = Arrays
+                    .stream(wl.split(","))
+                    .map(this::toSocketAddr)
+                    .toList();
+                builder.withNodeDistanceEvaluator(new NodeFilterToDistanceEvaluatorAdapter(n -> {
+                    return (n.getBroadcastAddress().isPresent() && addrs.contains(n.getBroadcastAddress().get()))
+                        || (n.getBroadcastRpcAddress().isPresent() && addrs.contains(n.getBroadcastRpcAddress().get()))
+                        || (n.getListenAddress().isPresent() && addrs.contains(n.getListenAddress().get()));
+                }));
+            });
+
+        cfg.getOptional("cloud_proxy_address").
+
+            ifPresent(cpa ->
+
+            {
+                String[] addr = cpa.split(":", 2);
+                if (addr.length == 1) {
+                    throw new RuntimeException("cloud_proxy_address must be specified in host:port form.");
+                }
+                builder.withCloudProxyAddress(InetSocketAddress.createUnresolved(addr[0], Integer.parseInt(addr[1])));
+            });
 
         NBConfiguration sslCfg = SSLKsFactory.get().getConfigModel().extractConfig(cfg);
 
@@ -257,14 +282,15 @@ public class Cqld4Space {
     public static NBConfigModel getConfigModel() {
         return ConfigModel.of(Cqld4Space.class)
             .add(Param.optional("localdc"))
-            .add(Param.optional("secureconnectbundle"))
+            .add(Param.optional(List.of("secureconnectbundle","scb")))
             .add(Param.optional("hosts"))
-            .add(Param.optional("driverconfig",String.class))
-            .add(Param.optional("username",String.class,"user name (see also password and passfile)"))
+            .add(Param.optional("driverconfig", String.class))
+            .add(Param.optional("username", String.class, "user name (see also password and passfile)"))
+            .add(Param.optional("userfile", String.class, "file to load the username from"))
             .add(Param.optional("password", String.class, "password (see also passfile)"))
-            .add(Param.optional("passfile",String.class,"file to load the password from"))
-            .add(Param.optional("whitelist",String.class,"list of whitelist hosts addresses"))
-            .add(Param.optional("cloud_proxy_address",String.class,"Cloud Proxy Address"))
+            .add(Param.optional("passfile", String.class, "file to load the password from"))
+            .add(Param.optional("whitelist", String.class, "list of whitelist hosts addresses"))
+            .add(Param.optional("cloud_proxy_address", String.class, "Cloud Proxy Address"))
             .add(SSLKsFactory.get().getConfigModel())
             .add(getDriverOptionsModel())
             .add(new OptionHelpers(new OptionsMap()).getConfigModel())

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java

@@ -199,7 +199,12 @@ public class NBCLI implements Function<String[], Integer> {
             Map<String, String> dashboardOptions = Map.of(
                 DockerMetricsManager.GRAFANA_TAG, globalOptions.getDockerGrafanaTag(),
                 DockerMetricsManager.PROM_TAG, globalOptions.getDockerPromTag(),
-                DockerMetricsManager.TSDB_RETENTION, String.valueOf(globalOptions.getDockerPromRetentionDays())
+                DockerMetricsManager.TSDB_RETENTION, String.valueOf(globalOptions.getDockerPromRetentionDays()),
+                DockerMetricsManager.GRAPHITE_SAMPLE_EXPIRY,"10m",
+                DockerMetricsManager.GRAPHITE_CACHE_SIZE,"5000",
+                DockerMetricsManager.GRAPHITE_LOG_LEVEL,globalOptions.getGraphiteLogLevel(),
+                DockerMetricsManager.GRAPHITE_LOG_FORMAT,"logfmt"
+
             );
             dmh.startMetrics(dashboardOptions);
             String warn = "Docker Containers are started, for grafana and prometheus, hit" +

engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java

@@ -97,6 +97,7 @@ public class NBCLIOptions {
     private static final String DASH_VVV_TRACE = "-vvv";
     private static final String REPORT_INTERVAL = "--report-interval";
     private static final String REPORT_GRAPHITE_TO = "--report-graphite-to";
+    private static final String GRAPHITE_LOG_LEVEL = "--graphite-log-level";
     private static final String REPORT_CSV_TO = "--report-csv-to";
     private static final String REPORT_SUMMARY_TO = "--report-summary-to";
     private final static String REPORT_SUMMARY_TO_DEFAULT = "stdout:60,_LOGS_/_SESSION_.summary";
@@ -183,6 +184,7 @@ public class NBCLIOptions {
     private String reportSummaryTo = REPORT_SUMMARY_TO_DEFAULT;
     private boolean enableAnsi = System.getenv("TERM")!=null && !System.getenv("TERM").isEmpty();
     private Maturity minMaturity = Maturity.Unspecified;
+    private String graphitelogLevel="info";
 
     public String getAnnotatorsConfig() {
         return annotatorsConfig;
@@ -213,6 +215,10 @@ public class NBCLIOptions {
         return logfileLoggingPattern;
     }
 
+    public String getGraphiteLogLevel() {
+        return this.graphitelogLevel;
+    }
+
     public enum Mode {
         ParseGlobalsOnly,
         ParseAllOptions
@@ -339,6 +345,10 @@ public class NBCLIOptions {
                     arglist.removeFirst();
                     reportGraphiteTo = arglist.removeFirst();
                     break;
+                case GRAPHITE_LOG_LEVEL:
+                    arglist.removeFirst();
+                    graphitelogLevel=arglist.removeFirst();
+                    break;
                 case METRICS_PREFIX:
                     arglist.removeFirst();
                     metricsPrefix = arglist.removeFirst();

engine-clients/src/main/java/io/nosqlbench/engine/clients/grafana/authorizers/CurlCmdInjector.java

@@ -24,10 +24,13 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.http.HttpRequest;
 import java.nio.CharBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Optional;
 
 public class CurlCmdInjector {
@@ -45,8 +48,21 @@ public class CurlCmdInjector {
                 .replace("ROLE", apirq.getRole())
                 .replaceAll("\n", "");
 
+            String binname=null;
+            for (String potentialPath : new String[]{"/bin", "/usr/bin", "/usr/local/bin"}) {
+                String filename = potentialPath+ File.separator + "curl";
+                if (Files.exists(Path.of(filename))) {
+                    binname=filename;
+                    break;
+                }
+            }
+            if (binname==null) {
+                logger.error("Unable to find binary path for curl command");
+                return Optional.empty();
+            }
+
             String[] args = new String[]{
-                "/usr/bin/curl", "-s", "-XPOST", rq.uri().toString(),
+                binname, "-s", "-XPOST", rq.uri().toString(),
                 "-H", "Content-Type: application/json",
                 "-d", requestJson
             };

engine-clients/src/main/java/io/nosqlbench/engine/clients/grafana/authorizers/RawSocketInjector.java

@@ -62,6 +62,7 @@ public class RawSocketInjector {
             String rqProtocol = """
                 POST __PATH__ HTTP/1.1
                 Host: __HOST__:__PORT__
+                Connection: close
                 Authorization: __AUTHORIZATION__
                 User-Agent: **nb**
                 Accept: */*
@@ -87,7 +88,8 @@ public class RawSocketInjector {
 
             BufferedReader reader = new BufferedReader(new InputStreamReader(fromServer));
             CharBuffer inbuf = CharBuffer.allocate(1000000);
-            while (reader.ready() && (reader.read(inbuf)) >= 0) {
+
+            while (reader.read(inbuf) >= 0) {
             }
             inbuf.flip();
             String response = inbuf.toString();
@@ -98,6 +100,7 @@ public class RawSocketInjector {
             String[] statusAndHeaders = headersAndBody[0].split("\r\n", 2);
             if (!statusAndHeaders[0].contains("200 OK")) {
                 logger.error("Status was unexpected: '" + statusAndHeaders[0] + "'");
+                logger.error("response was:\n" + response);
                 return Optional.empty();
             }
 
@@ -110,6 +113,7 @@ public class RawSocketInjector {
 
             Gson gson = new GsonBuilder().setPrettyPrinting().create();
             ApiToken apiToken = gson.fromJson(headersAndBody[1], ApiToken.class);
+            logger.info("Authorized local grafana client with Socket client: " + apiToken.toString());
             return Optional.of(apiToken);
 
         } catch (Exception e) {

engine-docker/src/main/java/io/nosqlbench/engine/docker/DockerMetricsManager.java

@@ -47,6 +47,10 @@ public class DockerMetricsManager {
     public static final String GRAFANA_TAG = "grafana_tag";
     public static final String PROM_TAG = "prom_tag";
     public static final String TSDB_RETENTION = "tsdb_days";
+    public static final String GRAPHITE_SAMPLE_EXPIRY = "graphite_sample_expiry";
+    public static final String GRAPHITE_CACHE_SIZE = "graphite_cache_size";
+    public static final String GRAPHITE_LOG_LEVEL = "graphite_log_level";
+    public static final String GRAPHITE_LOG_FORMAT = "graphite_log_format";
 
     private final DockerHelper dh;
 
@@ -60,7 +64,12 @@ public class DockerMetricsManager {
 
     public void startMetrics(Map<String, String> options) {
 
-        String ip = startGraphite();
+        String ip = startGraphite(
+                options.get(GRAPHITE_SAMPLE_EXPIRY),
+                options.get(GRAPHITE_CACHE_SIZE),
+                options.get(GRAPHITE_LOG_LEVEL),
+                options.get(GRAPHITE_LOG_FORMAT)
+            );
 
         startPrometheus(ip, options.get(PROM_TAG), options.get(TSDB_RETENTION));
 
@@ -141,12 +150,18 @@ public class DockerMetricsManager {
 
         String reload = "http://localhost:9090/-/reload";
         List<String> linkNames = new ArrayList();
+        linkNames.add("graphite-exporter");
         dh.startDocker(PROMETHEUS_IMG, tag, name, port, volumeDescList, envList, cmdList, reload, linkNames);
 
         logger.info("prometheus started and listening");
     }
 
-    private String startGraphite() {
+    private String startGraphite(
+        String graphite_sample_expiry,
+        String graphite_cache_size,
+        String graphite_log_level,
+        String graphite_log_format
+    ) {
 
         logger.info("preparing to start graphite exporter container...");
 
@@ -165,7 +180,13 @@ public class DockerMetricsManager {
         String reload = null;
         List<String> linkNames = new ArrayList();
 
-        List<String> cmdOpts = Arrays.asList("--graphite.mapping-config=/tmp/graphite_mapping.conf");
+        List<String> cmdOpts = Arrays.asList(
+            "--graphite.mapping-config=/tmp/graphite_mapping.conf",
+            "--graphite.sample-expiry="+graphite_sample_expiry,
+            "--graphite.cache-size="+graphite_cache_size,
+            "--log.level="+graphite_log_level,
+            "--log.format="+graphite_log_format
+        );
 
         dh.startDocker(GRAPHITE_EXPORTER_IMG, tag, name, port, volumeDescList, envList, cmdOpts, reload, linkNames);
 
@@ -214,7 +235,7 @@ public class DockerMetricsManager {
             throw new DockerInitError("IP for graphite container not found");
         }
 
-        datasource = datasource.replace("!!!GRAPHITE_IP!!!", ip);
+        datasource = datasource.replace("!!!GRAPHITE_IP!!!", "graphite-exporter");
 
         File nosqlbenchDir = new File(userHome, "/.nosqlbench/");
         mkdir(nosqlbenchDir);

engine-docker/src/main/resources/docker/prometheus/prometheus.yml

@@ -1,7 +1,7 @@
 # this directory was added by nosqlbench for docker-metrics
 # it is used by the prometheus docker container
 global:
-  scrape_interval: 15s
+  scrape_interval: 10s
   scrape_timeout: 10s
   evaluation_interval: 15s
 
@@ -14,19 +14,19 @@ alerting:
 
 scrape_configs:
   - job_name: prometheus
-    scrape_interval: 15s
-    scrape_timeout: 15s
+    scrape_interval: 10s
+    scrape_timeout: 10s
     metrics_path: /metrics
     scheme: http
     static_configs:
     - targets:
       - localhost:9090
 
-  - job_name: graphite_import 
-    scrape_interval: 15s 
-    scrape_timeout: 15s 
-    metrics_path: /metrics 
-    scheme: http 
-    static_configs: 
-    - targets: 
+  - job_name: graphite_import
+    scrape_interval: 10s
+    scrape_timeout: 10s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+    - targets:
       - !!!GRAPHITE_IP!!!:9108

virtdata-lang/src/main/java/io/nosqlbench/virtdata/lang/cqlast/CqlAstBuilder.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.lang.cqlast;
+
+import io.nosqlbench.virtdata.lang.generated.CqlParser;
+import io.nosqlbench.virtdata.lang.generated.CqlParserBaseListener;
+
+public class CqlAstBuilder extends CqlParserBaseListener {
+
+    CqlKeyspace keyspace = new CqlKeyspace();
+
+    @Override
+    public void enterCreateTable(CqlParser.CreateTableContext ctx) {
+        keyspace.addTable();
+    }
+
+    @Override
+    public void exitCreateTable(CqlParser.CreateTableContext ctx) {
+        keyspace.setTableName(ctx.table().OBJECT_NAME().getSymbol().getText());
+    }
+
+    @Override
+    public void enterColumnDefinition(CqlParser.ColumnDefinitionContext ctx) {
+        System.out.println("here");
+    }
+
+    @Override
+    public void exitColumnDefinition(CqlParser.ColumnDefinitionContext ctx) {
+        System.out.println("here");
+        keyspace.addTableColumn(
+            ctx.dataType().dataTypeName().getText(),
+            ctx.column().OBJECT_NAME().getSymbol().getText()
+        );
+    }
+
+    @Override
+    public String toString() {
+        return "CqlAstBuilder{" +
+            "keyspace=" + keyspace +
+            '}';
+    }
+}

virtdata-lang/src/main/java/io/nosqlbench/virtdata/lang/cqlast/CqlField.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.lang.cqlast;
+
+public class CqlField {
+    String name;
+    String type;
+
+    public CqlField(String type, String name) {
+        this.type = type;
+        this.name = name;
+    }
+
+    @Override
+    public String toString() {
+        return "CqlField{" +
+            "name='" + name + '\'' +
+            ", type='" + type + '\'' +
+            '}';
+    }
+}

virtdata-lang/src/main/java/io/nosqlbench/virtdata/lang/cqlast/CqlKeyspace.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.lang.cqlast;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class CqlKeyspace {
+    List<CqlTable> tables = new ArrayList<>();
+    CqlTable lastAddedTable = null;
+
+    public void addTable() {
+        lastAddedTable = new CqlTable();
+        tables.add(lastAddedTable);
+    }
+
+    public void addColumnDef(String type, String name) {
+        lastAddedTable.addcolumnDef(new CqlField(type, name));
+    }
+
+    public void setTableName(String tableName) {
+        lastAddedTable.setName(tableName);
+    }
+
+    public void addTableColumn(String type, String fieldName) {
+        lastAddedTable.addcolumnDef(type,fieldName);
+    }
+
+    @Override
+    public String toString() {
+        return "CqlKeyspace{" +
+            "tables=" + tables +
+            ", lastAddedTable=" + lastAddedTable +
+            '}';
+    }
+}

virtdata-lang/src/main/java/io/nosqlbench/virtdata/lang/cqlast/CqlTable.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.lang.cqlast;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class CqlTable {
+    String name = "";
+    List<CqlField> fields = new ArrayList();
+
+    public CqlTable() {
+    }
+
+    public CqlTable(String tableName) {
+        this.name = name;
+    }
+
+    public void addcolumnDef(CqlField cqlField) {
+        this.fields.add(cqlField);
+    }
+
+    public void setName(String tableName) {
+        this.name = name;
+    }
+
+    public void addcolumnDef(String type, String fieldName) {
+        fields.add(new CqlField(type, fieldName));
+    }
+
+    @Override
+    public String toString() {
+        return "CqlTable{" +
+            "name='" + name + '\'' +
+            ", fields=" + fields +
+            '}';
+    }
+}

virtdata-lang/src/main/java/io/nosqlbench/virtdata/lang/grammars/CqlLexer.g4

@@ -0,0 +1,224 @@
+lexer grammar CqlLexer;
+
+options {
+	caseInsensitive = true;
+}
+
+// Operators and Punctuators
+
+LR_BRACKET: '(';
+RR_BRACKET: ')';
+LC_BRACKET: '{';
+RC_BRACKET: '}';
+LS_BRACKET: '[';
+RS_BRACKET: ']';
+COMMA: ',';
+SEMI: ';';
+COLON: ':';
+DOT: '.';
+STAR: '*';
+DIVIDE: '/';
+MODULE: '%';
+PLUS: '+';
+MINUSMINUS: '--';
+MINUS: '-';
+DQUOTE: '"';
+SQUOTE: '\'';
+OPERATOR_EQ: '=';
+OPERATOR_LT: '<';
+OPERATOR_GT: '>';
+OPERATOR_LTE: '<=';
+OPERATOR_GTE: '>=';
+
+// Keywords
+
+K_ADD: 'ADD';
+K_AGGREGATE: 'AGGREGATE';
+K_ALL: 'ALL';
+K_ALLOW: 'ALLOW';
+K_ALTER: 'ALTER';
+K_AND: 'AND';
+K_ANY: 'ANY';
+K_APPLY: 'APPLY';
+K_AS: 'AS';
+K_ASC: 'ASC';
+K_AUTHORIZE: 'AUTHORIZE';
+K_BATCH: 'BATCH';
+K_BEGIN: 'BEGIN';
+K_BY: 'BY';
+K_CALLED: 'CALLED';
+K_CLUSTERING: 'CLUSTERING';
+K_COLUMNFAMILY: 'COLUMNFAMILY';
+K_COMPACT:  'COMPACT';
+K_CONSISTENCY: 'CONSISTENCY';
+K_CONTAINS: 'CONTAINS';
+K_CREATE: 'CREATE';
+K_CUSTOM: 'CUSTOM';
+K_DELETE: 'DELETE';
+K_DESC: 'DESC';
+K_DESCRIBE: 'DESCRIBE';
+K_DISTINCT: 'DISTINCT';
+K_DROP: 'DROP';
+K_DURABLE_WRITES: 'DURABLE_WRITES';
+K_EACH_QUORUM: 'EACH_QUORUM';
+K_ENTRIES: 'ENTRIES';
+K_EXECUTE: 'EXECUTE';
+K_EXISTS: 'EXISTS';
+K_FALSE: 'FALSE';
+K_FILTERING: 'FILTERING';
+K_FINALFUNC: 'FINALFUNC';
+K_FROM: 'FROM';
+K_FULL: 'FULL';
+K_FUNCTION: 'FUNCTION';
+K_FUNCTIONS: 'FUNCTIONS';
+K_GRANT: 'GRANT';
+K_IF: 'IF';
+K_IN: 'IN';
+K_INDEX: 'INDEX';
+K_INFINITY: 'INFINITY';
+K_INITCOND: 'INITCOND';
+K_INPUT: 'INPUT';
+K_INSERT: 'INSERT';
+K_INTO: 'INTO';
+K_IS: 'IS';
+K_JSON: 'JSON';
+K_KEY: 'KEY';
+K_KEYS: 'KEYS';
+K_KEYSPACE: 'KEYSPACE';
+K_KEYSPACES: 'KEYSPACES';
+K_LANGUAGE: 'LANGUAGE';
+K_LEVEL: 'LEVEL';
+K_LIMIT: 'LIMIT';
+K_LOCAL_ONE: 'LOCAL_ONE';
+K_LOCAL_QUORUM: 'LOCAL_QUORUM';
+K_LOGGED: 'LOGGED';
+K_LOGIN: 'LOGIN';
+K_MATERIALIZED: 'MATERIALIZED';
+K_MODIFY: 'MODIFY';
+K_NAN: 'NAN';
+K_NORECURSIVE: 'NORECURSIVE';
+K_NOSUPERUSER: 'NOSUPERUSER';
+K_NOT: 'NOT';
+K_NULL: 'NULL';
+K_OF: 'OF';
+K_ON: 'ON';
+K_ONE: 'ONE';
+K_OPTIONS: 'OPTIONS';
+K_OR: 'OR';
+K_ORDER: 'ORDER';
+K_PARTITION: 'PARTITION';
+K_PASSWORD: 'PASSWORD';
+K_PER: 'PER';
+K_PERMISSION: 'PERMISSION';
+K_PERMISSIONS: 'PERMISSIONS';
+K_PRIMARY: 'PRIMARY';
+K_QUORUM: 'QUORUM';
+K_RENAME: 'RENAME';
+K_REPLACE: 'REPLACE';
+K_REPLICATION: 'REPLICATION';
+K_RETURNS: 'RETURNS';
+K_REVOKE: 'REVOKE';
+K_ROLE: 'ROLE';
+K_ROLES: 'ROLES';
+K_SCHEMA: 'SCHEMA';
+K_SELECT: 'SELECT';
+K_SET: 'SET';
+K_SFUNC: 'SFUNC';
+K_STATIC: 'STATIC';
+K_STORAGE: 'STORAGE';
+K_STYPE: 'STYPE';
+K_SUPERUSER: 'SUPERUSER';
+K_TABLE: 'TABLE';
+K_THREE: 'THREE';
+K_TIMESTAMP: 'TIMESTAMP';
+K_TO: 'TO';
+K_TOKEN: 'TOKEN';
+K_TRIGGER: 'TRIGGER';
+K_TRUE: 'TRUE';
+K_TRUNCATE: 'TRUNCATE';
+K_TTL: 'TTL';
+K_TWO: 'TWO';
+K_TYPE: 'TYPE';
+K_UNLOGGED: 'UNLOGGED';
+K_UPDATE: 'UPDATE';
+K_USE: 'USE';
+K_USER: 'USER';
+K_USING: 'USING';
+K_UUID: 'UUID';
+K_VALUES: 'VALUES';
+K_VIEW: 'VIEW';
+K_WHERE: 'WHERE';
+K_WITH: 'WITH';
+K_WRITETIME: 'WRITETIME';
+K_ASCII: 'ASCII';
+K_BIGINT: 'BIGINT';
+K_BLOB: 'BLOB';
+K_BOOLEAN: 'BOOLEAN';
+K_COUNTER: 'COUNTER';
+K_DATE: 'DATE';
+K_DECIMAL: 'DECIMAL';
+K_DOUBLE: 'DOUBLE';
+K_FLOAT: 'FLOAT';
+K_FROZEN: 'FROZEN';
+K_INET: 'INET';
+K_INT: 'INT';
+K_LIST: 'LIST';
+K_MAP: 'MAP';
+K_SMALLINT: 'SMALLINT';
+K_TEXT: 'TEXT';
+K_TIMEUUID: 'TIMEUUID';
+K_TIME: 'TIME';
+K_TINYINT: 'TINYINT';
+K_TUPLE: 'TUPLE';
+K_VARCHAR: 'VARCHAR';
+K_VARINT: 'VARINT';
+
+// Literals
+
+CODE_BLOCK: '$$' (~ '$' | '$' ~'$')* '$$';
+
+STRING_LITERAL: '\'' ('\\' . | '\'\'' | ~('\'' | '\\'))* '\'';
+
+DECIMAL_LITERAL: DEC_DIGIT+;
+
+FLOAT_LITERAL: MINUS? [0-9]+ (DOT [0-9]+)?;
+
+HEXADECIMAL_LITERAL
+   : 'X' '\'' (HEX_DIGIT HEX_DIGIT)+ '\''
+	| '0X' HEX_DIGIT+
+   ;
+
+REAL_LITERAL
+	: DEC_DIGIT+ '.'? EXPONENT_NUM_PART
+	| DEC_DIGIT* '.' DEC_DIGIT+ EXPONENT_NUM_PART?
+   ;
+
+OBJECT_NAME
+   : [A-Z] [A-Z0-9_$]*
+	| '"' ~'"'+ '"'
+   ;
+
+UUID: HEX_4DIGIT HEX_4DIGIT '-' HEX_4DIGIT '-' HEX_4DIGIT '-' HEX_4DIGIT '-' HEX_4DIGIT HEX_4DIGIT HEX_4DIGIT;
+
+// Hidden
+
+SPACE: [ \t\r\n]+ -> channel (HIDDEN);
+SPEC_MYSQL_COMMENT: '/*!' .+? '*/' -> channel (HIDDEN);
+COMMENT_INPUT: '/*' .*? '*/' -> channel (HIDDEN);
+LINE_COMMENT: (
+		('-- ' | '#' | '//') ~ [\r\n]* ('\r'? '\n' | EOF)
+		| '--' ('\r'? '\n' | EOF)
+	) -> channel (HIDDEN);
+
+// Fragments
+
+fragment HEX_4DIGIT: [0-9A-F] [0-9A-F] [0-9A-F] [0-9A-F];
+
+fragment HEX_DIGIT: [0-9A-F];
+
+fragment DEC_DIGIT: [0-9];
+
+fragment EXPONENT_NUM_PART: 'E' '-'? DEC_DIGIT+;
+
+
+

virtdata-lang/src/main/java/io/nosqlbench/virtdata/lang/parser/CqlParserHarness.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.lang.parser;
+
+import io.nosqlbench.virtdata.lang.cqlast.CqlAstBuilder;
+import io.nosqlbench.virtdata.lang.generated.CqlLexer;
+import io.nosqlbench.virtdata.lang.generated.CqlParser;
+import org.antlr.v4.runtime.CharStreams;
+import org.antlr.v4.runtime.CodePointCharStream;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+
+public class CqlParserHarness {
+    private final static Logger logger  = LogManager.getLogger(CqlParserHarness.class);
+
+    public void parse(String input) {
+
+        try {
+            CodePointCharStream cstream = CharStreams.fromString(input);
+
+            CqlLexer lexer = new CqlLexer(cstream);
+            CommonTokenStream tokens = new CommonTokenStream(lexer);
+            CqlParser parser = new CqlParser(tokens);
+
+
+            CqlAstBuilder astListener = new CqlAstBuilder();
+            parser.addParseListener(astListener);
+
+            CqlParser.RootContext keyspaceParser = parser.root();
+            String tree = keyspaceParser.toStringTree();
+
+//            System.out.println("parsetree:\n" + tree);
+            System.out.println(astListener.toString());
+
+
+
+
+//            VirtDataParser.VirtdataFlowContext virtdataFlowContext = parser.virtdataFlow();
+//            logger.trace("parse tree: " + virtdataFlowContext.toStringTree(parser));
+
+//            if (astListener.hasErrors()) {
+//                System.out.println(astListener.getErrorNodes());
+//            }
+
+//            VirtDataAST ast = astListener.getModel();
+//            List<VirtDataFlow> flows = ast.getFlows();
+//            if (flows.size() > 1) {
+//                throw new RuntimeException("Only one flow expected here.");
+//            }
+//
+//            if (astListener.hasErrors()) {
+//                throw new RuntimeException("Error parsing input '" + input + "'");
+//            }
+//
+//            return new ParseResult(flows.get(0));
+
+        } catch (Exception e) {
+            logger.warn("Error while parsing flow:" + e.getMessage());
+            throw e;
+//            return new ParseResult(e);
+        }
+    }
+
+}

virtdata-lang/src/test/java/io/nosqlbench/virtdata/lang/parser/CqlParserHarnessTest.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2022 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.lang.parser;
+
+import org.junit.jupiter.api.Test;
+
+public class CqlParserHarnessTest {
+
+    @Test
+    public void testCqlParserHarness() {
+        CqlParserHarness harness = new CqlParserHarness();
+        harness.parse("""
+
+            CREATE KEYSPACE cycling
+              WITH REPLICATION = {\s
+               'class' : 'SimpleStrategy',\s
+               'replication_factor' : 1\s
+              };
+
+            CREATE TABLE cycling.race_winners (
+               race_name text,\s
+               race_position int,\s
+               cyclist_name FROZEN<fullname>,\s
+               PRIMARY KEY (race_name, race_position));
+            """);
+
+
+    }
+
+}