fully parsing the example file now

adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CQBErrorListener.java

@@ -20,12 +20,15 @@ import org.antlr.v4.runtime.BaseErrorListener;
 import org.antlr.v4.runtime.RecognitionException;
 import org.antlr.v4.runtime.Recognizer;
 
+import java.io.File;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
+import java.util.function.Supplier;
 
-public class CQBErrorListener extends BaseErrorListener {
+public class CQBErrorListener extends BaseErrorListener implements Supplier<List<String>> {
     List<String> errors = new ArrayList<>();
 
     private final Path origin;
@@ -37,20 +40,35 @@ public class CQBErrorListener extends BaseErrorListener {
     @Override
     public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) {
         String locref= getLocationRef(origin,line, charPositionInLine);
-        String errmsg = "Error in " + origin.toString()+ "\n:"+locref;
+        String errmsg = "Error in " + origin.toString()+ ":\n" +
+            e.toString() + ":\n"+locref;
         errors.add(errmsg);
     }
 
     public String getLocationRef(Path srcpath, int line, int charPositionInLine) {
+        Path cwd = Path.of(".").toAbsolutePath().normalize();
+        Path vcwd = cwd;
+
+        while (vcwd!=null) {
+            if (Files.exists(Path.of(vcwd + File.separator + ".git"))) {
+                break;
+            }
+            vcwd = vcwd.getParent();
+        }
+
         boolean inij = System.getProperty("sun.java.command","").toLowerCase(Locale.ROOT).contains("intellij");
-        Path vcwd = Path.of(".").toAbsolutePath().normalize();
+        vcwd = inij ? cwd.getParent().normalize() : vcwd;
-        vcwd = inij ? vcwd.getParent().normalize() : vcwd;
+
         Path relpath = vcwd.relativize(srcpath.toAbsolutePath());
+
         if (inij) {
-            relpath = Path.of("local/verizon/");
+            relpath = Path.of(relpath.toString().replace("target/classes/","src/main/resources/"));
-//            relpath = Path.of(relpath.toString().replace("target/classes/","src/main/resources/"));
         }
         return "\tat (" + relpath + ":" + line+":"+charPositionInLine + ")";
     }
 
+    @Override
+    public List<String> get() {
+        return this.errors;
+    }
 }

adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModel.java

@@ -16,32 +16,32 @@
 
 package io.nosqlbench.converters.cql.cqlast;
 
-import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Supplier;
 
 public class CqlModel {
 
+    private final Supplier<List<String>> errors;
     Map<String, CqlKeyspace> keyspaces = new LinkedHashMap<>();
+
     Map<String, Map<String, CqlTable>> tables = new LinkedHashMap<>();
 
-    public List<String> getErrors() {
-        return errors;
-    }
-
-    public void setErrors(List<String> errors) {
-        this.errors = errors;
-    }
-
-    List<String> errors = new ArrayList<>();
-
     transient
     CqlKeyspace keyspace = null;
     transient
     CqlTable table;
 
 
+    public CqlModel(Supplier<List<String>> errorSource) {
+        this.errors = errorSource;
+    }
+
+    public List<String> getErrors() {
+        return errors.get();
+    }
+
     public void newKeyspace() {
         keyspace = new CqlKeyspace();
     }
@@ -87,12 +87,12 @@ public class CqlModel {
         for (String ks : keyspaces.keySet()) {
             CqlKeyspace keyspace = keyspaces.get(ks);
             sb.append("keyspace '").append(keyspace.getKeyspaceName()).append("':\n");
-            sb.append(keyspace.toString()).append("\n");
+            sb.append(keyspace).append("\n");
 
             tables.getOrDefault(ks,Map.of()).values().stream()
                 .forEach(table -> {
                     sb.append("table '").append(table.getTableName()).append("':\n");
-                    sb.append(table.toString());
+                    sb.append(table);
                 });
         }
         return sb.toString();

adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/cqlast/CqlModelBuilder.java

@@ -28,10 +28,11 @@ import java.util.List;
 public class CqlModelBuilder extends CqlParserBaseListener {
 
     private final CQBErrorListener errorListener;
-    CqlModel model = new CqlModel();
+    private final CqlModel model;
 
     public CqlModelBuilder(CQBErrorListener errorListener) {
         this.errorListener = errorListener;
+        this.model = new CqlModel(errorListener);
     }
 
     @Override

adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/exporters/CqlWorkloadExporter.java

@@ -77,10 +77,10 @@ public class CqlWorkloadExporter {
         }
         Path srcpath = Path.of(args[0]);
         if (!srcpath.toString().endsWith(".cql")) {
-            throw new RuntimeException("File '" + srcpath.toString() + "' must end in .cql");
+            throw new RuntimeException("File '" + srcpath + "' must end in .cql");
         }
         if (!Files.exists(srcpath)) {
-            throw new RuntimeException("File '" + srcpath.toString() + "' does not exist.");
+            throw new RuntimeException("File '" + srcpath + "' does not exist.");
         }
 
         Path target = Path.of(srcpath.toString().replace("\\.cql", "\\.yaml"));
@@ -91,7 +91,7 @@ public class CqlWorkloadExporter {
             throw new RuntimeException("Target file must end in .yaml");
         }
         if (Files.exists(target) && !target.toString().startsWith("_")) {
-            throw new RuntimeException("Target file '" + target.toString() + "' exists. Please remove it first or use a different target file name.");
+            throw new RuntimeException("Target file '" + target + "' exists. Please remove it first or use a different target file name.");
         }
 
         CqlWorkloadExporter exporter = new CqlWorkloadExporter(srcpath);
@@ -128,7 +128,7 @@ public class CqlWorkloadExporter {
             model.getAllTables()
                 .stream()
                 .collect(Collectors.toMap(
-                    t -> "insert-" + t.getTableName(),
+                    t -> "insert-" + t.getKeySpace()+"__"+t.getTableName(),
                     this::genUpsertTemplate)
                 )
         );
@@ -137,7 +137,7 @@ public class CqlWorkloadExporter {
             model.getAllTables()
                 .stream()
                 .collect(Collectors.toMap(
-                    t -> "select-" + t.getTableName(),
+                    t -> "select-" + t.getKeySpace()+"__"+t.getTableName(),
                     this::genSelectTemplate)
                 ));
 
@@ -150,7 +150,7 @@ public class CqlWorkloadExporter {
         Map<String, String> rampupOpTemplates = model.getAllTables()
             .stream()
             .collect(Collectors.toMap(
-                t -> "insert-" + t.getTableName(),
+                t -> "insert-" + t.getKeySpace()+"__"+t.getTableName(),
                 this::genUpsertTemplate)
             );
 

adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/grammars/CqlLexer.g4

@@ -87,7 +87,8 @@ K_KEYS: 'KEYS';
 K_KEYSPACE: 'KEYSPACE';
 K_KEYSPACES: 'KEYSPACES';
 K_LANGUAGE: 'LANGUAGE';
-K_LEVEL: 'LEVEL';
+// Disabled because there was no definitive reference to this as a bare keyword in the specs
+//K_LEVEL: 'LEVEL';
 K_LIMIT: 'LIMIT';
 K_LOCAL_ONE: 'LOCAL_ONE';
 K_LOCAL_QUORUM: 'LOCAL_QUORUM';

adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/grammars/CqlParser.g4

@@ -367,8 +367,9 @@ dropIndex
    ;
 
 // TODO: update to https://docs.datastax.com/en/dse/6.8/cql/cql/cql_reference/cql_commands/cqlCreateTable.html
+// TODO: Update optional sequence for compact storage [options] clustering order [options] as documented above
 createTable
-   : kwCreate kwTable ifNotExist? (keyspace DOT)? table syntaxBracketLr columnDefinitionList syntaxBracketRr (withElement (kwAnd tableOptions)*)?
+   : kwCreate kwTable ifNotExist? (keyspace DOT)? table syntaxBracketLr columnDefinitionList syntaxBracketRr (kwWith tableOptions (kwAnd tableOptions)*)?
    ;
 
 withElement
@@ -386,15 +387,22 @@ tableOptions
 tableOptionItem
    : tableOptionName OPERATOR_EQ tableOptionValue
    | tableOptionName OPERATOR_EQ optionHash
+   | kwCompactStorage
+   | clusteringOrder
    ;
 
 tableOptionName
    : OBJECT_NAME
    ;
 
+kwCompactStorage
+   : K_COMPACT K_STORAGE
+   ;
+
 tableOptionValue
    : stringLiteral
    | floatLiteral
+   | booleanLiteral
    ;
 
 optionHash
@@ -447,7 +455,7 @@ compoundKey
    ;
 
 compositeKey
-   : syntaxBracketLr partitionKeyList syntaxBracketRr (syntaxComma clusteringKeyList)
+   : syntaxBracketLr partitionKeyList syntaxBracketRr (syntaxComma clusteringKeyList)?
    ;
 
 partitionKeyList
@@ -778,11 +786,15 @@ column
    ;
 
 dataType
-   : dataTypeName dataTypeDefinition?
+   : dataTypeName
+   | K_FROZEN syntaxBracketLa dataType syntaxBracketRa
+   | K_SET syntaxBracketLa dataType syntaxBracketRa
+   | K_LIST syntaxBracketLa dataType syntaxBracketRa
+   | K_MAP syntaxBracketLa dataType syntaxComma dataType syntaxBracketRa
    ;
 
 dataTypeName
-   : OBJECT_NAME
+   : (keyspace DOT)? OBJECT_NAME
    | K_TIMESTAMP
    | K_SET
    | K_ASCII
@@ -794,7 +806,7 @@ dataTypeName
    | K_DECIMAL
    | K_DOUBLE
    | K_FLOAT
-   | K_FROZEN
+//   | K_FROZEN
    | K_INET
    | K_INT
    | K_LIST
@@ -812,8 +824,11 @@ dataTypeName
    ;
 
 dataTypeDefinition
-   : syntaxBracketLa dataTypeName (syntaxComma dataTypeName)* syntaxBracketRa
+   : syntaxBracketLa dataType syntaxBracketRa
-   | syntaxBracketLa dataType syntaxBracketRa
+   | syntaxBracketLa dataTypeName (syntaxBracketLa dataTypeName syntaxBracketRa) syntaxBracketRa
+   | syntaxBracketLa dataTypeName (syntaxComma dataTypeName)* syntaxBracketRa
+   | syntaxBracketLa dataTypeName (syntaxComma syntaxBracketLa dataTypeName syntaxBracketRa)* syntaxBracketRa
+//   | syntaxBracketLa dataTypeName (syntaxComma dataTypeDefinition)* syntaxBracketRa
    ;
 
 orderDirection

adapter-cqld4/src/main/java/io/nosqlbench/converters/cql/parser/CqlModelParser.java

@@ -65,7 +65,7 @@ public class CqlModelParser {
             CqlModel model = cqlModelBuilder.getModel();
             if (model.getErrors().size()>0) {
                 System.out.println(model.getErrors());
-                throw new RuntimeException("Unable to render model for unparsable input with " + model.getErrors() + " errors");
+                throw new RuntimeException("Unable to render model for unparsable input with errors:\n" + model.getErrors());
             } else {
                 return model;
             }