Merge branch 'main' into my-astra-schema-examples

mvn-defaults/pom.xml

@@ -573,6 +573,9 @@
                     </includes>
                     <useSystemClassLoader>false
                     </useSystemClassLoader> <!-- fixes reflection tests -->
+                    <environmentVariables>
+                        <TEST_ENV_VAR>value</TEST_ENV_VAR>
+                    </environmentVariables>
                 </configuration>
             </plugin>
 

nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java

@@ -72,6 +72,7 @@ public class CGWorkloadExporter implements BundledApp {
     private String namingTemplate;
     private double partitionMultiplier;
     private int quantizerDigits;
+    private boolean enableIfExists = true;
     private Map<String, List<String>> blockplan = Map.of();
 
     private final Map<String, Double> timeouts = new HashMap<String, Double>(Map.of(
@@ -163,6 +164,9 @@ public class CGWorkloadExporter implements BundledApp {
         configureTimeouts(cfgmap.get("timeouts"));
         configureBlocks(cfgmap.get("blockplan"));
         configureQuantizerDigits(cfgmap.get("quantizer_digits"));
+        if (cfgmap.get("enable_if_exists").equals(false)) {
+            enableIfExists = false;
+        }
 
         this.model = CqlModelParser.parse(ddl, srcpath);
         List<String> errorlist = model.getReferenceErrors();
@@ -639,7 +643,7 @@ public class CGWorkloadExporter implements BundledApp {
             ops.put(
                     namer.nameFor(table, "optype", "drop", "blockname", blockname),
                     Map.of(
-                            "simple", "drop table if exists " + table.getFullName() + ";",
+                            "simple", (enableIfExists ? "drop table if exists " : "drop table ") + table.getFullName() + ";",
                             "timeout", timeouts.get("drop")
                     )
             );
@@ -655,7 +659,7 @@ public class CGWorkloadExporter implements BundledApp {
             ops.put(
                     namer.nameFor(type, "optype", "drop-type", "blockname", blockname),
                     Map.of(
-                            "simple", "drop type if exists " + type.getKeyspace() + "." + type.getName() + ";",
+                            "simple", (enableIfExists ? "drop type if exists " : "drop type ") + "." + type.getName() + ";",
                             "timeout", timeouts.get("drop")
                     )
             );
@@ -671,7 +675,7 @@ public class CGWorkloadExporter implements BundledApp {
             ops.put(
                     namer.nameFor(type, "optype", "drop-keyspace", "blockname", blockname),
                     Map.of(
-                            "simple", "drop keyspace if exists " + type.getKeyspace() + ";",
+                            "simple", (enableIfExists ? "drop keyspace if exists " : "drop keyspace ") + type.getKeyspace() + ";",
                             "timeout", timeouts.get("drop")
                     )
             );
@@ -736,9 +740,10 @@ public class CGWorkloadExporter implements BundledApp {
 
     private String genKeyspaceDDL(CqlKeyspaceDef keyspace) {
         return """
-                create keyspace KEYSPACE
+                create keyspace IF_NOT_EXISTS KEYSPACE
                 with replication = {REPLICATION}DURABLEWRITES?;
                 """
+                .replace("IF_NOT_EXISTS", enableIfExists ? "if not exists" : "")
                 .replace("KEYSPACE", keyspace.getName())
                 .replace("REPLICATION", keyspace.getReplicationData())
                 .replace("DURABLEWRITES?", keyspace.isDurableWrites() ? "" : "\n and durable writes = false")
@@ -766,10 +771,11 @@ public class CGWorkloadExporter implements BundledApp {
 
     private String genTypeDDL(CqlType type) {
         return """
-                create type KEYSPACE.TYPENAME (
+                create type IF_NOT_EXISTS KEYSPACE.TYPENAME (
                 TYPEDEF
                 );
                 """
+                .replace("IF_NOT_EXISTS", enableIfExists ? "if not exists" : "")
                 .replace("KEYSPACE", type.getKeyspace().getName())
                 .replace("TYPENAME", type.getName())
                 .replace("TYPEDEF", type.getColumnDefs().stream()
@@ -782,11 +788,12 @@ public class CGWorkloadExporter implements BundledApp {
         }
 
         return """
-                create table if not exists KEYSPACE.TABLE (
+                create table IF_NOT_EXISTS KEYSPACE.TABLE (
                 COLUMN_DEFS,
                 primary key (PRIMARYKEY)
                 )CLUSTERING;
                 """
+                .replace("IF_NOT_EXISTS", enableIfExists ? "if not exists" : "")
                 .replace("KEYSPACE", cqltable.getKeyspace().getName())
                 .replace("TABLE", cqltable.getName())
                 .replace("COLUMN_DEFS", genTableColumnDDL(cqltable))

nb-adapters/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf

@@ -145,6 +145,6 @@ blockplan:
   # not needed when tags=block:'main.*'
   # main: insert, select, scan-10, update
 
-
+# Configuration option for adding 'IF NOT EXISTS' or 'IF EXISTS' in all generated DDL statements
-
+enable_if_exists: true
 

nb-apis/nb-api/src/test/java/io/nosqlbench/nb/api/SystemIdTest.java

@@ -56,14 +56,14 @@ public class SystemIdTest {
     @Test
     public void testPackedNodeId() {
         String packedNodeId = SystemId.getPackedNodeId();
-        assertThat(packedNodeId).matches("[0-9A-Za-z_-]+");
+        assertThat(packedNodeId).matches("[0-9A-Za-z_~-]+");
         logger.info("packed node id: " + packedNodeId);
     }
 
     @Test
     public void testGenSessionCode() {
         String sessionCode=SystemId.genSessionCode(234L);
-        assertThat(sessionCode).matches("[0-9a-zA-Z~-]+");
+        assertThat(sessionCode).matches("[0-9a-zA-Z_~-]+");
         logger.info("session code: " + sessionCode);
     }
 

nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java

@@ -19,6 +19,7 @@ package io.nosqlbench.engine.cli.atfiles;
 import io.nosqlbench.nb.api.nbio.Content;
 import io.nosqlbench.nb.api.nbio.NBIO;
 import io.nosqlbench.nb.api.nbio.NBPathsAPI;
+import io.nosqlbench.nb.api.system.NBEnvironment;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
@@ -166,8 +167,10 @@ public class NBAtFile {
         while (iter.hasNext()) {
             String word = iter.next();
             String modified = word.replaceAll("\\$\\{DIR}",parent.toString());
+            Optional<String> interpolatedString = NBEnvironment.INSTANCE.interpolate(modified);
+            String value = interpolatedString.orElseThrow(() -> new RuntimeException("Unable to find environment variable or property in text '"+modified+"' in atfile '" + atPath + "'"));
             iter.remove();
-            iter.add(modified);
+            iter.add(value);
         }
         return formatted;
     }

nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java

@@ -91,4 +91,15 @@ class NBAtFileTest {
         assertThat(strings).containsExactly("arg1","arg1","arg1","arg2","arg3","arg3","arg3","deepval");
     }
 
+    @Test
+    public void testAtfileEnvironmentVariable() {
+        LinkedList<String> strings = NBAtFile.includeAt("@src/test/resources/atfiles/environment_variable.yaml");
+        assertThat(strings).containsExactly("My value environment");
+    }
+
+    @Test
+    public void testAtfileMissingEnvironmentVariable() {
+        assertThrows(RuntimeException.class, () -> NBAtFile.includeAt("@src/test/resources/atfiles/environment_variable_missing.yaml:>:"));
+    }
+
 }

nb-engine/nb-engine-cli/src/test/resources/atfiles/environment_variable.yaml

@@ -0,0 +1 @@
+- My ${TEST_ENV_VAR} environment

nb-engine/nb-engine-cli/src/test/resources/atfiles/environment_variable_missing.yaml

@@ -0,0 +1 @@
+- My ${MISSING_ENV_VAR} environment