From fbc6651e566430dc8ed89556bfdbd9bb64a01b7a Mon Sep 17 00:00:00 2001 From: Dave Fisher Date: Tue, 9 Jul 2024 12:54:09 -0700 Subject: [PATCH 1/7] Interpolate environment in at file processing --- .../main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java index f47ce4dfa..184a4e6d3 100644 --- a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java +++ b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java @@ -19,6 +19,7 @@ package io.nosqlbench.engine.cli.atfiles; import io.nosqlbench.nb.api.nbio.Content; import io.nosqlbench.nb.api.nbio.NBIO; import io.nosqlbench.nb.api.nbio.NBPathsAPI; +import io.nosqlbench.nb.api.system.NBEnvironment; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.snakeyaml.engine.v2.api.Load; @@ -145,8 +146,9 @@ public class NBAtFile { while (iter.hasNext()) { String word = iter.next(); String modified = word.replaceAll("\\$\\{DIR}",parent.toString()); + Optional interpolatedString = NBEnvironment.INSTANCE.interpolate(modified); iter.remove(); - iter.add(modified); + iter.add(interpolatedString.orElse("")); } return formatted; } From e2fdbd756a7db7e94da4d6ba61a6d2854860caa3 Mon Sep 17 00:00:00 2001 From: Pawan Kumar Date: Wed, 17 Jul 2024 10:28:56 +0530 Subject: [PATCH 2/7] Add conf option to disable/enable 'if not exists' for DDL statements. Closes #817 --- .../nosqlbench/cqlgen/core/CGWorkloadExporter.java | 13 ++++++++++--- .../src/main/resources/cqlgen/cqlgen.conf | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java index 0a4ec8cde..4678917db 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java @@ -72,6 +72,7 @@ public class CGWorkloadExporter implements BundledApp { private String namingTemplate; private double partitionMultiplier; private int quantizerDigits; + private boolean disableIfNotExists = false; private Map> blockplan = Map.of(); private final Map timeouts = new HashMap(Map.of( @@ -163,6 +164,9 @@ public class CGWorkloadExporter implements BundledApp { configureTimeouts(cfgmap.get("timeouts")); configureBlocks(cfgmap.get("blockplan")); configureQuantizerDigits(cfgmap.get("quantizer_digits")); + if (cfgmap.get("disable_if_not_exists").equals(true)) { + disableIfNotExists = true; + } this.model = CqlModelParser.parse(ddl, srcpath); List errorlist = model.getReferenceErrors(); @@ -736,9 +740,10 @@ public class CGWorkloadExporter implements BundledApp { private String genKeyspaceDDL(CqlKeyspaceDef keyspace) { return """ - create keyspace KEYSPACE + create keyspace IF_NOT_EXISTS KEYSPACE with replication = {REPLICATION}DURABLEWRITES?; """ + .replace("IF_NOT_EXISTS", disableIfNotExists ? "" : "if not exists") .replace("KEYSPACE", keyspace.getName()) .replace("REPLICATION", keyspace.getReplicationData()) .replace("DURABLEWRITES?", keyspace.isDurableWrites() ? "" : "\n and durable writes = false") @@ -766,10 +771,11 @@ public class CGWorkloadExporter implements BundledApp { private String genTypeDDL(CqlType type) { return """ - create type KEYSPACE.TYPENAME ( + create type IF_NOT_EXISTS KEYSPACE.TYPENAME ( TYPEDEF ); """ + .replace("IF_NOT_EXISTS", disableIfNotExists ? "" : "if not exists") .replace("KEYSPACE", type.getKeyspace().getName()) .replace("TYPENAME", type.getName()) .replace("TYPEDEF", type.getColumnDefs().stream() @@ -782,11 +788,12 @@ public class CGWorkloadExporter implements BundledApp { } return """ - create table if not exists KEYSPACE.TABLE ( + create table IF_NOT_EXISTS KEYSPACE.TABLE ( COLUMN_DEFS, primary key (PRIMARYKEY) )CLUSTERING; """ + .replace("IF_NOT_EXISTS", disableIfNotExists ? "" : "if not exists") .replace("KEYSPACE", cqltable.getKeyspace().getName()) .replace("TABLE", cqltable.getName()) .replace("COLUMN_DEFS", genTableColumnDDL(cqltable)) diff --git a/nb-adapters/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf b/nb-adapters/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf index 0aab9c35c..0a8453e35 100644 --- a/nb-adapters/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf +++ b/nb-adapters/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf @@ -145,6 +145,6 @@ blockplan: # not needed when tags=block:'main.*' # main: insert, select, scan-10, update - - +# Configuration option for removing 'IF NOT EXISTS' in all generated DDL statements +disable_if_not_exists: false From 310de7eee708102ad9586d2b318b2e2ef50f0d88 Mon Sep 17 00:00:00 2001 From: Pawan Kumar Date: Thu, 18 Jul 2024 12:22:50 +0530 Subject: [PATCH 3/7] rename conf disable_if_not_exists to enable_if_exists --- .../cqlgen/core/CGWorkloadExporter.java | 18 +++++++++--------- .../src/main/resources/cqlgen/cqlgen.conf | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java index 4678917db..4f7efcea8 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java @@ -72,7 +72,7 @@ public class CGWorkloadExporter implements BundledApp { private String namingTemplate; private double partitionMultiplier; private int quantizerDigits; - private boolean disableIfNotExists = false; + private boolean enableIfExists = true; private Map> blockplan = Map.of(); private final Map timeouts = new HashMap(Map.of( @@ -164,8 +164,8 @@ public class CGWorkloadExporter implements BundledApp { configureTimeouts(cfgmap.get("timeouts")); configureBlocks(cfgmap.get("blockplan")); configureQuantizerDigits(cfgmap.get("quantizer_digits")); - if (cfgmap.get("disable_if_not_exists").equals(true)) { - disableIfNotExists = true; + if (cfgmap.get("enable_if_exists").equals(false)) { + enableIfExists = false; } this.model = CqlModelParser.parse(ddl, srcpath); @@ -643,7 +643,7 @@ public class CGWorkloadExporter implements BundledApp { ops.put( namer.nameFor(table, "optype", "drop", "blockname", blockname), Map.of( - "simple", "drop table if exists " + table.getFullName() + ";", + "simple", (enableIfExists ? "drop table if exists " : "drop table ") + table.getFullName() + ";", "timeout", timeouts.get("drop") ) ); @@ -659,7 +659,7 @@ public class CGWorkloadExporter implements BundledApp { ops.put( namer.nameFor(type, "optype", "drop-type", "blockname", blockname), Map.of( - "simple", "drop type if exists " + type.getKeyspace() + "." + type.getName() + ";", + "simple", (enableIfExists ? "drop type if exists " : "drop type ") + "." + type.getName() + ";", "timeout", timeouts.get("drop") ) ); @@ -675,7 +675,7 @@ public class CGWorkloadExporter implements BundledApp { ops.put( namer.nameFor(type, "optype", "drop-keyspace", "blockname", blockname), Map.of( - "simple", "drop keyspace if exists " + type.getKeyspace() + ";", + "simple", (enableIfExists ? "drop keyspace if exists " : "drop keyspace ") + type.getKeyspace() + ";", "timeout", timeouts.get("drop") ) ); @@ -743,7 +743,7 @@ public class CGWorkloadExporter implements BundledApp { create keyspace IF_NOT_EXISTS KEYSPACE with replication = {REPLICATION}DURABLEWRITES?; """ - .replace("IF_NOT_EXISTS", disableIfNotExists ? "" : "if not exists") + .replace("IF_NOT_EXISTS", enableIfExists ? "if not exists" : "") .replace("KEYSPACE", keyspace.getName()) .replace("REPLICATION", keyspace.getReplicationData()) .replace("DURABLEWRITES?", keyspace.isDurableWrites() ? "" : "\n and durable writes = false") @@ -775,7 +775,7 @@ public class CGWorkloadExporter implements BundledApp { TYPEDEF ); """ - .replace("IF_NOT_EXISTS", disableIfNotExists ? "" : "if not exists") + .replace("IF_NOT_EXISTS", enableIfExists ? "if not exists" : "") .replace("KEYSPACE", type.getKeyspace().getName()) .replace("TYPENAME", type.getName()) .replace("TYPEDEF", type.getColumnDefs().stream() @@ -793,7 +793,7 @@ public class CGWorkloadExporter implements BundledApp { primary key (PRIMARYKEY) )CLUSTERING; """ - .replace("IF_NOT_EXISTS", disableIfNotExists ? "" : "if not exists") + .replace("IF_NOT_EXISTS", enableIfExists ? "if not exists" : "") .replace("KEYSPACE", cqltable.getKeyspace().getName()) .replace("TABLE", cqltable.getName()) .replace("COLUMN_DEFS", genTableColumnDDL(cqltable)) diff --git a/nb-adapters/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf b/nb-adapters/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf index 0a8453e35..494fb4424 100644 --- a/nb-adapters/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf +++ b/nb-adapters/adapter-cqld4/src/main/resources/cqlgen/cqlgen.conf @@ -145,6 +145,6 @@ blockplan: # not needed when tags=block:'main.*' # main: insert, select, scan-10, update -# Configuration option for removing 'IF NOT EXISTS' in all generated DDL statements -disable_if_not_exists: false +# Configuration option for adding 'IF NOT EXISTS' or 'IF EXISTS' in all generated DDL statements +enable_if_exists: true From 1ff76aebd4e9dcc18e2a4cf1f56dc3552f2f8ac6 Mon Sep 17 00:00:00 2001 From: Dave Fisher Date: Thu, 18 Jul 2024 11:07:30 -0700 Subject: [PATCH 4/7] Throw error when envvar is missing and provide unit tests --- mvn-defaults/pom.xml | 3 +++ .../io/nosqlbench/engine/cli/atfiles/NBAtFile.java | 3 ++- .../nosqlbench/engine/cli/atfiles/NBAtFileTest.java | 11 +++++++++++ .../atfiles/environment_variable_missing.yaml | 1 + .../test/resources/atfiles/envronment_variable.yaml | 1 + 5 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 nb-engine/nb-engine-cli/src/test/resources/atfiles/environment_variable_missing.yaml create mode 100644 nb-engine/nb-engine-cli/src/test/resources/atfiles/envronment_variable.yaml diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index 4a3e01b5b..38641c0bd 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -573,6 +573,9 @@ false + + value + diff --git a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java index aab40dc5f..a81292c71 100644 --- a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java +++ b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java @@ -169,8 +169,9 @@ public class NBAtFile { String word = iter.next(); String modified = word.replaceAll("\\$\\{DIR}",parent.toString()); Optional interpolatedString = NBEnvironment.INSTANCE.interpolate(modified); + String value = interpolatedString.orElseThrow(() -> new RuntimeException("Unable to find environment variable or property in text '"+modified+"' in atfile '" + atPath + "'")); iter.remove(); - iter.add(interpolatedString.orElse("")); + iter.add(value); } return formatted; } diff --git a/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java b/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java index be2b6e08e..5979472d7 100644 --- a/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java +++ b/nb-engine/nb-engine-cli/src/test/java/io/nosqlbench/engine/cli/atfiles/NBAtFileTest.java @@ -91,4 +91,15 @@ class NBAtFileTest { assertThat(strings).containsExactly("arg1","arg1","arg1","arg2","arg3","arg3","arg3","deepval"); } + @Test + public void testAtfileEnvironmentVariable() { + LinkedList strings = NBAtFile.includeAt("@src/test/resources/atfiles/environment_variable.yaml"); + assertThat(strings).containsExactly("My value environment"); + } + + @Test + public void testAtfileMissingEnvironmentVariable() { + assertThrows(RuntimeException.class, () -> NBAtFile.includeAt("@src/test/resources/atfiles/environment_variable_missing.yaml:>:")); + } + } diff --git a/nb-engine/nb-engine-cli/src/test/resources/atfiles/environment_variable_missing.yaml b/nb-engine/nb-engine-cli/src/test/resources/atfiles/environment_variable_missing.yaml new file mode 100644 index 000000000..48a8a26df --- /dev/null +++ b/nb-engine/nb-engine-cli/src/test/resources/atfiles/environment_variable_missing.yaml @@ -0,0 +1 @@ +- My ${MISSING_ENV_VAR} environment diff --git a/nb-engine/nb-engine-cli/src/test/resources/atfiles/envronment_variable.yaml b/nb-engine/nb-engine-cli/src/test/resources/atfiles/envronment_variable.yaml new file mode 100644 index 000000000..71f5e2282 --- /dev/null +++ b/nb-engine/nb-engine-cli/src/test/resources/atfiles/envronment_variable.yaml @@ -0,0 +1 @@ +- My ${TEST_ENV_VAR} environment From a8c71656f97a0c2a9170642625d0cba2e64d0b30 Mon Sep 17 00:00:00 2001 From: Dave Fisher Date: Thu, 18 Jul 2024 11:17:02 -0700 Subject: [PATCH 5/7] Correct filename --- .../{envronment_variable.yaml => environment_variable.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename nb-engine/nb-engine-cli/src/test/resources/atfiles/{envronment_variable.yaml => environment_variable.yaml} (100%) diff --git a/nb-engine/nb-engine-cli/src/test/resources/atfiles/envronment_variable.yaml b/nb-engine/nb-engine-cli/src/test/resources/atfiles/environment_variable.yaml similarity index 100% rename from nb-engine/nb-engine-cli/src/test/resources/atfiles/envronment_variable.yaml rename to nb-engine/nb-engine-cli/src/test/resources/atfiles/environment_variable.yaml From 34a3ad478a50217a33becd4dcc91d012d7f71ba2 Mon Sep 17 00:00:00 2001 From: Dave Fisher Date: Thu, 18 Jul 2024 11:32:03 -0700 Subject: [PATCH 6/7] OF: Use spaces not tab --- .../main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java index a81292c71..3901ea6ce 100644 --- a/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java +++ b/nb-engine/nb-engine-cli/src/main/java/io/nosqlbench/engine/cli/atfiles/NBAtFile.java @@ -169,7 +169,7 @@ public class NBAtFile { String word = iter.next(); String modified = word.replaceAll("\\$\\{DIR}",parent.toString()); Optional interpolatedString = NBEnvironment.INSTANCE.interpolate(modified); - String value = interpolatedString.orElseThrow(() -> new RuntimeException("Unable to find environment variable or property in text '"+modified+"' in atfile '" + atPath + "'")); + String value = interpolatedString.orElseThrow(() -> new RuntimeException("Unable to find environment variable or property in text '"+modified+"' in atfile '" + atPath + "'")); iter.remove(); iter.add(value); } From 0b7292c3b53c4bc61b949c004e077dfdf1ca6d5c Mon Sep 17 00:00:00 2001 From: Dave Fisher Date: Tue, 23 Jul 2024 12:59:15 -0700 Subject: [PATCH 7/7] Fix test regex --- .../src/test/java/io/nosqlbench/nb/api/SystemIdTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nb-apis/nb-api/src/test/java/io/nosqlbench/nb/api/SystemIdTest.java b/nb-apis/nb-api/src/test/java/io/nosqlbench/nb/api/SystemIdTest.java index f2b5d3a5c..b9c751964 100644 --- a/nb-apis/nb-api/src/test/java/io/nosqlbench/nb/api/SystemIdTest.java +++ b/nb-apis/nb-api/src/test/java/io/nosqlbench/nb/api/SystemIdTest.java @@ -56,14 +56,14 @@ public class SystemIdTest { @Test public void testPackedNodeId() { String packedNodeId = SystemId.getPackedNodeId(); - assertThat(packedNodeId).matches("[0-9A-Za-z_-]+"); + assertThat(packedNodeId).matches("[0-9A-Za-z_~-]+"); logger.info("packed node id: " + packedNodeId); } @Test public void testGenSessionCode() { String sessionCode=SystemId.genSessionCode(234L); - assertThat(sessionCode).matches("[0-9a-zA-Z~-]+"); + assertThat(sessionCode).matches("[0-9a-zA-Z_~-]+"); logger.info("session code: " + sessionCode); }