Merge pull request #2027 from nosqlbench/spanner

Adding Create & Database DDL options to the existing ops

.run/nb --list-drivers.run.xml

@@ -5,7 +5,7 @@
       <option name="region" />
       <option name="useCurrentConnection" value="false" />
     </extension>
-    <option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5-5.21.1-SNAPSHOT-jar-with-dependencies.jar" />
+    <option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
     <option name="PROGRAM_PARAMETERS" value="--list-drivers --show-stacktraces" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/local/" />
     <option name="ALTERNATIVE_JRE_PATH" />

.run/nb --list-scenarios.run.xml

@@ -5,7 +5,7 @@
       <option name="region" />
       <option name="useCurrentConnection" value="false" />
     </extension>
-    <option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5-5.21.1-SNAPSHOT-jar-with-dependencies.jar" />
+    <option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
     <option name="PROGRAM_PARAMETERS" value="--list-scenarios --show-stacktraces" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/local/" />
     <option name="ALTERNATIVE_JRE_PATH" />

.run/spanner_schema_database_create.run.xml

@@ -0,0 +1,18 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="spanner_schema_database_create" type="JarApplication" folderName="Google Spanner">
+    <extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
+      <option name="credential" />
+      <option name="region" />
+      <option name="useCurrentConnection" value="false" />
+    </extension>
+    <option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
+    <option name="VM_PARAMETERS" value="--enable-preview" />
+    <option name="PROGRAM_PARAMETERS" value="create_database_ddl default.create_db_ddl project_id=stratperf instance_id=perftest-spanner database_id=baselines1 service_account_file=./stratperf-068083077315.json --progress console:1s -vv --add-labels=&quot;target:spanner,instance:vectors,vendor:gcp&quot; --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators &quot;[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]&quot; --report-interval 10 --show-stacktraces --logs-max 5" />
+    <option name="WORKING_DIRECTORY" value="$ProjectFileDir$/local/google_spanner" />
+    <option name="ALTERNATIVE_JRE_PATH" value="jdk21" />
+    <envs>
+      <env name="GOOGLE_APPLICATION_CREDENTIALS" value="$ProjectFileDir$/local/google_spanner/stratperf-068083077315.json" />
+    </envs>
+    <method v="2" />
+  </configuration>
+</component>

nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java

@@ -55,6 +55,10 @@ public class GCPSpannerOpMapper implements OpMapper<GCPSpannerBaseOp<?>> {
         logger.info(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'");
 
         return switch (typeAndTarget.enumId) {
+            case drop_database_ddl ->
+                new GCPSpannerDropDatabaseDdlOpDispenser(adapter, op, typeAndTarget.targetFunction);
+            case create_database_ddl ->
+                new GCPSpannerCreateDatabaseDdlOpDispenser(adapter, op, typeAndTarget.targetFunction);
             case update_database_ddl ->
                 new GCPSpannerUpdateDatabaseDdlOpDispenser(adapter, op, typeAndTarget.targetFunction);
             case insert ->

nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateDatabaseDdlOpDispenser.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2020-2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.adapter.gcpspanner.opdispensers;
+
+
+import io.nosqlbench.adapter.gcpspanner.GCPSpannerDriverAdapter;
+import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp;
+import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerCreateDatabaseDdlOp;
+import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerUpdateDatabaseDdlOp;
+import io.nosqlbench.adapters.api.templating.ParsedOp;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.function.LongFunction;
+
+/**
+ * Dispenser class for creating databases of {@link GCPSpannerCreateDatabaseDdlOp}.
+ *
+ * @see <a href="https://cloud.google.com/spanner/docs/reference/rpc/google.spanner.admin.database.v1#createdatabaserequest">
+ *     CreateDatabaseRequest</a> which can be a stretch goal to combine all of DB, Table(s), and Indexes into one-single call.
+ */
+public class GCPSpannerCreateDatabaseDdlOpDispenser extends GCPSpannerBaseOpDispenser {
+    private static final Logger logger = LogManager.getLogger(GCPSpannerCreateDatabaseDdlOpDispenser.class);
+    private final LongFunction<GCPSpannerCreateDatabaseDdlOp> opFunction;
+
+    /**
+     * Constructor for {@link GCPSpannerCreateDatabaseDdlOpDispenser}.
+     *
+     * @param adapter        the {@link GCPSpannerDriverAdapter} instance
+     * @param op             the {@link ParsedOp} instance
+     * @param targetFunction a {@link LongFunction} that provides the target string
+     */
+    public GCPSpannerCreateDatabaseDdlOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction<String> targetFunction) {
+        super(adapter, op, targetFunction);
+        this.opFunction = createOpFunction(op);
+    }
+
+    /**
+     * Creates a {@link LongFunction} that generates {@link GCPSpannerUpdateDatabaseDdlOp} instances.
+     *
+     * @param op the {@link ParsedOp} instance
+     * @return a {@link LongFunction} that generates {@link GCPSpannerUpdateDatabaseDdlOp} instances
+     */
+    private LongFunction<GCPSpannerCreateDatabaseDdlOp> createOpFunction(ParsedOp op) {
+        return (l) -> new GCPSpannerCreateDatabaseDdlOp(
+            spaceFunction.apply(l).getSpanner(),
+            l,
+            targetFunction.apply(l),
+            spaceFunction.apply(l).getDbAdminClient(),
+            spaceFunction.apply(l).getInstanceId()
+        );
+    }
+
+    /**
+     * Retrieves an operation instance based on the provided value.
+     *
+     * @param value the long value used to generate the operation
+     * @return a {@link GCPSpannerBaseOp} instance
+     */
+    @Override
+    public GCPSpannerBaseOp<?> getOp(long value) {
+        return opFunction.apply(value);
+    }
+}

nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerDropDatabaseDdlOpDispenser.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2020-2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.adapter.gcpspanner.opdispensers;
+
+
+import io.nosqlbench.adapter.gcpspanner.GCPSpannerDriverAdapter;
+import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp;
+import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerDropDatabaseDdlOp;
+import io.nosqlbench.adapters.api.templating.ParsedOp;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.function.LongFunction;
+
+/**
+ * Dispenser class for creating databases of {@link GCPSpannerDropDatabaseDdlOp}.
+ *
+ * @see <a href="https://cloud.google.com/spanner/docs/reference/rpc/google.spanner.admin.database.v1#dropdatabaserequest">
+ *     DropDatabaseRequest</a> which can be a stretch goal to combine all of DB, Table(s), and Indexes into one-single call.
+ */
+public class GCPSpannerDropDatabaseDdlOpDispenser extends GCPSpannerBaseOpDispenser {
+    private static final Logger logger = LogManager.getLogger(GCPSpannerDropDatabaseDdlOpDispenser.class);
+    private final LongFunction<GCPSpannerDropDatabaseDdlOp> opFunction;
+
+    /**
+     * Constructor for {@link GCPSpannerDropDatabaseDdlOpDispenser}.
+     *
+     * @param adapter        the {@link GCPSpannerDriverAdapter} instance
+     * @param op             the {@link ParsedOp} instance
+     * @param targetFunction a {@link LongFunction} that provides the target string
+     */
+    public GCPSpannerDropDatabaseDdlOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction<String> targetFunction) {
+        super(adapter, op, targetFunction);
+        this.opFunction = DropOpFunction(op);
+    }
+
+    /**
+     * Drops a {@link LongFunction} that generates {@link GCPSpannerDropDatabaseDdlOp} instances.
+     *
+     * @param op the {@link ParsedOp} instance
+     * @return a {@link LongFunction} that generates {@link GCPSpannerDropDatabaseDdlOp} instances
+     */
+    private LongFunction<GCPSpannerDropDatabaseDdlOp> DropOpFunction(ParsedOp op) {
+        return (l) -> new GCPSpannerDropDatabaseDdlOp(
+            spaceFunction.apply(l).getSpanner(),
+            l,
+            targetFunction.apply(l),
+            spaceFunction.apply(l).getDbAdminClient(),
+            spaceFunction.apply(l).getDbAdminClient().getDatabase(spaceFunction.apply(l).getInstanceId(), spaceFunction.apply(l).getDatabaseIdString())
+        );
+    }
+
+    /**
+     * Retrieves an operation instance based on the provided value.
+     *
+     * @param value the long value used to generate the operation
+     * @return a {@link GCPSpannerBaseOp} instance
+     */
+    @Override
+    public GCPSpannerBaseOp<?> getOp(long value) {
+        return opFunction.apply(value);
+    }
+}

nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateDatabaseDdlOp.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2020-2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.adapter.gcpspanner.ops;
+
+import com.google.api.gax.longrunning.OperationFuture;
+import com.google.cloud.spanner.Database;
+import com.google.cloud.spanner.DatabaseAdminClient;
+import com.google.cloud.spanner.Spanner;
+import com.google.spanner.admin.database.v1.CreateDatabaseMetadata;
+
+/**
+ * This class represents an operation to create the database DDL (Data Definition Language) in Google Cloud Spanner.
+ * It extends the {@link GCPSpannerBaseOp} class and provides the implementation for applying the DDL update operation.
+ */
+public class GCPSpannerCreateDatabaseDdlOp extends GCPSpannerBaseOp<Long> {
+    private final String createDatbaseStatement;
+    private final DatabaseAdminClient dbAdminClient;
+    private final String instanceId;
+
+    /**
+     * Constructs a new {@link GCPSpannerUpdateDatabaseDdlOp}.
+     *
+     * @param searchIndexClient the {@link Spanner} client
+     * @param requestParam the request parameter
+     * @param createDatbaseStatement the SQL statement to create the table
+     * @param dbAdminClient the {@link DatabaseAdminClient} to execute the DDL update
+     * @param instanceId the instance ID string representing the target spanner instance
+     */
+    public GCPSpannerCreateDatabaseDdlOp(Spanner searchIndexClient, Long requestParam, String createDatbaseStatement,
+                                         DatabaseAdminClient dbAdminClient, String instanceId) {
+        super(searchIndexClient, requestParam);
+        this.createDatbaseStatement = createDatbaseStatement;
+        this.dbAdminClient = dbAdminClient;
+        this.instanceId = instanceId;
+    }
+
+    /**
+     * Applies the DDL create operation.
+     *
+     * @param value the value to be used in the operation
+     * @return the result of the operation
+     * @throws RuntimeException if an error occurs during the operation
+     */
+    @Override
+    public Object applyOp(long value) {
+        OperationFuture<Database, CreateDatabaseMetadata> operation = dbAdminClient.createDatabase(
+            instanceId,
+            createDatbaseStatement,
+            null,
+            null);
+        try {
+            return operation.get();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerDropDatabaseDdlOp.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2020-2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.adapter.gcpspanner.ops;
+
+import com.google.cloud.spanner.Database;
+import com.google.cloud.spanner.DatabaseAdminClient;
+import com.google.cloud.spanner.Spanner;
+
+/**
+ * This class represents an operation to Drop the database DDL (Data Definition Language) in Google Cloud Spanner.
+ * It extends the {@link GCPSpannerBaseOp} class and provides the implementation for applying the DDL update operation.
+ */
+public class GCPSpannerDropDatabaseDdlOp extends GCPSpannerBaseOp<Long> {
+    private final String databaseId;
+    private final DatabaseAdminClient dbAdminClient;
+    private final Database db;
+
+    /**
+     * Constructs a new {@link GCPSpannerUpdateDatabaseDdlOp}.
+     *
+     * @param searchIndexClient the {@link Spanner} client
+     * @param requestParam the request parameter
+     * @param databaseId the database ID to be dropped
+     * @param dbAdminClient the {@link DatabaseAdminClient} to execute the DDL update
+     * @param db the {@link Database} to be dropped
+     */
+    public GCPSpannerDropDatabaseDdlOp(Spanner searchIndexClient, Long requestParam, String databaseId,
+                                         DatabaseAdminClient dbAdminClient, Database db) {
+        super(searchIndexClient, requestParam);
+        this.databaseId = databaseId;
+        this.dbAdminClient = dbAdminClient;
+        this.db = db;
+    }
+
+    /**
+     * Applies the DDL drop operation.
+     *
+     * @param value the value to be used in the operation
+     * @return the result of the operation
+     * @throws RuntimeException if an error occurs during the operation
+     */
+    @Override
+    public Object applyOp(long value) {
+        try {
+            db.drop();
+        } catch (Exception e) {
+            logger.warn("Error dropping database using the Database object: {}. Will re-try using the DBAdminClient now...", e.getMessage());
+            try {
+                dbAdminClient.dropDatabase(databaseId, null);
+            } catch (Exception ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+        return null;
+    }
+}

nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java

@@ -30,6 +30,8 @@ package io.nosqlbench.adapter.gcpspanner.types;
  * <a href="https://cloud.google.com/spanner/docs/find-approximate-nearest-neighbors#query-vector-embeddings">here</a></a>
  */
 public enum GCPSpannerOpType {
+    create_database_ddl,
+    drop_database_ddl,
     update_database_ddl,
     insert,
     execute_dml,

nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml

@@ -0,0 +1,11 @@
+scenarios:
+  default:
+    create_db_ddl: run driver=spanner tags==block:schema_db service_account_file=TEMPLATE(service_account_file)
+      project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1
+
+blocks:
+  schema_db:
+    ops:
+      op1:
+        create_database_ddl: |
+          CREATE DATABASE TEMPLATE(database_id)

nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml

@@ -0,0 +1,10 @@
+scenarios:
+  default:
+    drop_db_ddl: run driver=spanner tags==block:schema_drop_db service_account_file=TEMPLATE(service_account_file)
+      project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1
+
+blocks:
+  schema_drop_db:
+    ops:
+      op1:
+        drop_database_ddl: TEMPLATE(database_id)

nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml

@@ -1,6 +1,6 @@
 scenarios:
   default:
-    execute_ddl: run driver=spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file)
+    execute_ddl: run driver=spanner tags==block:execute_ddl service_account_file=TEMPLATE(service_account_file)
       project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1
 
 # https://cloud.google.com/spanner/docs/reference/standard-sql/data-definition-language#vector_index_option_list
@@ -9,5 +9,5 @@ blocks:
     ops:
       op1:
         update_database_ddl: |
-          CREATE VECTOR INDEX VectorsIndex ON vectors(value)
+          CREATE VECTOR INDEX IF NOT EXISTS VectorsIndex ON vectors(value)
           OPTIONS (distance_type = 'COSINE', tree_depth = 3, num_branches=1000, num_leaves = 1000000);

nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml

@@ -1,6 +1,6 @@
 scenarios:
   default:
-    execute_dml: run driver=spanner tags==blocks:execute_dml service_account_file=TEMPLATE(service_account_file)
+    execute_dml: run driver=spanner tags==block:execute_dml service_account_file=TEMPLATE(service_account_file)
       project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=TEMPLATE(cycles)
 
 bindings:

nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml

@@ -1,6 +1,6 @@
 scenarios:
   default:
-    execute_ddl: run driver=spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file)
+    execute_ddl: run driver=spanner tags==block:execute_ddl service_account_file=TEMPLATE(service_account_file)
       project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1
 
 blocks:
@@ -8,4 +8,4 @@ blocks:
     ops:
       op1:
         update_database_ddl: |
-          CREATE TABLE vectors (keycol STRING(100),value ARRAY<FLOAT32>(vector_length=>25) NOT NULL) PRIMARY KEY(keycol)
+          CREATE TABLE IF NOT EXISTS vectors (keycol STRING(100),value ARRAY<FLOAT32>(vector_length=>25) NOT NULL) PRIMARY KEY(keycol)

nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml

@@ -1,7 +1,7 @@
 scenarios:
   default:
     insert_vector: >-
-      run driver=spanner tags==blocks:insert_vector service_account_file=TEMPLATE(service_account_file)
+      run driver=spanner tags==block:insert_vector service_account_file=TEMPLATE(service_account_file)
       project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=TEMPLATE(cycles)
 
 bindings:

nb-adapters/adapter-gcp-spanner/src/main/resources/spanner.md

@@ -19,12 +19,14 @@ to the adapter at runtime.
 ## Op Templates
 
 The Google Cloud Spanner adapter supports the following operations:
+
+* `create_database_ddl` - Data Definition Language operations such as creating and dropping databases.
 * `update_database_ddl` - Data Definition Language operations such as creating and dropping tables, indexes, etc.
 * `execute_dml` - Data Manipulation Language operations. Read only operations are supported at this time, including queries
 and vector queries.
 * `insert` - Insert a single record, vector or non-vector, of data into the database.
+
 ## Examples
 
-
+Checkout the sample workload files [here](./activities).
-
 ---