Merge pull request #2038 from nosqlbench/spanner

Spanner - Handle Drop DB more elegantly

.run/spanner_rampup.run.xml

@@ -7,7 +7,7 @@
     </extension>
     <option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
     <option name="VM_PARAMETERS" value="--enable-preview" />
-    <option name="PROGRAM_PARAMETERS" value="insert_vector default.insert_vector project_id=stratperf instance_id=spanner-vector database_id=baselines1 service_account_file=./stratperf-068083077315.json cycles=1180 dataset=https://testeng-assets.s3.us-east-2.amazonaws.com/vector/testdata/vector/ANN/glove-25-angular/glove-25-angular.hdf5 --progress console:1s -v --add-labels=&quot;target:spanner,instance:vectors,vendor:gcp&quot; --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators &quot;[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]&quot; --report-interval 10 --show-stacktraces --logs-max 5" />
+    <option name="PROGRAM_PARAMETERS" value="insert_vector default.insert_vector project_id=stratperf instance_id=spanner-vector database_id=baselines12 service_account_file=./stratperf-068083077315.json cycles=1180 dataset=https://testeng-assets.s3.us-east-2.amazonaws.com/vector/testdata/vector/ANN/glove-25-angular/glove-25-angular.hdf5 --progress console:1s -v --add-labels=&quot;target:spanner,instance:vectors,vendor:gcp&quot; --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators &quot;[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]&quot; --report-interval 10 --show-stacktraces --logs-max 5" />
     <option name="WORKING_DIRECTORY" value="$ProjectFileDir$/local/google_spanner" />
     <option name="ALTERNATIVE_JRE_PATH" value="jdk21" />
     <envs>

.run/spanner_schema_database_create.run.xml

@@ -7,7 +7,7 @@
     </extension>
     <option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
     <option name="VM_PARAMETERS" value="--enable-preview" />
-    <option name="PROGRAM_PARAMETERS" value="create_database_ddl default.create_db_ddl project_id=stratperf instance_id=spanner-vector database_id=baselines1 service_account_file=./stratperf-068083077315.json --progress console:1s -v --add-labels=&quot;target:spanner,instance:vectors,vendor:gcp&quot; --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators &quot;[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]&quot; --report-interval 10 --show-stacktraces --logs-max 5" />
+    <option name="PROGRAM_PARAMETERS" value="create_database_ddl default.create_db_ddl project_id=stratperf instance_id=spanner-vector database_id=baselines12 service_account_file=./stratperf-068083077315.json --progress console:1s -v --add-labels=&quot;target:spanner,instance:vectors,vendor:gcp&quot; --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators &quot;[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]&quot; --report-interval 10 --show-stacktraces --logs-max 5" />
     <option name="WORKING_DIRECTORY" value="$ProjectFileDir$/local/google_spanner" />
     <option name="ALTERNATIVE_JRE_PATH" value="jdk21" />
     <envs>

.run/spanner_schema_database_drop.run.xml

@@ -7,7 +7,7 @@
     </extension>
     <option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
     <option name="VM_PARAMETERS" value="--enable-preview" />
-    <option name="PROGRAM_PARAMETERS" value="drop_database_ddl default.drop_db_ddl project_id=stratperf instance_id=spanner-vector database_id=baselines1 service_account_file=./stratperf-068083077315.json --progress console:1s -v --add-labels=&quot;target:spanner,instance:vectors,vendor:gcp&quot; --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators &quot;[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]&quot; --report-interval 10 --show-stacktraces --logs-max 5" />
+    <option name="PROGRAM_PARAMETERS" value="drop_database_ddl default.drop_db_ddl project_id=stratperf instance_id=spanner-vector database_id=baselines12 service_account_file=./stratperf-068083077315.json --progress console:1s -v --add-labels=&quot;target:spanner,instance:vectors,vendor:gcp&quot; --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators &quot;[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]&quot; --report-interval 10 --show-stacktraces --logs-max 5" />
     <option name="WORKING_DIRECTORY" value="$ProjectFileDir$/local/google_spanner" />
     <option name="ALTERNATIVE_JRE_PATH" value="jdk21" />
     <envs>

mvn-defaults/pom.xml

@@ -418,7 +418,7 @@
             <dependency>
                 <groupId>com.google.cloud</groupId>
                 <artifactId>google-cloud-spanner</artifactId>
-                <version>6.76.0</version>
+                <version>6.77.0</version>
             </dependency>
 
         </dependencies>
@@ -778,7 +778,7 @@
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-compiler-plugin</artifactId>
-                    <version>3.10.1</version>
+                    <version>3.13.0</version>
                 </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>

nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerDropDatabaseDdlOpDispenser.java

@@ -17,7 +17,10 @@
 package io.nosqlbench.adapter.gcpspanner.opdispensers;
 
 
+import com.google.cloud.spanner.DatabaseAdminClient;
+import com.google.cloud.spanner.Options;
 import io.nosqlbench.adapter.gcpspanner.GCPSpannerDriverAdapter;
+import io.nosqlbench.adapter.gcpspanner.GCPSpannerSpace;
 import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp;
 import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerDropDatabaseDdlOp;
 import io.nosqlbench.adapters.api.templating.ParsedOp;
@@ -45,7 +48,7 @@ public class GCPSpannerDropDatabaseDdlOpDispenser extends GCPSpannerBaseOpDispen
      */
     public GCPSpannerDropDatabaseDdlOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction<String> targetFunction) {
         super(adapter, op, targetFunction);
-        this.opFunction = DropOpFunction(op);
+        this.opFunction = dropOpFunction(op);
     }
 
     /**
@@ -54,15 +57,28 @@ public class GCPSpannerDropDatabaseDdlOpDispenser extends GCPSpannerBaseOpDispen
      * @param op the {@link ParsedOp} instance
      * @return a {@link LongFunction} that generates {@link GCPSpannerDropDatabaseDdlOp} instances
      */
-    private LongFunction<GCPSpannerDropDatabaseDdlOp> DropOpFunction(ParsedOp op) {
+    private LongFunction<GCPSpannerDropDatabaseDdlOp> dropOpFunction(ParsedOp op) {
-        return (l) -> new GCPSpannerDropDatabaseDdlOp(
+        return l -> {
-            spaceFunction.apply(l).getSpanner(),
+            GCPSpannerSpace space = spaceFunction.apply(l);
+            if (space.getSpanner() != null && space.getDbAdminClient() != null && space.getDbClient() != null) {
+                String dbId = targetFunction.apply(l);
+                DatabaseAdminClient dbAdminClient = space.getDbAdminClient();
+
+                if (dbAdminClient.listDatabases(spaceFunction.apply(l).getInstanceId(), Options.filter(null))
+                    .streamValues().anyMatch(d -> d.getId().getDatabase().equals(dbId))) {
+                    return new GCPSpannerDropDatabaseDdlOp(
+                        space.getSpanner(),
                         l,
-            targetFunction.apply(l),
+                        dbId,
-            spaceFunction.apply(l).getDbAdminClient(),
+                        space.getDbAdminClient(),
-            spaceFunction.apply(l).getDbAdminClient().getDatabase(spaceFunction.apply(l).getInstanceId(), spaceFunction.apply(l).getDatabaseIdString())
+                        space.getDbAdminClient().getDatabase(space.getInstanceId(), space.getDatabaseIdString())
                     );
                 }
+            }
+            logger.info("Database to be deleted not found [{}]", space.getDatabaseId().getName());
+            return null;
+        };
+    }
 
     /**
      * Retrieves an operation instance based on the provided value.
@@ -72,6 +88,6 @@ public class GCPSpannerDropDatabaseDdlOpDispenser extends GCPSpannerBaseOpDispen
      */
     @Override
     public GCPSpannerBaseOp<?> getOp(long value) {
-        return opFunction.apply(value);
+        return opFunction != null ? opFunction.apply(value) : null;
     }
 }

nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerDropDatabaseDdlOp.java

@@ -18,6 +18,7 @@ package io.nosqlbench.adapter.gcpspanner.ops;
 
 import com.google.cloud.spanner.Database;
 import com.google.cloud.spanner.DatabaseAdminClient;
+import com.google.cloud.spanner.DatabaseNotFoundException;
 import com.google.cloud.spanner.Spanner;
 
 /**
@@ -56,11 +57,17 @@ public class GCPSpannerDropDatabaseDdlOp extends GCPSpannerBaseOp<Long> {
     @Override
     public Object applyOp(long value) {
         try {
+            if (null != db && db.exists()) {
                 db.drop();
+            }
         } catch (Exception e) {
             logger.warn("Error dropping database using the Database object: {}. Will re-try using the DBAdminClient now...", e.getMessage());
             try {
+                if (null != dbAdminClient) {
                     dbAdminClient.dropDatabase(databaseId, null);
+                }
+            } catch (DatabaseNotFoundException noDB) {
+                logger.info("Database does not exist. {}", noDB.getMessage());
             } catch (Exception ex) {
                 throw new RuntimeException(ex);
             }

nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpDef.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022-2023 nosqlbench
+ * Copyright (c) 2020-2024 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -105,7 +105,6 @@ public class OpDef extends OpTemplate {
         return tagsWithName;
     }
 
-    @Override
     public String toString() {
         return "stmt(name:" + getName() + ", stmt:" + getOp() + ", tags:(" + getTags() + "), params:(" + getParams() + "), bindings:(" + getBindings() + "))";
     }