Merge pull request #2038 from nosqlbench/spanner

Spanner - Handle Drop DB more elegantly
2025-02-25 18:55:28 -06:00 · 2024-10-09 14:27:41 -04:00 · 2024-10-09 14:27:41 -04:00 · 930daed65f
commit 930daed65f
parent 941b623f63 18e432cffd
7 changed files with 43 additions and 21 deletions
--- a/.run/spanner_rampup.run.xml
+++ b/.run/spanner_rampup.run.xml
@ -7,7 +7,7 @@
    </extension>
    <option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
    <option name="VM_PARAMETERS" value="--enable-preview" />
-    <option name="PROGRAM_PARAMETERS" value="insert_vector default.insert_vector project_id=stratperf instance_id=spanner-vector database_id=baselines1 service_account_file=./stratperf-068083077315.json cycles=1180 dataset=https://testeng-assets.s3.us-east-2.amazonaws.com/vector/testdata/vector/ANN/glove-25-angular/glove-25-angular.hdf5 --progress console:1s -v --add-labels=&quot;target:spanner,instance:vectors,vendor:gcp&quot; --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators &quot;[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]&quot; --report-interval 10 --show-stacktraces --logs-max 5" />
+    <option name="PROGRAM_PARAMETERS" value="insert_vector default.insert_vector project_id=stratperf instance_id=spanner-vector database_id=baselines12 service_account_file=./stratperf-068083077315.json cycles=1180 dataset=https://testeng-assets.s3.us-east-2.amazonaws.com/vector/testdata/vector/ANN/glove-25-angular/glove-25-angular.hdf5 --progress console:1s -v --add-labels=&quot;target:spanner,instance:vectors,vendor:gcp&quot; --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators &quot;[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]&quot; --report-interval 10 --show-stacktraces --logs-max 5" />
    <option name="WORKING_DIRECTORY" value="$ProjectFileDir$/local/google_spanner" />
    <option name="ALTERNATIVE_JRE_PATH" value="jdk21" />
    <envs>
--- a/.run/spanner_schema_database_create.run.xml
+++ b/.run/spanner_schema_database_create.run.xml
@ -7,7 +7,7 @@
    </extension>
    <option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
    <option name="VM_PARAMETERS" value="--enable-preview" />
-    <option name="PROGRAM_PARAMETERS" value="create_database_ddl default.create_db_ddl project_id=stratperf instance_id=spanner-vector database_id=baselines1 service_account_file=./stratperf-068083077315.json --progress console:1s -v --add-labels=&quot;target:spanner,instance:vectors,vendor:gcp&quot; --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators &quot;[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]&quot; --report-interval 10 --show-stacktraces --logs-max 5" />
+    <option name="PROGRAM_PARAMETERS" value="create_database_ddl default.create_db_ddl project_id=stratperf instance_id=spanner-vector database_id=baselines12 service_account_file=./stratperf-068083077315.json --progress console:1s -v --add-labels=&quot;target:spanner,instance:vectors,vendor:gcp&quot; --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators &quot;[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]&quot; --report-interval 10 --show-stacktraces --logs-max 5" />
    <option name="WORKING_DIRECTORY" value="$ProjectFileDir$/local/google_spanner" />
    <option name="ALTERNATIVE_JRE_PATH" value="jdk21" />
    <envs>
@ -15,4 +15,4 @@
    </envs>
    <method v="2" />
  </configuration>
-</component>
+</component>
--- a/.run/spanner_schema_database_drop.run.xml
+++ b/.run/spanner_schema_database_drop.run.xml
@ -7,7 +7,7 @@
    </extension>
    <option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
    <option name="VM_PARAMETERS" value="--enable-preview" />
-    <option name="PROGRAM_PARAMETERS" value="drop_database_ddl default.drop_db_ddl project_id=stratperf instance_id=spanner-vector database_id=baselines1 service_account_file=./stratperf-068083077315.json --progress console:1s -v --add-labels=&quot;target:spanner,instance:vectors,vendor:gcp&quot; --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators &quot;[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]&quot; --report-interval 10 --show-stacktraces --logs-max 5" />
+    <option name="PROGRAM_PARAMETERS" value="drop_database_ddl default.drop_db_ddl project_id=stratperf instance_id=spanner-vector database_id=baselines12 service_account_file=./stratperf-068083077315.json --progress console:1s -v --add-labels=&quot;target:spanner,instance:vectors,vendor:gcp&quot; --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators &quot;[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]&quot; --report-interval 10 --show-stacktraces --logs-max 5" />
    <option name="WORKING_DIRECTORY" value="$ProjectFileDir$/local/google_spanner" />
    <option name="ALTERNATIVE_JRE_PATH" value="jdk21" />
    <envs>
@ -15,4 +15,4 @@
    </envs>
    <method v="2" />
  </configuration>
-</component>
+</component>
--- a/mvn-defaults/pom.xml
+++ b/mvn-defaults/pom.xml
@ -418,7 +418,7 @@
            <dependency>
                <groupId>com.google.cloud</groupId>
                <artifactId>google-cloud-spanner</artifactId>
-                <version>6.76.0</version>
+                <version>6.77.0</version>
            </dependency>

        </dependencies>
@ -778,7 +778,7 @@
                <plugin>
                    <groupId>org.apache.maven.plugins</groupId>
                    <artifactId>maven-compiler-plugin</artifactId>
-                    <version>3.10.1</version>
+                    <version>3.13.0</version>
                </plugin>
                <plugin>
                    <groupId>org.apache.maven.plugins</groupId>
--- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerDropDatabaseDdlOpDispenser.java
+++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerDropDatabaseDdlOpDispenser.java
@ -17,7 +17,10 @@
 package io.nosqlbench.adapter.gcpspanner.opdispensers;


+import com.google.cloud.spanner.DatabaseAdminClient;
+import com.google.cloud.spanner.Options;
 import io.nosqlbench.adapter.gcpspanner.GCPSpannerDriverAdapter;
+import io.nosqlbench.adapter.gcpspanner.GCPSpannerSpace;
 import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp;
 import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerDropDatabaseDdlOp;
 import io.nosqlbench.adapters.api.templating.ParsedOp;
@ -45,7 +48,7 @@ public class GCPSpannerDropDatabaseDdlOpDispenser extends GCPSpannerBaseOpDispen
     */
    public GCPSpannerDropDatabaseDdlOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction<String> targetFunction) {
        super(adapter, op, targetFunction);
-        this.opFunction = DropOpFunction(op);
+        this.opFunction = dropOpFunction(op);
    }

    /**
@ -54,14 +57,27 @@ public class GCPSpannerDropDatabaseDdlOpDispenser extends GCPSpannerBaseOpDispen
     * @param op the {@link ParsedOp} instance
     * @return a {@link LongFunction} that generates {@link GCPSpannerDropDatabaseDdlOp} instances
     */
-    private LongFunction<GCPSpannerDropDatabaseDdlOp> DropOpFunction(ParsedOp op) {
-        return (l) -> new GCPSpannerDropDatabaseDdlOp(
-            spaceFunction.apply(l).getSpanner(),
-            l,
-            targetFunction.apply(l),
-            spaceFunction.apply(l).getDbAdminClient(),
-            spaceFunction.apply(l).getDbAdminClient().getDatabase(spaceFunction.apply(l).getInstanceId(), spaceFunction.apply(l).getDatabaseIdString())
-        );
+    private LongFunction<GCPSpannerDropDatabaseDdlOp> dropOpFunction(ParsedOp op) {
+        return l -> {
+            GCPSpannerSpace space = spaceFunction.apply(l);
+            if (space.getSpanner() != null && space.getDbAdminClient() != null && space.getDbClient() != null) {
+                String dbId = targetFunction.apply(l);
+                DatabaseAdminClient dbAdminClient = space.getDbAdminClient();
+
+                if (dbAdminClient.listDatabases(spaceFunction.apply(l).getInstanceId(), Options.filter(null))
+                    .streamValues().anyMatch(d -> d.getId().getDatabase().equals(dbId))) {
+                    return new GCPSpannerDropDatabaseDdlOp(
+                        space.getSpanner(),
+                        l,
+                        dbId,
+                        space.getDbAdminClient(),
+                        space.getDbAdminClient().getDatabase(space.getInstanceId(), space.getDatabaseIdString())
+                    );
+                }
+            }
+            logger.info("Database to be deleted not found [{}]", space.getDatabaseId().getName());
+            return null;
+        };
    }

    /**
@ -72,6 +88,6 @@ public class GCPSpannerDropDatabaseDdlOpDispenser extends GCPSpannerBaseOpDispen
     */
    @Override
    public GCPSpannerBaseOp<?> getOp(long value) {
-        return opFunction.apply(value);
+        return opFunction != null ? opFunction.apply(value) : null;
    }
 }
--- a/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerDropDatabaseDdlOp.java
+++ b/nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerDropDatabaseDdlOp.java
@ -18,6 +18,7 @@ package io.nosqlbench.adapter.gcpspanner.ops;

 import com.google.cloud.spanner.Database;
 import com.google.cloud.spanner.DatabaseAdminClient;
+import com.google.cloud.spanner.DatabaseNotFoundException;
 import com.google.cloud.spanner.Spanner;

 /**
@ -56,11 +57,17 @@ public class GCPSpannerDropDatabaseDdlOp extends GCPSpannerBaseOp<Long> {
    @Override
    public Object applyOp(long value) {
        try {
-            db.drop();
+            if (null != db && db.exists()) {
+                db.drop();
+            }
        } catch (Exception e) {
            logger.warn("Error dropping database using the Database object: {}. Will re-try using the DBAdminClient now...", e.getMessage());
            try {
-                dbAdminClient.dropDatabase(databaseId, null);
+                if (null != dbAdminClient) {
+                    dbAdminClient.dropDatabase(databaseId, null);
+                }
+            } catch (DatabaseNotFoundException noDB) {
+                logger.info("Database does not exist. {}", noDB.getMessage());
            } catch (Exception ex) {
                throw new RuntimeException(ex);
            }
--- a/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpDef.java
+++ b/nb-apis/adapters-api/src/main/java/io/nosqlbench/adapters/api/activityconfig/yaml/OpDef.java
@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022-2023 nosqlbench
+ * Copyright (c) 2020-2024 nosqlbench
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@ -105,7 +105,6 @@ public class OpDef extends OpTemplate {
        return tagsWithName;
    }

-    @Override
    public String toString() {
        return "stmt(name:" + getName() + ", stmt:" + getOp() + ", tags:(" + getTags() + "), params:(" + getParams() + "), bindings:(" + getBindings() + "))";
    }