IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into dependabot/maven/mvn-defaults/commons-io-commons-io-2.14.0

Browse Source
This commit is contained in:
Jonathan Shook 2024-10-03 17:31:30 -05:00 committed by GitHub
commit a1a65f83ea
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 355 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.run/nb --list-drivers.run.xml
View File

@ -5,10 +5,10 @@
<option name="region" />
<option name="useCurrentConnection" value="false" />
</extension>
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5-5.21.1-SNAPSHOT-jar-with-dependencies.jar" />
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
<option name="PROGRAM_PARAMETERS" value="--list-drivers --show-stacktraces" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/local/" />
<option name="ALTERNATIVE_JRE_PATH" />
<method v="2" />
</configuration>
</component>
</component>

4
.run/nb --list-scenarios.run.xml
View File

@ -5,10 +5,10 @@
<option name="region" />
<option name="useCurrentConnection" value="false" />
</extension>
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5-5.21.1-SNAPSHOT-jar-with-dependencies.jar" />
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
<option name="PROGRAM_PARAMETERS" value="--list-scenarios --show-stacktraces" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/local/" />
<option name="ALTERNATIVE_JRE_PATH" />
<method v="2" />
</configuration>
</component>
</component>

18
.run/spanner_schema_database_create.run.xml Normal file
View File

@ -0,0 +1,18 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="spanner_schema_database_create" type="JarApplication" folderName="Google Spanner">
<extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
<option name="credential" />
<option name="region" />
<option name="useCurrentConnection" value="false" />
</extension>
<option name="JAR_PATH" value="$PROJECT_DIR$/nb5/target/nb5.jar" />
<option name="VM_PARAMETERS" value="--enable-preview" />
<option name="PROGRAM_PARAMETERS" value="create_database_ddl default.create_db_ddl project_id=stratperf instance_id=perftest-spanner database_id=baselines1 service_account_file=./stratperf-068083077315.json --progress console:1s -vv --add-labels=&quot;target:spanner,instance:vectors,vendor:gcp&quot; --report-prompush-to https://vector-perf.feat.apps.paas.datastax.com:8427/api/v1/import/prometheus/metrics/job/nosqlbench/instance/vectors --annotators &quot;[{'type':'log','level':'info'},{'type':'grafana','baseurl':'https://vector-perf.feat.apps.paas.datastax.com/'}]&quot; --report-interval 10 --show-stacktraces --logs-max 5" />
<option name="WORKING_DIRECTORY" value="$ProjectFileDir$/local/google_spanner" />
<option name="ALTERNATIVE_JRE_PATH" value="jdk21" />
<envs>
<env name="GOOGLE_APPLICATION_CREDENTIALS" value="$ProjectFileDir$/local/google_spanner/stratperf-068083077315.json" />
</envs>
<method v="2" />
</configuration>
</component>

4
nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/GCPSpannerOpMapper.java
View File

@ -55,6 +55,10 @@ public class GCPSpannerOpMapper implements OpMapper<GCPSpannerBaseOp<?>> {
logger.info(() -> "Using '" + typeAndTarget.enumId + "' op type for op template '" + op.getName() + "'");
return switch (typeAndTarget.enumId) {
case drop_database_ddl ->
new GCPSpannerDropDatabaseDdlOpDispenser(adapter, op, typeAndTarget.targetFunction);
case create_database_ddl ->
new GCPSpannerCreateDatabaseDdlOpDispenser(adapter, op, typeAndTarget.targetFunction);
case update_database_ddl ->
new GCPSpannerUpdateDatabaseDdlOpDispenser(adapter, op, typeAndTarget.targetFunction);
case insert ->

78
nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerCreateDatabaseDdlOpDispenser.java Normal file
View File

@ -0,0 +1,78 @@
/*
* Copyright (c) 2020-2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.gcpspanner.opdispensers;
import io.nosqlbench.adapter.gcpspanner.GCPSpannerDriverAdapter;
import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp;
import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerCreateDatabaseDdlOp;
import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerUpdateDatabaseDdlOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.function.LongFunction;
/**
* Dispenser class for creating databases of {@link GCPSpannerCreateDatabaseDdlOp}.
*
* @see <a href="https://cloud.google.com/spanner/docs/reference/rpc/google.spanner.admin.database.v1#createdatabaserequest">
* CreateDatabaseRequest</a> which can be a stretch goal to combine all of DB, Table(s), and Indexes into one-single call.
*/
public class GCPSpannerCreateDatabaseDdlOpDispenser extends GCPSpannerBaseOpDispenser {
private static final Logger logger = LogManager.getLogger(GCPSpannerCreateDatabaseDdlOpDispenser.class);
private final LongFunction<GCPSpannerCreateDatabaseDdlOp> opFunction;
/**
* Constructor for {@link GCPSpannerCreateDatabaseDdlOpDispenser}.
*
* @param adapter the {@link GCPSpannerDriverAdapter} instance
* @param op the {@link ParsedOp} instance
* @param targetFunction a {@link LongFunction} that provides the target string
*/
public GCPSpannerCreateDatabaseDdlOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction<String> targetFunction) {
super(adapter, op, targetFunction);
this.opFunction = createOpFunction(op);
}
/**
* Creates a {@link LongFunction} that generates {@link GCPSpannerUpdateDatabaseDdlOp} instances.
*
* @param op the {@link ParsedOp} instance
* @return a {@link LongFunction} that generates {@link GCPSpannerUpdateDatabaseDdlOp} instances
*/
private LongFunction<GCPSpannerCreateDatabaseDdlOp> createOpFunction(ParsedOp op) {
return (l) -> new GCPSpannerCreateDatabaseDdlOp(
spaceFunction.apply(l).getSpanner(),
l,
targetFunction.apply(l),
spaceFunction.apply(l).getDbAdminClient(),
spaceFunction.apply(l).getInstanceId()
);
}
/**
* Retrieves an operation instance based on the provided value.
*
* @param value the long value used to generate the operation
* @return a {@link GCPSpannerBaseOp} instance
*/
@Override
public GCPSpannerBaseOp<?> getOp(long value) {
return opFunction.apply(value);
}
}

77
nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/opdispensers/GCPSpannerDropDatabaseDdlOpDispenser.java Normal file
View File

@ -0,0 +1,77 @@
/*
* Copyright (c) 2020-2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.gcpspanner.opdispensers;
import io.nosqlbench.adapter.gcpspanner.GCPSpannerDriverAdapter;
import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerBaseOp;
import io.nosqlbench.adapter.gcpspanner.ops.GCPSpannerDropDatabaseDdlOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.function.LongFunction;
/**
* Dispenser class for creating databases of {@link GCPSpannerDropDatabaseDdlOp}.
*
* @see <a href="https://cloud.google.com/spanner/docs/reference/rpc/google.spanner.admin.database.v1#dropdatabaserequest">
* DropDatabaseRequest</a> which can be a stretch goal to combine all of DB, Table(s), and Indexes into one-single call.
*/
public class GCPSpannerDropDatabaseDdlOpDispenser extends GCPSpannerBaseOpDispenser {
private static final Logger logger = LogManager.getLogger(GCPSpannerDropDatabaseDdlOpDispenser.class);
private final LongFunction<GCPSpannerDropDatabaseDdlOp> opFunction;
/**
* Constructor for {@link GCPSpannerDropDatabaseDdlOpDispenser}.
*
* @param adapter the {@link GCPSpannerDriverAdapter} instance
* @param op the {@link ParsedOp} instance
* @param targetFunction a {@link LongFunction} that provides the target string
*/
public GCPSpannerDropDatabaseDdlOpDispenser(GCPSpannerDriverAdapter adapter, ParsedOp op, LongFunction<String> targetFunction) {
super(adapter, op, targetFunction);
this.opFunction = DropOpFunction(op);
}
/**
* Drops a {@link LongFunction} that generates {@link GCPSpannerDropDatabaseDdlOp} instances.
*
* @param op the {@link ParsedOp} instance
* @return a {@link LongFunction} that generates {@link GCPSpannerDropDatabaseDdlOp} instances
*/
private LongFunction<GCPSpannerDropDatabaseDdlOp> DropOpFunction(ParsedOp op) {
return (l) -> new GCPSpannerDropDatabaseDdlOp(
spaceFunction.apply(l).getSpanner(),
l,
targetFunction.apply(l),
spaceFunction.apply(l).getDbAdminClient(),
spaceFunction.apply(l).getDbAdminClient().getDatabase(spaceFunction.apply(l).getInstanceId(), spaceFunction.apply(l).getDatabaseIdString())
);
}
/**
* Retrieves an operation instance based on the provided value.
*
* @param value the long value used to generate the operation
* @return a {@link GCPSpannerBaseOp} instance
*/
@Override
public GCPSpannerBaseOp<?> getOp(long value) {
return opFunction.apply(value);
}
}

71
nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerCreateDatabaseDdlOp.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Copyright (c) 2020-2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.gcpspanner.ops;
import com.google.api.gax.longrunning.OperationFuture;
import com.google.cloud.spanner.Database;
import com.google.cloud.spanner.DatabaseAdminClient;
import com.google.cloud.spanner.Spanner;
import com.google.spanner.admin.database.v1.CreateDatabaseMetadata;
/**
* This class represents an operation to create the database DDL (Data Definition Language) in Google Cloud Spanner.
* It extends the {@link GCPSpannerBaseOp} class and provides the implementation for applying the DDL update operation.
*/
public class GCPSpannerCreateDatabaseDdlOp extends GCPSpannerBaseOp<Long> {
private final String createDatbaseStatement;
private final DatabaseAdminClient dbAdminClient;
private final String instanceId;
/**
* Constructs a new {@link GCPSpannerUpdateDatabaseDdlOp}.
*
* @param searchIndexClient the {@link Spanner} client
* @param requestParam the request parameter
* @param createDatbaseStatement the SQL statement to create the table
* @param dbAdminClient the {@link DatabaseAdminClient} to execute the DDL update
* @param instanceId the instance ID string representing the target spanner instance
*/
public GCPSpannerCreateDatabaseDdlOp(Spanner searchIndexClient, Long requestParam, String createDatbaseStatement,
DatabaseAdminClient dbAdminClient, String instanceId) {
super(searchIndexClient, requestParam);
this.createDatbaseStatement = createDatbaseStatement;
this.dbAdminClient = dbAdminClient;
this.instanceId = instanceId;
}
/**
* Applies the DDL create operation.
*
* @param value the value to be used in the operation
* @return the result of the operation
* @throws RuntimeException if an error occurs during the operation
*/
@Override
public Object applyOp(long value) {
OperationFuture<Database, CreateDatabaseMetadata> operation = dbAdminClient.createDatabase(
instanceId,
createDatbaseStatement,
null,
null);
try {
return operation.get();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}

70
nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/ops/GCPSpannerDropDatabaseDdlOp.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Copyright (c) 2020-2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.gcpspanner.ops;
import com.google.cloud.spanner.Database;
import com.google.cloud.spanner.DatabaseAdminClient;
import com.google.cloud.spanner.Spanner;
/**
* This class represents an operation to Drop the database DDL (Data Definition Language) in Google Cloud Spanner.
* It extends the {@link GCPSpannerBaseOp} class and provides the implementation for applying the DDL update operation.
*/
public class GCPSpannerDropDatabaseDdlOp extends GCPSpannerBaseOp<Long> {
private final String databaseId;
private final DatabaseAdminClient dbAdminClient;
private final Database db;
/**
* Constructs a new {@link GCPSpannerUpdateDatabaseDdlOp}.
*
* @param searchIndexClient the {@link Spanner} client
* @param requestParam the request parameter
* @param databaseId the database ID to be dropped
* @param dbAdminClient the {@link DatabaseAdminClient} to execute the DDL update
* @param db the {@link Database} to be dropped
*/
public GCPSpannerDropDatabaseDdlOp(Spanner searchIndexClient, Long requestParam, String databaseId,
DatabaseAdminClient dbAdminClient, Database db) {
super(searchIndexClient, requestParam);
this.databaseId = databaseId;
this.dbAdminClient = dbAdminClient;
this.db = db;
}
/**
* Applies the DDL drop operation.
*
* @param value the value to be used in the operation
* @return the result of the operation
* @throws RuntimeException if an error occurs during the operation
*/
@Override
public Object applyOp(long value) {
try {
db.drop();
} catch (Exception e) {
logger.warn("Error dropping database using the Database object: {}. Will re-try using the DBAdminClient now...", e.getMessage());
try {
dbAdminClient.dropDatabase(databaseId, null);
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
return null;
}
}

2
nb-adapters/adapter-gcp-spanner/src/main/java/io/nosqlbench/adapter/gcpspanner/types/GCPSpannerOpType.java
View File

@ -30,6 +30,8 @@ package io.nosqlbench.adapter.gcpspanner.types;
* <a href="https://cloud.google.com/spanner/docs/find-approximate-nearest-neighbors#query-vector-embeddings">here</a></a>
*/
public enum GCPSpannerOpType {
create_database_ddl,
drop_database_ddl,
update_database_ddl,
insert,
execute_dml,

11
nb-adapters/adapter-gcp-spanner/src/main/resources/activities/create_database_ddl.yaml Normal file
View File

@ -0,0 +1,11 @@
scenarios:
default:
create_db_ddl: run driver=spanner tags==block:schema_db service_account_file=TEMPLATE(service_account_file)
project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1
blocks:
schema_db:
ops:
op1:
create_database_ddl: |
CREATE DATABASE TEMPLATE(database_id)

10
nb-adapters/adapter-gcp-spanner/src/main/resources/activities/drop_database_ddl.yaml Normal file
View File

@ -0,0 +1,10 @@
scenarios:
default:
drop_db_ddl: run driver=spanner tags==block:schema_drop_db service_account_file=TEMPLATE(service_account_file)
project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1
blocks:
schema_drop_db:
ops:
op1:
drop_database_ddl: TEMPLATE(database_id)

4
nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_index_ddl.yaml
View File

@ -1,6 +1,6 @@
scenarios:
default:
execute_ddl: run driver=spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file)
execute_ddl: run driver=spanner tags==block:execute_ddl service_account_file=TEMPLATE(service_account_file)
project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1
# https://cloud.google.com/spanner/docs/reference/standard-sql/data-definition-language#vector_index_option_list
@ -9,5 +9,5 @@ blocks:
ops:
op1:
update_database_ddl: |
CREATE VECTOR INDEX VectorsIndex ON vectors(value)
CREATE VECTOR INDEX IF NOT EXISTS VectorsIndex ON vectors(value)
OPTIONS (distance_type = 'COSINE', tree_depth = 3, num_branches=1000, num_leaves = 1000000);

2
nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_queryvector_dml.yaml
View File

@ -1,6 +1,6 @@
scenarios:
default:
execute_dml: run driver=spanner tags==blocks:execute_dml service_account_file=TEMPLATE(service_account_file)
execute_dml: run driver=spanner tags==block:execute_dml service_account_file=TEMPLATE(service_account_file)
project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=TEMPLATE(cycles)
bindings:

4
nb-adapters/adapter-gcp-spanner/src/main/resources/activities/execute_table_ddl.yaml
View File

@ -1,6 +1,6 @@
scenarios:
default:
execute_ddl: run driver=spanner tags==blocks:execute_ddl service_account_file=TEMPLATE(service_account_file)
execute_ddl: run driver=spanner tags==block:execute_ddl service_account_file=TEMPLATE(service_account_file)
project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=1
blocks:
@ -8,4 +8,4 @@ blocks:
ops:
op1:
update_database_ddl: |
CREATE TABLE vectors (keycol STRING(100),value ARRAY<FLOAT32>(vector_length=>25) NOT NULL) PRIMARY KEY(keycol)
CREATE TABLE IF NOT EXISTS vectors (keycol STRING(100),value ARRAY<FLOAT32>(vector_length=>25) NOT NULL) PRIMARY KEY(keycol)

2
nb-adapters/adapter-gcp-spanner/src/main/resources/activities/insert_vector.yaml
View File

@ -1,7 +1,7 @@
scenarios:
default:
insert_vector: >-
run driver=spanner tags==blocks:insert_vector service_account_file=TEMPLATE(service_account_file)
run driver=spanner tags==block:insert_vector service_account_file=TEMPLATE(service_account_file)
project_id=TEMPLATE(project_id) instance_id=TEMPLATE(instance_id) database_id=TEMPLATE(database_id) cycles=TEMPLATE(cycles)
bindings:

6
nb-adapters/adapter-gcp-spanner/src/main/resources/spanner.md
View File

@ -19,12 +19,14 @@ to the adapter at runtime.
## Op Templates
The Google Cloud Spanner adapter supports the following operations:
* `create_database_ddl` - Data Definition Language operations such as creating and dropping databases.
* `update_database_ddl` - Data Definition Language operations such as creating and dropping tables, indexes, etc.
* `execute_dml` - Data Manipulation Language operations. Read only operations are supported at this time, including queries
and vector queries.
* `insert` - Insert a single record, vector or non-vector, of data into the database.
## Examples
Checkout the sample workload files [here](./activities).
---