IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2024-12-22 15:13:41 -06:00
Code Issues Packages Projects Releases Wiki Activity

improve op types

Browse Source
This commit is contained in:
Jonathan Shook 2024-02-07 16:03:36 -06:00
parent 25630e01c0
commit 55f27dfb05
9 changed files with 99 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/OpenSearchOpMapper.java
View File

@ -39,11 +39,11 @@ public class OpenSearchOpMapper implements OpMapper<Op> {
TypeAndTarget<OpenSearchOpTypes, String> typeAndTarget =
op.getTypeAndTarget(OpenSearchOpTypes.class, String.class, "verb", "index");
return switch (typeAndTarget.enumId) {
case create_index -> new CreateIndexOpDispenser(adapter, op);
case delete_index -> new DeleteIndexOpDispenser(adapter, op);
case index -> new IndexOpDispenser(adapter,op);
case update -> new UpdateOpDispenser(adapter,op);
case delete -> new DeleteOpDispenser(adapter,op);
case create_index -> new CreateIndexOpDispenser(adapter, op, typeAndTarget.targetFunction);
case delete_index -> new DeleteIndexOpDispenser(adapter, op, typeAndTarget.targetFunction);
case index -> new IndexOpDispenser(adapter,op, typeAndTarget.targetFunction);
case update -> new UpdateOpDispenser(adapter,op, typeAndTarget.targetFunction);
case delete -> new DeleteOpDispenser(adapter,op, typeAndTarget.targetFunction);
default -> throw new RuntimeException("Unrecognized op type '" + typeAndTarget.enumId.name() + "' while " +
"mapping parsed op " + op);
};

59
adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/CreateIndexOpDispenser.java
View File

@ -29,37 +29,19 @@ import java.util.function.LongFunction;
public class CreateIndexOpDispenser extends BaseOpenSearchOpDispenser {
public CreateIndexOpDispenser(OpenSearchAdapter adapter, ParsedOp op) {
private final LongFunction<String> targetF;
public CreateIndexOpDispenser(OpenSearchAdapter adapter, ParsedOp op, LongFunction<String> targetF) {
super(adapter, op);
this.targetF = targetF;
}
/**
* {@see
* <a href="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-vector-search.html">doc
* </a>}
* <pre>{@code
* {
* "mappings": {
* "properties": {
* "value": {
* "type": "dense_vector",
* "dims": TEMPLATE(dimensions, 25),
* "index": true,
* "similarity": "TEMPLATE(similarity_function, cosine)"
* },
* "key": {
* "type": "text"
* }
* }
* }
* }}</pre>
*
* @return
*/
@Override
public LongFunction<CreateIndexOp> createOpFunc(LongFunction<OpenSearchClient> clientF, ParsedOp op) {
CreateIndexRequest.Builder eb = new CreateIndexRequest.Builder();
LongFunction<CreateIndexRequest.Builder> bfunc = l -> new CreateIndexRequest.Builder().index("testindex1");
LongFunction<CreateIndexRequest.Builder> bfunc =
l -> new CreateIndexRequest.Builder()
.index(targetF.apply(1));
bfunc = op.enhanceFunc(bfunc, "mappings", Map.class, this::resolveTypeMapping);
LongFunction<CreateIndexRequest.Builder> finalBfunc = bfunc;
@ -70,18 +52,23 @@ public class CreateIndexOpDispenser extends BaseOpenSearchOpDispenser {
private CreateIndexRequest.Builder resolveTypeMapping(CreateIndexRequest.Builder eb, Map<?, ?> mappings) {
TypeMapping.Builder builder = new TypeMapping.Builder().properties(
Map.of(
"p1",
new Property.Builder().knnVector(new KnnVectorProperty.Builder()
.dimension(23)
.method(
new KnnVectorMethod.Builder()
.name("hnsw")
.engine("faiss")
.spaceType("l2")
.parameters(Map.of("ef_construction", JsonData.of(256),"m",JsonData.of(8)))
.build()
"key",
new Property.Builder()
.text(b -> b)
.build(),
"value",
new Property.Builder()
.knnVector(new KnnVectorProperty.Builder()
.dimension(23)
.method(
new KnnVectorMethod.Builder()
.name("hnsw")
.engine("faiss")
.spaceType("l2")
.parameters(Map.of("ef_construction", JsonData.of(256), "m", JsonData.of(8)))
.build()
).build()
).build()
).build()
))
.fieldNames(new FieldNamesField.Builder()
.enabled(true).build()

72
adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/DeleteIndexOpDispenser.java
View File

@ -17,78 +17,28 @@
package io.nosqlbench.adapter.opensearch.dispensers;
import io.nosqlbench.adapter.opensearch.OpenSearchAdapter;
import io.nosqlbench.adapter.opensearch.ops.CreateIndexOp;
import io.nosqlbench.adapter.opensearch.ops.DeleteIndexOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import org.opensearch.client.json.JsonData;
import org.opensearch.client.opensearch.OpenSearchClient;
import org.opensearch.client.opensearch._types.mapping.*;
import org.opensearch.client.opensearch.indices.CreateIndexRequest;
import org.opensearch.client.opensearch.indices.DeleteIndexRequest;
import java.util.Map;
import java.util.function.LongFunction;
public class DeleteIndexOpDispenser extends BaseOpenSearchOpDispenser {
public DeleteIndexOpDispenser(OpenSearchAdapter adapter, ParsedOp op) {
private final LongFunction<String> targetF;
public DeleteIndexOpDispenser(OpenSearchAdapter adapter, ParsedOp op, LongFunction<String> targetF) {
super(adapter, op);
this.targetF = targetF;
}
/**
* {@see
* <a href="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-vector-search.html">doc
* </a>}
* <pre>{@code
* {
* "mappings": {
* "properties": {
* "value": {
* "type": "dense_vector",
* "dims": TEMPLATE(dimensions, 25),
* "index": true,
* "similarity": "TEMPLATE(similarity_function, cosine)"
* },
* "key": {
* "type": "text"
* }
* }
* }
* }}</pre>
*
* @return
*/
@Override
public LongFunction<CreateIndexOp> createOpFunc(LongFunction<OpenSearchClient> clientF, ParsedOp op) {
CreateIndexRequest.Builder eb = new CreateIndexRequest.Builder();
LongFunction<CreateIndexRequest.Builder> bfunc = l -> new CreateIndexRequest.Builder().index("testindex1");
bfunc = op.enhanceFunc(bfunc, "mappings", Map.class, this::resolveTypeMapping);
LongFunction<CreateIndexRequest.Builder> finalBfunc = bfunc;
return (long l) -> new CreateIndexOp(clientF.apply(l), finalBfunc.apply(l).build());
public LongFunction<DeleteIndexOp> createOpFunc(LongFunction<OpenSearchClient> clientF, ParsedOp op) {
DeleteIndexRequest.Builder eb = new DeleteIndexRequest.Builder();
LongFunction<DeleteIndexRequest.Builder> f =
l -> new DeleteIndexRequest.Builder().index(targetF.apply(l));
return l -> new DeleteIndexOp(clientF.apply(l),f.apply(1).build());
}
// https://opensearch.org/docs/latest/search-plugins/knn/knn-index/
private CreateIndexRequest.Builder resolveTypeMapping(CreateIndexRequest.Builder eb, Map<?, ?> mappings) {
TypeMapping.Builder builder = new TypeMapping.Builder().properties(
Map.of(
"p1",
new Property.Builder().knnVector(new KnnVectorProperty.Builder()
.dimension(23)
.method(
new KnnVectorMethod.Builder()
.name("hnsw")
.engine("faiss")
.spaceType("l2")
.parameters(Map.of("ef_construction", JsonData.of(256),"m",JsonData.of(8)))
.build()
).build()
).build()
))
.indexField(new IndexField.Builder()
.enabled(true).build())
.fieldNames(new FieldNamesField.Builder()
.enabled(true).build()
);
return eb.mappings(b -> builder);
}
}

66
adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/DeleteOpDispenser.java
View File

@ -18,10 +18,12 @@ package io.nosqlbench.adapter.opensearch.dispensers;
import io.nosqlbench.adapter.opensearch.OpenSearchAdapter;
import io.nosqlbench.adapter.opensearch.ops.CreateIndexOp;
import io.nosqlbench.adapter.opensearch.ops.DeleteOp;
import io.nosqlbench.adapters.api.templating.ParsedOp;
import org.opensearch.client.json.JsonData;
import org.opensearch.client.opensearch.OpenSearchClient;
import org.opensearch.client.opensearch._types.mapping.*;
import org.opensearch.client.opensearch.core.DeleteRequest;
import org.opensearch.client.opensearch.indices.CreateIndexRequest;
import java.util.Map;
@ -29,66 +31,18 @@ import java.util.function.LongFunction;
public class DeleteOpDispenser extends BaseOpenSearchOpDispenser {
public DeleteOpDispenser(OpenSearchAdapter adapter, ParsedOp op) {
private final LongFunction<String> targetF;
public DeleteOpDispenser(OpenSearchAdapter adapter, ParsedOp op, LongFunction<String> targetF) {
super(adapter, op);
this.targetF = targetF;
}
/**
* {@see
* <a href="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-vector-search.html">doc
* </a>}
* <pre>{@code
* {
* "mappings": {
* "properties": {
* "value": {
* "type": "dense_vector",
* "dims": TEMPLATE(dimensions, 25),
* "index": true,
* "similarity": "TEMPLATE(similarity_function, cosine)"
* },
* "key": {
* "type": "text"
* }
* }
* }
* }}</pre>
*
* @return
*/
@Override
public LongFunction<CreateIndexOp> createOpFunc(LongFunction<OpenSearchClient> clientF, ParsedOp op) {
CreateIndexRequest.Builder eb = new CreateIndexRequest.Builder();
LongFunction<CreateIndexRequest.Builder> bfunc = l -> new CreateIndexRequest.Builder().index("testindex1");
bfunc = op.enhanceFunc(bfunc, "mappings", Map.class, this::resolveTypeMapping);
LongFunction<CreateIndexRequest.Builder> finalBfunc = bfunc;
return (long l) -> new CreateIndexOp(clientF.apply(l), finalBfunc.apply(l).build());
}
// https://opensearch.org/docs/latest/search-plugins/knn/knn-index/
private CreateIndexRequest.Builder resolveTypeMapping(CreateIndexRequest.Builder eb, Map<?, ?> mappings) {
TypeMapping.Builder builder = new TypeMapping.Builder().properties(
Map.of(
"p1",
new Property.Builder().knnVector(new KnnVectorProperty.Builder()
.dimension(23)
.method(
new KnnVectorMethod.Builder()
.name("hnsw")
.engine("faiss")
.spaceType("l2")
.parameters(Map.of("ef_construction", JsonData.of(256),"m",JsonData.of(8)))
.build()
).build()
).build()
))
.indexField(new IndexField.Builder()
.enabled(true).build())
.fieldNames(new FieldNamesField.Builder()
.enabled(true).build()
);
return eb.mappings(b -> builder);
public LongFunction<DeleteOp> createOpFunc(LongFunction<OpenSearchClient> clientF, ParsedOp op) {
DeleteRequest.Builder eb = new DeleteRequest.Builder();
LongFunction<DeleteRequest.Builder> bfunc = l -> new DeleteRequest.Builder().index(targetF.apply(l));
return (long l) -> new DeleteOp(clientF.apply(l), bfunc.apply(l).build());
}
}

5
adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/IndexOpDispenser.java
View File

@ -29,8 +29,11 @@ import java.util.Map;
import java.util.function.LongFunction;
public class IndexOpDispenser extends BaseOpenSearchOpDispenser {
public IndexOpDispenser(OpenSearchAdapter adapter, ParsedOp op) {
private final LongFunction<String> targetF;
public IndexOpDispenser(OpenSearchAdapter adapter, ParsedOp op, LongFunction<String> targetF) {
super(adapter, op);
this.targetF =targetF;
}
@Override

39
adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/UpdateOpDispenser.java
View File

@ -31,43 +31,18 @@ import java.util.function.LongFunction;
public class UpdateOpDispenser extends BaseOpenSearchOpDispenser {
public UpdateOpDispenser(OpenSearchAdapter adapter, ParsedOp op) {
private final LongFunction<String> targetF;
public UpdateOpDispenser(OpenSearchAdapter adapter, ParsedOp op, LongFunction<String> targetF) {
super(adapter, op);
this.targetF = targetF;
}
/**
* {@see
* <a href="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-vector-search.html">doc
* </a>}
* <pre>{@code
* {
* "mappings": {
* "properties": {
* "value": {
* "type": "dense_vector",
* "dims": TEMPLATE(dimensions, 25),
* "index": true,
* "similarity": "TEMPLATE(similarity_function, cosine)"
* },
* "key": {
* "type": "text"
* }
* }
* }
* }}</pre>
*
* @return
*/
@Override
public LongFunction<UpdateOp> createOpFunc(LongFunction<OpenSearchClient> clientF, ParsedOp op) {
return null;
// LongFunction<UpdateRequest.Builder> bfunc = l -> new UpdateRequest.Builder();
// op.getAsRequiredFunction("type")
// return l -> new UpdateOp(clientF.apply(l),bfunc.apply(l).build());
// bfunc = op.enhanceFunc(bfunc, "mappings", Map.class, this::resolveTypeMapping);
//
// LongFunction<CreateIndexRequest.Builder> finalBfunc = bfunc;
// return (long l) -> new CreateIndexOp(clientF.apply(l), finalBfunc.apply(l).build());
LongFunction<UpdateRequest.Builder> bfunc = l -> new UpdateRequest.Builder().index(targetF.apply(l));
// TODO: add details here
return l -> new UpdateOp(clientF.apply(l),bfunc.apply(l).build(),Object.class);
}
}

2
adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/CreateIndexOp.java
View File

@ -20,8 +20,6 @@ import org.opensearch.client.opensearch.OpenSearchClient;
import org.opensearch.client.opensearch.indices.CreateIndexRequest;
import org.opensearch.client.opensearch.indices.CreateIndexResponse;
import java.io.IOException;
public class CreateIndexOp extends BaseOpenSearchOp {
private final CreateIndexRequest rq;

36
adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/DeleteOp.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (c) 2024 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.adapter.opensearch.ops;
import org.opensearch.client.opensearch.OpenSearchClient;
import org.opensearch.client.opensearch.core.DeleteRequest;
import org.opensearch.client.opensearch.core.IndexRequest;
import java.io.IOException;
public class DeleteOp extends BaseOpenSearchOp {
private final DeleteRequest rq;
public DeleteOp(OpenSearchClient client, DeleteRequest rq) {
super(client);
this.rq = rq;
}
public Object applyOp(long value) throws IOException {
return client.delete(rq);
}
}

6
virtdata-api/src/main/java/io/nosqlbench/adapters/api/templating/TriFunction.java
View File

@ -2,13 +2,13 @@ package io.nosqlbench.adapters.api.templating;
/*
* Copyright (c) 2022 nosqlbench
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY