From a60c43c765b0d0729b752db6b5e89608eb8e4297 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 7 Feb 2024 14:21:27 -0600 Subject: [PATCH] incremental progress on opensearch --- .../opensearch/OpenSearchOpMapper.java | 7 +- .../adapter/opensearch/OpenSearchOpTypes.java | 6 +- .../dispensers/DeleteIndexOpDispenser.java | 94 +++++++++++++++++++ .../dispensers/DeleteOpDispenser.java | 94 +++++++++++++++++++ .../dispensers/IndexOpDispenser.java | 60 ++++++++++++ .../dispensers/UpdateOpDispenser.java | 73 ++++++++++++++ .../opensearch/ops/BaseOpenSearchOp.java | 15 ++- .../adapter/opensearch/ops/CreateIndexOp.java | 11 +-- .../adapter/opensearch/ops/DeleteIndexOp.java | 8 +- .../adapter/opensearch/ops/IndexOp.java | 38 ++++++++ .../adapter/opensearch/ops/UpdateOp.java | 38 ++++++++ 11 files changed, 428 insertions(+), 16 deletions(-) create mode 100644 adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/DeleteIndexOpDispenser.java create mode 100644 adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/DeleteOpDispenser.java create mode 100644 adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/IndexOpDispenser.java create mode 100644 adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/UpdateOpDispenser.java create mode 100644 adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/IndexOp.java create mode 100644 adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/UpdateOp.java diff --git a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/OpenSearchOpMapper.java b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/OpenSearchOpMapper.java index cef7c1b54..4126f6c26 100644 --- a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/OpenSearchOpMapper.java +++ b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/OpenSearchOpMapper.java @@ -16,7 +16,8 @@ package io.nosqlbench.adapter.opensearch; -import io.nosqlbench.adapter.opensearch.dispensers.CreateIndexOpDispenser; +import io.nosqlbench.adapter.opensearch.dispensers.*; +import io.nosqlbench.adapter.opensearch.ops.UpdateOp; import io.nosqlbench.adapters.api.activityimpl.OpDispenser; import io.nosqlbench.adapters.api.activityimpl.OpMapper; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverSpaceCache; @@ -39,6 +40,10 @@ public class OpenSearchOpMapper implements OpMapper { op.getTypeAndTarget(OpenSearchOpTypes.class, String.class, "verb", "index"); return switch (typeAndTarget.enumId) { case create_index -> new CreateIndexOpDispenser(adapter, op); + case delete_index -> new DeleteIndexOpDispenser(adapter, op); + case index -> new IndexOpDispenser(adapter,op); + case update -> new UpdateOpDispenser(adapter,op); + case delete -> new DeleteOpDispenser(adapter,op); default -> throw new RuntimeException("Unrecognized op type '" + typeAndTarget.enumId.name() + "' while " + "mapping parsed op " + op); }; diff --git a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/OpenSearchOpTypes.java b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/OpenSearchOpTypes.java index 54649451e..a926bedce 100644 --- a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/OpenSearchOpTypes.java +++ b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/OpenSearchOpTypes.java @@ -17,5 +17,9 @@ package io.nosqlbench.adapter.opensearch; public enum OpenSearchOpTypes { - create_index + create_index, + delete_index, + index, + update, + delete } diff --git a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/DeleteIndexOpDispenser.java b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/DeleteIndexOpDispenser.java new file mode 100644 index 000000000..40f201421 --- /dev/null +++ b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/DeleteIndexOpDispenser.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.opensearch.dispensers; + +import io.nosqlbench.adapter.opensearch.OpenSearchAdapter; +import io.nosqlbench.adapter.opensearch.ops.CreateIndexOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch._types.mapping.*; +import org.opensearch.client.opensearch.indices.CreateIndexRequest; + +import java.util.Map; +import java.util.function.LongFunction; + +public class DeleteIndexOpDispenser extends BaseOpenSearchOpDispenser { + + public DeleteIndexOpDispenser(OpenSearchAdapter adapter, ParsedOp op) { + super(adapter, op); + } + + /** + * {@see + * doc + * } + * 
{@code
+     *           {
+     *             "mappings": {
+     *               "properties": {
+     *                 "value": {
+     *                   "type": "dense_vector",
+     *                   "dims": TEMPLATE(dimensions, 25),
+     *                   "index": true,
+     *                   "similarity": "TEMPLATE(similarity_function, cosine)"
+     *                 },
+     *                 "key": {
+     *                   "type": "text"
+     *                 }
+     *               }
+     *             }
+     *           }}
+ * + * @return + */ + @Override + public LongFunction createOpFunc(LongFunction clientF, ParsedOp op) { + CreateIndexRequest.Builder eb = new CreateIndexRequest.Builder(); + LongFunction bfunc = l -> new CreateIndexRequest.Builder().index("testindex1"); + bfunc = op.enhanceFunc(bfunc, "mappings", Map.class, this::resolveTypeMapping); + + LongFunction finalBfunc = bfunc; + return (long l) -> new CreateIndexOp(clientF.apply(l), finalBfunc.apply(l).build()); + } + + // https://opensearch.org/docs/latest/search-plugins/knn/knn-index/ + private CreateIndexRequest.Builder resolveTypeMapping(CreateIndexRequest.Builder eb, Map mappings) { + TypeMapping.Builder builder = new TypeMapping.Builder().properties( + Map.of( + "p1", + new Property.Builder().knnVector(new KnnVectorProperty.Builder() + .dimension(23) + .method( + new KnnVectorMethod.Builder() + .name("hnsw") + .engine("faiss") + .spaceType("l2") + .parameters(Map.of("ef_construction", JsonData.of(256),"m",JsonData.of(8))) + .build() + ).build() + ).build() + )) + .indexField(new IndexField.Builder() + .enabled(true).build()) + .fieldNames(new FieldNamesField.Builder() + .enabled(true).build() + ); + return eb.mappings(b -> builder); + } + +} diff --git a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/DeleteOpDispenser.java b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/DeleteOpDispenser.java new file mode 100644 index 000000000..5f654b0e8 --- /dev/null +++ b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/DeleteOpDispenser.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.opensearch.dispensers; + +import io.nosqlbench.adapter.opensearch.OpenSearchAdapter; +import io.nosqlbench.adapter.opensearch.ops.CreateIndexOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch._types.mapping.*; +import org.opensearch.client.opensearch.indices.CreateIndexRequest; + +import java.util.Map; +import java.util.function.LongFunction; + +public class DeleteOpDispenser extends BaseOpenSearchOpDispenser { + + public DeleteOpDispenser(OpenSearchAdapter adapter, ParsedOp op) { + super(adapter, op); + } + + /** + * {@see + * doc + * } + * 
{@code
+     *           {
+     *             "mappings": {
+     *               "properties": {
+     *                 "value": {
+     *                   "type": "dense_vector",
+     *                   "dims": TEMPLATE(dimensions, 25),
+     *                   "index": true,
+     *                   "similarity": "TEMPLATE(similarity_function, cosine)"
+     *                 },
+     *                 "key": {
+     *                   "type": "text"
+     *                 }
+     *               }
+     *             }
+     *           }}
+ * + * @return + */ + @Override + public LongFunction createOpFunc(LongFunction clientF, ParsedOp op) { + CreateIndexRequest.Builder eb = new CreateIndexRequest.Builder(); + LongFunction bfunc = l -> new CreateIndexRequest.Builder().index("testindex1"); + bfunc = op.enhanceFunc(bfunc, "mappings", Map.class, this::resolveTypeMapping); + + LongFunction finalBfunc = bfunc; + return (long l) -> new CreateIndexOp(clientF.apply(l), finalBfunc.apply(l).build()); + } + + // https://opensearch.org/docs/latest/search-plugins/knn/knn-index/ + private CreateIndexRequest.Builder resolveTypeMapping(CreateIndexRequest.Builder eb, Map mappings) { + TypeMapping.Builder builder = new TypeMapping.Builder().properties( + Map.of( + "p1", + new Property.Builder().knnVector(new KnnVectorProperty.Builder() + .dimension(23) + .method( + new KnnVectorMethod.Builder() + .name("hnsw") + .engine("faiss") + .spaceType("l2") + .parameters(Map.of("ef_construction", JsonData.of(256),"m",JsonData.of(8))) + .build() + ).build() + ).build() + )) + .indexField(new IndexField.Builder() + .enabled(true).build()) + .fieldNames(new FieldNamesField.Builder() + .enabled(true).build() + ); + return eb.mappings(b -> builder); + } + +} diff --git a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/IndexOpDispenser.java b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/IndexOpDispenser.java new file mode 100644 index 000000000..86b1e7c0e --- /dev/null +++ b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/IndexOpDispenser.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.opensearch.dispensers; + +import io.nosqlbench.adapter.opensearch.OpenSearchAdapter; +import io.nosqlbench.adapter.opensearch.ops.IndexOp; +import io.nosqlbench.adapters.api.activityimpl.uniform.flowtypes.Op; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch._types.OpType; +import org.opensearch.client.opensearch._types.VersionType; +import org.opensearch.client.opensearch.core.IndexRequest; + +import java.util.Map; +import java.util.function.LongFunction; + +public class IndexOpDispenser extends BaseOpenSearchOpDispenser { + public IndexOpDispenser(OpenSearchAdapter adapter, ParsedOp op) { + super(adapter, op); + } + + @Override + public LongFunction createOpFunc(LongFunction clientF, ParsedOp op) { + LongFunction> func = l -> new IndexRequest.Builder<>(); + func = op.enhanceFuncOptionally(func, "index",String.class, IndexRequest.Builder::index); + func = op.enhanceFuncOptionally(func,"id",String.class, IndexRequest.Builder::id); + func = op.enhanceFuncOptionally(func,"ifPrimaryTerm",long.class, IndexRequest.Builder::ifPrimaryTerm); + func = op.enhanceFuncOptionally(func,"ifSeqNo",long.class,IndexRequest.Builder::ifSeqNo); + func = op.enhanceFuncOptionally(func,"pipeline", String.class, IndexRequest.Builder::pipeline); + func = op.enhanceFuncOptionally(func,"routing", String.class, IndexRequest.Builder::routing); + func = op.enhanceFuncOptionally(func,"requireAlias", boolean.class, IndexRequest.Builder::requireAlias); + func = op.enhanceFuncOptionally(func,"version", long.class, IndexRequest.Builder::version); + func = op.enhanceEnumOptionally(func,"opType", OpType.class,IndexRequest.Builder::opType); + func = op.enhanceEnumOptionally(func,"versionType", VersionType.class,IndexRequest.Builder::versionType); + func = op.enhanceFunc(func,"document",Object.class,(b1,d) -> this.bindDocument(b1,d)); + // TODO: func = op.enhanceFuncOptionally(func,"timeout",) ... + + LongFunction> finalFunc = func; + return l -> new IndexOp(clientF.apply(l), finalFunc.apply(l).build()); + } + + private IndexRequest.Builder bindDocument(IndexRequest.Builder builder, Object docdata) { + return builder.document(docdata); + } + +} diff --git a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/UpdateOpDispenser.java b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/UpdateOpDispenser.java new file mode 100644 index 000000000..16983e113 --- /dev/null +++ b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/dispensers/UpdateOpDispenser.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.opensearch.dispensers; + +import io.nosqlbench.adapter.opensearch.OpenSearchAdapter; +import io.nosqlbench.adapter.opensearch.ops.CreateIndexOp; +import io.nosqlbench.adapter.opensearch.ops.UpdateOp; +import io.nosqlbench.adapters.api.templating.ParsedOp; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch._types.mapping.*; +import org.opensearch.client.opensearch.core.UpdateRequest; +import org.opensearch.client.opensearch.indices.CreateIndexRequest; + +import java.util.Map; +import java.util.function.LongFunction; + +public class UpdateOpDispenser extends BaseOpenSearchOpDispenser { + + public UpdateOpDispenser(OpenSearchAdapter adapter, ParsedOp op) { + super(adapter, op); + } + + /** + * {@see + * doc + * } + * 
{@code
+     *           {
+     *             "mappings": {
+     *               "properties": {
+     *                 "value": {
+     *                   "type": "dense_vector",
+     *                   "dims": TEMPLATE(dimensions, 25),
+     *                   "index": true,
+     *                   "similarity": "TEMPLATE(similarity_function, cosine)"
+     *                 },
+     *                 "key": {
+     *                   "type": "text"
+     *                 }
+     *               }
+     *             }
+     *           }}
+ * + * @return + */ + @Override + public LongFunction createOpFunc(LongFunction clientF, ParsedOp op) { + return null; +// LongFunction bfunc = l -> new UpdateRequest.Builder(); +// op.getAsRequiredFunction("type") +// return l -> new UpdateOp(clientF.apply(l),bfunc.apply(l).build()); +// bfunc = op.enhanceFunc(bfunc, "mappings", Map.class, this::resolveTypeMapping); +// +// LongFunction finalBfunc = bfunc; +// return (long l) -> new CreateIndexOp(clientF.apply(l), finalBfunc.apply(l).build()); + } + +} diff --git a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/BaseOpenSearchOp.java b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/BaseOpenSearchOp.java index cb046af63..a54c2e67e 100644 --- a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/BaseOpenSearchOp.java +++ b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/BaseOpenSearchOp.java @@ -27,5 +27,18 @@ public abstract class BaseOpenSearchOp implements CycleOp { } @Override - public abstract Object apply(long value); + public final Object apply(long value) { + try { + Object result = applyOp(value); + return result; + } catch (Exception e) { + if (e instanceof RuntimeException rte) { + throw rte; + } else { + throw new RuntimeException(e); + } + } + }; + + public abstract Object applyOp(long value) throws Exception; } diff --git a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/CreateIndexOp.java b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/CreateIndexOp.java index f20cce7c1..94866f35f 100644 --- a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/CreateIndexOp.java +++ b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/CreateIndexOp.java @@ -31,12 +31,9 @@ public class CreateIndexOp extends BaseOpenSearchOp { } @Override - public Object apply(long value) { - try { - CreateIndexResponse response = client.indices().create(rq); - return response; - } catch (IOException e) { - throw new RuntimeException(e); - } + public Object applyOp(long value) throws Exception { + CreateIndexResponse response = client.indices().create(rq); + return response; } + } diff --git a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/DeleteIndexOp.java b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/DeleteIndexOp.java index 9de29080f..2ed88188c 100644 --- a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/DeleteIndexOp.java +++ b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/DeleteIndexOp.java @@ -31,11 +31,7 @@ public class DeleteIndexOp extends BaseOpenSearchOp { } @Override - public Object apply(long value) { - try { - return client.indices().delete(rq); - } catch (IOException e) { - throw new RuntimeException(e); - } + public Object applyOp(long value) throws IOException { + return client.indices().delete(rq); } } diff --git a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/IndexOp.java b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/IndexOp.java new file mode 100644 index 000000000..a2a9991bb --- /dev/null +++ b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/IndexOp.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.opensearch.ops; + +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.core.IndexRequest; +import org.opensearch.client.opensearch.core.IndexResponse; +import org.opensearch.client.opensearch.core.UpdateRequest; + +import java.io.IOException; + +public class IndexOp extends BaseOpenSearchOp { + private final IndexRequest rq; + + public IndexOp(OpenSearchClient client, IndexRequest rq) { + super(client); + this.rq = rq; + } + + + public Object applyOp(long value) throws IOException { + return client.index(rq); + } +} diff --git a/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/UpdateOp.java b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/UpdateOp.java new file mode 100644 index 000000000..4b2e18534 --- /dev/null +++ b/adapter-aws-opensearch/src/main/java/io/nosqlbench/adapter/opensearch/ops/UpdateOp.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024 nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.adapter.opensearch.ops; + +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.core.UpdateRequest; + +import java.io.IOException; + +public class UpdateOp extends BaseOpenSearchOp { + private final UpdateRequest rq; + private final Class doctype; + + public UpdateOp(OpenSearchClient client, UpdateRequest rq, Class doctype) { + super(client); + this.rq = rq; + this.doctype = doctype; + } + + @Override + public Object applyOp(long value) throws IOException { + return client.update(rq, doctype); + } +}