separate milvus-only tree because its grpc deps do not play well with others

This commit is contained in:
Jonathan Shook 2024-03-24 15:16:39 -05:00
parent 623087ad42
commit aaa1d1dcab
9 changed files with 95 additions and 289 deletions

View File

@ -50,9 +50,10 @@ public class MilvusLoadCollectionOpDispenser extends MilvusBaseOpDispenser<LoadC
ebF = op.enhanceFuncOptionally(ebF,"sync_load",Boolean.class,LoadCollectionParam.Builder::withSyncLoad); ebF = op.enhanceFuncOptionally(ebF,"sync_load",Boolean.class,LoadCollectionParam.Builder::withSyncLoad);
ebF = op.enhanceFuncOptionally(ebF,"replica_number",Integer.class,LoadCollectionParam.Builder::withReplicaNumber); ebF = op.enhanceFuncOptionally(ebF,"replica_number",Integer.class,LoadCollectionParam.Builder::withReplicaNumber);
ebF = op.enhanceFuncOptionally(ebF,"resource_groups", List.class,LoadCollectionParam.Builder::withResourceGroups); ebF = op.enhanceFuncOptionally(ebF,"resource_groups", List.class,LoadCollectionParam.Builder::withResourceGroups);
ebF = op.enhanceFuncOptionally(ebF,"sync_load_waiting_interval",Long.class,LoadCollectionParam.Builder::withSyncLoadWaitingInterval); ebF = op.enhanceFuncOptionally(ebF,"sync_load_waiting_interval",Number.class,
ebF = op.enhanceFuncOptionally(ebF,"sync_load_waiting_timeout",Long.class, (LoadCollectionParam.Builder b,Number n) -> b.withSyncLoadWaitingInterval(n.longValue()));
LoadCollectionParam.Builder::withSyncLoadWaitingTimeout); ebF = op.enhanceFuncOptionally(ebF,"sync_load_waiting_timeout",Number.class,
(LoadCollectionParam.Builder b,Number n) -> b.withSyncLoadWaitingTimeout(n.longValue()));
final LongFunction<LoadCollectionParam.Builder> lastF = ebF; final LongFunction<LoadCollectionParam.Builder> lastF = ebF;
final LongFunction<LoadCollectionParam> collectionParamF = l -> lastF.apply(l).build(); final LongFunction<LoadCollectionParam> collectionParamF = l -> lastF.apply(l).build();

View File

@ -17,6 +17,8 @@
package io.nosqlbench.adapter.milvus.ops; package io.nosqlbench.adapter.milvus.ops;
import io.milvus.client.MilvusServiceClient; import io.milvus.client.MilvusServiceClient;
import io.milvus.param.R;
import io.milvus.param.RpcStatus;
import io.milvus.param.collection.CreateCollectionParam; import io.milvus.param.collection.CreateCollectionParam;
import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.adapters.api.templating.ParsedOp;
@ -33,6 +35,7 @@ public class MilvusCreateCollectionOp extends MilvusBaseOp<CreateCollectionParam
@Override @Override
public Object applyOp(long value) { public Object applyOp(long value) {
return client.createCollection(request); R<RpcStatus> collection = client.createCollection(request);
return collection;
} }
} }

View File

@ -17,6 +17,8 @@
package io.nosqlbench.adapter.milvus.ops; package io.nosqlbench.adapter.milvus.ops;
import io.milvus.client.MilvusServiceClient; import io.milvus.client.MilvusServiceClient;
import io.milvus.param.R;
import io.milvus.param.RpcStatus;
import io.milvus.param.collection.CreateDatabaseParam; import io.milvus.param.collection.CreateDatabaseParam;
public class MilvusCreateDatabaseOp extends MilvusBaseOp<CreateDatabaseParam> { public class MilvusCreateDatabaseOp extends MilvusBaseOp<CreateDatabaseParam> {
@ -26,6 +28,7 @@ public class MilvusCreateDatabaseOp extends MilvusBaseOp<CreateDatabaseParam> {
@Override @Override
public Object applyOp(long value) { public Object applyOp(long value) {
return client.createDatabase(request); R<RpcStatus> database = client.createDatabase(request);
return database;
} }
} }

View File

@ -36,6 +36,7 @@ public class MilvusInsertOp extends MilvusBaseOp<InsertParam> {
@Override @Override
public R<MutationResult> applyOp(long value) { public R<MutationResult> applyOp(long value) {
return client.insert(request); R<MutationResult> insert = client.insert(request);
return insert;
} }
} }

View File

@ -0,0 +1,6 @@
This driver has been disconnected from the main build and artifacts due to
it being abandoned by pinecone.
If it is needed for anything in the future, a special build should be done
which disables other modules, as there are incompatible versions of grpc
between this module and others.

View File

@ -49,7 +49,7 @@
<dependency> <dependency>
<groupId>io.pinecone</groupId> <groupId>io.pinecone</groupId>
<artifactId>pinecone-client</artifactId> <artifactId>pinecone-client</artifactId>
<version>0.7.4</version> <version>1.0.0-rc.1</version>
</dependency> </dependency>
</dependencies> </dependencies>

View File

@ -34,7 +34,7 @@
</description> </description>
<properties> <properties>
<pulsar.version>3.2.0</pulsar.version> <pulsar.version>3.2.1</pulsar.version>
</properties> </properties>
<dependencies> <dependencies>
@ -49,6 +49,12 @@
<groupId>org.apache.pulsar</groupId> <groupId>org.apache.pulsar</groupId>
<artifactId>pulsar-client</artifactId> <artifactId>pulsar-client</artifactId>
<version>${pulsar.version}</version> <version>${pulsar.version}</version>
<exclusions>
<exclusion>
<artifactId>protobuf-java</artifactId>
<groupId>com.google.protobuf</groupId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>

View File

@ -61,71 +61,67 @@
<version>${revision}</version> <version>${revision}</version>
</dependency> </dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-pinecone</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-dynamodb</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-http</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-tcp</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-pulsar</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-kafka</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-amqp</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-jdbc</artifactId>
<version>${revision}</version>
</dependency>
<dependency> <dependency>
<groupId>io.nosqlbench</groupId> <groupId>io.nosqlbench</groupId>
<artifactId>adapter-milvus</artifactId> <artifactId>adapter-milvus</artifactId>
<version>${revision}</version> <version>${revision}</version>
</dependency> </dependency>
<dependency>
<groupId>io.nosqlbench</groupId> <!-- <dependency>-->
<artifactId>adapter-mongodb</artifactId> <!-- <groupId>io.nosqlbench</groupId>-->
<version>${revision}</version> <!-- <artifactId>adapter-dynamodb</artifactId>-->
</dependency> <!-- <version>${revision}</version>-->
<dependency> <!-- </dependency>-->
<groupId>io.nosqlbench</groupId> <!-- <dependency>-->
<artifactId>adapter-aws-opensearch</artifactId> <!-- <groupId>io.nosqlbench</groupId>-->
<version>${revision}</version> <!-- <artifactId>adapter-http</artifactId>-->
</dependency> <!-- <version>${revision}</version>-->
<dependency> <!-- </dependency>-->
<groupId>io.nosqlbench</groupId> <!-- <dependency>-->
<artifactId>adapter-cqld4</artifactId> <!-- <groupId>io.nosqlbench</groupId>-->
<version>${revision}</version> <!-- <artifactId>adapter-tcp</artifactId>-->
</dependency> <!-- <version>${revision}</version>-->
<dependency> <!-- </dependency>-->
<groupId>io.nosqlbench</groupId> <!-- <dependency>-->
<artifactId>adapter-s4j</artifactId> <!-- <groupId>io.nosqlbench</groupId>-->
<version>${revision}</version> <!-- <artifactId>adapter-pulsar</artifactId>-->
</dependency> <!-- <version>${revision}</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>io.nosqlbench</groupId>-->
<!-- <artifactId>adapter-kafka</artifactId>-->
<!-- <version>${revision}</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>io.nosqlbench</groupId>-->
<!-- <artifactId>adapter-amqp</artifactId>-->
<!-- <version>${revision}</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>io.nosqlbench</groupId>-->
<!-- <artifactId>adapter-jdbc</artifactId>-->
<!-- <version>${revision}</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>io.nosqlbench</groupId>-->
<!-- <artifactId>adapter-mongodb</artifactId>-->
<!-- <version>${revision}</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>io.nosqlbench</groupId>-->
<!-- <artifactId>adapter-aws-opensearch</artifactId>-->
<!-- <version>${revision}</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>io.nosqlbench</groupId>-->
<!-- <artifactId>adapter-cqld4</artifactId>-->
<!-- <version>${revision}</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>io.nosqlbench</groupId>-->
<!-- <artifactId>adapter-s4j</artifactId>-->
<!-- <version>${revision}</version>-->
<!-- </dependency>-->
</dependencies> </dependencies>
@ -250,214 +246,5 @@
</plugins> </plugins>
</build> </build>
</profile> </profile>
<profile>
<id>adapter-opensearch</id>
<activation>
<!-- <activeByDefault>true</activeByDefault>-->
<file>
<exists>../adapter-opensearch/target</exists>
</file>
</activation>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-aws-opensearch</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>adapter-cqld4</id>
<activation>
<!-- <activeByDefault>true</activeByDefault>-->
<file>
<exists>../adapter-cqld4/target</exists>
</file>
</activation>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-cqld4</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>adapter-s4j</id>
<activation>
<!-- <activeByDefault>true</activeByDefault>-->
<file>
<exists>../adapter-s4j/target</exists>
</file>
</activation>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-s4j</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>adapter-pinecone</id>
<activation>
<!-- <activeByDefault>true</activeByDefault>-->
<file>
<exists>../adapter-pinecone/target</exists>
</file>
</activation>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-pinecone</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>adapter-mongodb</id>
<activation>
<!-- <activeByDefault>true</activeByDefault>-->
<file>
<exists>../adapter-mongodb/target</exists>
</file>
</activation>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-mongodb</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>adapter-tcp</id>
<activation>
<!-- <activeByDefault>true</activeByDefault>-->
<file>
<exists>../adapter-tcp/target</exists>
</file>
</activation>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-tcp</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>adapter-dynamodb</id>
<activation>
<!-- <activeByDefault>true</activeByDefault>-->
<file>
<exists>../adapter-dynamodb/target</exists>
</file>
</activation>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-dynamodb</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>adapter-http</id>
<activation>
<!-- <activeByDefault>true</activeByDefault>-->
<file>
<exists>../adapter-http/target</exists>
</file>
</activation>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-http</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>adapter-pulsar</id>
<activation>
<!-- <activeByDefault>true</activeByDefault>-->
<file>
<exists>../adapter-pulsar/target</exists>
</file>
</activation>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-pulsar</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>adapter-kafka</id>
<activation>
<!-- <activeByDefault>true</activeByDefault>-->
<file>
<exists>../adapter-kafka/target</exists>
</file>
</activation>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-kafka</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>adapter-amqp</id>
<activation>
<!-- <activeByDefault>true</activeByDefault>-->
<file>
<exists>../adapter-amqp/target</exists>
</file>
</activation>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-amqp</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>adapter-jdbc</id>
<activation>
<!-- <activeByDefault>true</activeByDefault>-->
<file>
<exists>../adapter-jdbc/target</exists>
</file>
</activation>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-jdbc</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>adapter-milvus</id>
<activation>
<!-- <activeByDefault>true</activeByDefault>-->
<file>
<exists>../adapter-milvus/target</exists>
</file>
</activation>
<dependencies>
<dependency>
<groupId>io.nosqlbench</groupId>
<artifactId>adapter-milvus</artifactId>
<version>${revision}</version>
</dependency>
</dependencies>
</profile>
</profiles> </profiles>
</project> </project>

23
pom.xml
View File

@ -59,19 +59,18 @@
<module>adapter-diag</module> <module>adapter-diag</module>
<module>adapter-stdout</module> <module>adapter-stdout</module>
<module>adapter-pinecone</module>
<module>adapter-dynamodb</module>
<module>adapter-http</module>
<module>adapter-tcp</module>
<module>adapter-pulsar</module>
<module>adapter-kafka</module>
<module>adapter-amqp</module>
<module>adapter-jdbc</module>
<module>adapter-milvus</module> <module>adapter-milvus</module>
<module>adapter-mongodb</module> <!-- <module>adapter-dynamodb</module>-->
<module>adapter-aws-opensearch</module> <!-- <module>adapter-http</module>-->
<module>adapter-cqld4</module> <!-- <module>adapter-tcp</module>-->
<module>adapter-s4j</module> <!-- <module>adapter-pulsar</module>-->
<!-- <module>adapter-kafka</module>-->
<!-- <module>adapter-amqp</module>-->
<!-- <module>adapter-jdbc</module>-->
<!-- <module>adapter-mongodb</module>-->
<!-- <module>adapter-aws-opensearch</module>-->
<!-- <module>adapter-cqld4</module>-->
<!-- <module>adapter-s4j</module>-->
<!-- VIRTDATA MODULES --> <!-- VIRTDATA MODULES -->
<module>virtdata-api</module> <module>virtdata-api</module>