From 61d0a017e80c4fff84b4e9bd3241a58b2d0abfe9 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Mon, 13 May 2024 17:21:07 -0400 Subject: [PATCH] Updates to Filter building search points, count points and create payload index --- .../opdispensers/QdrantBaseOpDispenser.java | 470 ++++++++++++++++++ .../QdrantCountPointsOpDispenser.java | 5 + .../QdrantCreatePayloadIndexOpDispenser.java | 47 +- .../QdrantSearchPointsOpDispenser.java | 48 +- .../ops/QdrantCreatePayloadIndexOp.java | 28 +- .../activities/qdrant_vectors_live.yaml | 104 ++++ 6 files changed, 684 insertions(+), 18 deletions(-) diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantBaseOpDispenser.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantBaseOpDispenser.java index fc3937c74..39f9edb87 100644 --- a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantBaseOpDispenser.java +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantBaseOpDispenser.java @@ -16,16 +16,31 @@ package io.nosqlbench.adapter.qdrant.opdispensers; +import com.google.protobuf.Timestamp; import io.nosqlbench.adapter.qdrant.QdrantDriverAdapter; import io.nosqlbench.adapter.qdrant.QdrantSpace; import io.nosqlbench.adapter.qdrant.ops.QdrantBaseOp; import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser; import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter; import io.nosqlbench.adapters.api.templating.ParsedOp; +import io.nosqlbench.nb.api.errors.OpConfigError; +import io.qdrant.client.PointIdFactory; import io.qdrant.client.QdrantClient; +import io.qdrant.client.grpc.Points; +import io.qdrant.client.grpc.Points.Condition; +import io.qdrant.client.grpc.Points.DatetimeRange; +import io.qdrant.client.grpc.Points.Filter; +import io.qdrant.client.grpc.Points.Range; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.function.LongFunction; +import static io.qdrant.client.ConditionFactory.*; + public abstract class QdrantBaseOpDispenser extends BaseOpDispenser, QdrantSpace> { protected final LongFunction qdrantSpaceFunction; @@ -61,4 +76,459 @@ public abstract class QdrantBaseOpDispenser extends BaseOpDispenser getOp(long value) { return opF.apply(value); } + + /** + * Builds the complete {@link Filter.Builder} from the {@code ParsedOp} + * + * @param {@link ParsedOp} + * @return {@code LongFunction} + * @see Filtering + */ + protected LongFunction getFilterFromOp(ParsedOp op) { + Optional> filterFunc = op.getAsOptionalFunction("filter", List.class); + return filterFunc.>map(filterListLongF -> l -> { + Filter.Builder filterBuilder = Filter.newBuilder(); + List> filters = filterListLongF.apply(l); + return constructFilterBuilder(filters, filterBuilder); + }).orElse(null); + } + + private Filter.Builder constructFilterBuilder(List> filters, Filter.Builder filterBuilder) { + List mustClauseList = new ArrayList<>(); + List mustNotClauseList = new ArrayList<>(); + List shouldClauseList = new ArrayList<>(); + for (Map filterFields : filters) { + switch ((String) filterFields.get("clause")) { + case "must" -> { + logger.debug("[QdrantBaseOpDispenser] - Building 'must' filter clause"); + switch (filterFields.get("condition").toString()) { + case "match" -> //filterBuilder.addMust(getMatchFilterCondition(filterFields)); + mustClauseList.add(getMatchFilterCondition(filterFields)); + case "match_any" -> mustClauseList.add(getMatchAnyExceptCondition(filterFields, "match_any")); + case "match_except" -> + mustClauseList.add(getMatchAnyExceptCondition(filterFields, "match_except")); + case "text" -> mustClauseList.add(getMatchTextCondition(filterFields)); + case "range" -> mustClauseList.add(getRangeCondition(filterFields)); + case "geo_bounding_box" -> mustClauseList.add(getGeoBoundingBoxCondition(filterFields)); + case "geo_radius" -> mustClauseList.add(getGeoRadiusCondition(filterFields)); + case "geo_polygon" -> mustClauseList.add(getGeoPolygonCondition(filterFields)); + case "values_count" -> mustClauseList.add(getValuesCountCondition(filterFields)); + case "is_empty" -> mustClauseList.add(getIsEmptyCondition(filterFields)); + case "is_null" -> mustClauseList.add(getIsNullCondition(filterFields)); + case "has_id" -> mustClauseList.add(getHasIdCondition(filterFields)); + case "nested" -> mustClauseList.add(getNestedCondition(filterFields)); + default -> + logger.warn("Filter condition '{}' is not supported", filterFields.get("condition").toString()); + } + } + case "must_not" -> { + logger.debug("[QdrantBaseOpDispenser] - Building 'must_not' filter clause"); + switch (filterFields.get("condition").toString()) { + case "match" -> mustNotClauseList.add(getMatchFilterCondition(filterFields)); + case "match_any" -> + mustNotClauseList.add(getMatchAnyExceptCondition(filterFields, "match_any")); + case "match_except" -> + mustNotClauseList.add(getMatchAnyExceptCondition(filterFields, "match_except")); + case "range" -> mustNotClauseList.add(getRangeCondition(filterFields)); + case "geo_bounding_box" -> mustNotClauseList.add(getGeoBoundingBoxCondition(filterFields)); + case "geo_radius" -> mustNotClauseList.add(getGeoRadiusCondition(filterFields)); + case "values_count" -> mustNotClauseList.add(getValuesCountCondition(filterFields)); + case "is_empty" -> mustNotClauseList.add(getIsEmptyCondition(filterFields)); + case "is_null" -> mustNotClauseList.add(getIsNullCondition(filterFields)); + case "has_id" -> mustClauseList.add(getHasIdCondition(filterFields)); + default -> + logger.warn("Filter condition '{}' is not supported", filterFields.get("condition").toString()); + } + } + case "should" -> { + logger.debug("[QdrantBaseOpDispenser] - Building 'should' filter clause"); + switch (filterFields.get("condition").toString()) { + case "match" -> shouldClauseList.add(getMatchFilterCondition(filterFields)); + case "match_any" -> shouldClauseList.add(getMatchAnyExceptCondition(filterFields, "match_any")); + case "match_except" -> + shouldClauseList.add(getMatchAnyExceptCondition(filterFields, "match_except")); + case "range" -> shouldClauseList.add(getRangeCondition(filterFields)); + case "geo_bounding_box" -> shouldClauseList.add(getGeoBoundingBoxCondition(filterFields)); + case "geo_radius" -> shouldClauseList.add(getGeoRadiusCondition(filterFields)); + case "values_count" -> shouldClauseList.add(getValuesCountCondition(filterFields)); + case "is_empty" -> shouldClauseList.add(getIsEmptyCondition(filterFields)); + case "is_null" -> shouldClauseList.add(getIsNullCondition(filterFields)); + case "has_id" -> mustClauseList.add(getHasIdCondition(filterFields)); + default -> + logger.warn("Filter condition '{}' is not supported", filterFields.get("condition").toString()); + } + } + default -> logger.error("Clause '{}' is not supported", filterFields.get("clause")); + } + } + if (!mustClauseList.isEmpty()) { + filterBuilder.addAllMust(mustClauseList); + } + if (!mustNotClauseList.isEmpty()) { + filterBuilder.addAllMustNot(mustNotClauseList); + } + if (!shouldClauseList.isEmpty()) { + filterBuilder.addAllShould(shouldClauseList); + } + return filterBuilder; + } + + private Condition getMatchFilterCondition(Map filterFields) { + logger.debug("[QdrantBaseOpDispenser] - Building 'match' filter condition"); + if (filterFields.get("value") instanceof String) { + return matchKeyword((String) filterFields.get("key"), (String) filterFields.get("value")); + } else if (filterFields.get("value") instanceof Number) { + return match((String) filterFields.get("key"), ((Number) filterFields.get("value")).intValue()); + } else if (filterFields.get("value") instanceof Boolean) { + return match((String) filterFields.get("key"), ((Boolean) filterFields.get("value"))); + } else if (filterFields.containsKey("text")) { + // special case of 'match' + // https://qdrant.tech/documentation/concepts/filtering/#full-text-match + return getMatchTextCondition(filterFields); + } else { + throw new OpConfigError("Unsupported value type [" + filterFields.get("value").getClass().getSimpleName() + "] for 'match' condition"); + } + } + + private Condition getMatchAnyExceptCondition(Map filterFields, String filterCondition) { + logger.debug("[QdrantBaseOpDispenser] - Building 'match_any'/'match_except' filter condition"); + if (filterFields.get("value") instanceof List) { + switch (((List) filterFields.get("value")).getFirst().getClass().getSimpleName()) { + case "String" -> { + if ("match_any".equals(filterCondition)) { + return matchKeywords(filterFields.get("key").toString(), (List) filterFields.get("value")); + } else if ("match_except".equals(filterCondition)) { + return matchExceptKeywords(filterFields.get("key").toString(), (List) filterFields.get("value")); + } else { + throw new OpConfigError("Unsupported filter condition [" + filterCondition + "]"); + } + } + case "Long" -> { + if ("match_any".equals(filterCondition)) { + return matchValues(filterFields.get("key").toString(), (List) filterFields.get("value")); + } else if ("match_except".equals(filterCondition)) { + return matchExceptValues(filterFields.get("key").toString(), (List) filterFields.get("value")); + } else { + throw new OpConfigError("Unsupported filter condition [" + filterCondition + "]"); + } + } + case "Integer" -> { + List convertedIntToLongValues = new ArrayList<>(); + for (Integer intVal : (List) filterFields.get("value")) { + convertedIntToLongValues.add(intVal.longValue()); + } + if ("match_any".equals(filterCondition)) { + return matchValues(filterFields.get("key").toString(), convertedIntToLongValues); + } else if ("match_except".equals(filterCondition)) { + return matchExceptValues(filterFields.get("key").toString(), convertedIntToLongValues); + } else { + throw new OpConfigError("Unsupported filter condition [" + filterCondition + "]"); + } + } + default -> throw new OpConfigError("Unsupported value type [" + + filterFields.get("value").getClass().getSimpleName() + + "] within value list for 'match_any'/'match_except' condition.\n" + filterFields.get("value")); + } + } else { + throw new OpConfigError("Unsupported value type [" + filterFields.get("value").getClass().getSimpleName() + + "] for 'match_any'/'match_except' condition"); + } + } + + private Condition getMatchTextCondition(Map filterFields) { + logger.debug("[QdrantBaseOpDispenser] - Building 'match_text' filter condition"); + if (filterFields.get("text") instanceof String) { + return matchText((String) filterFields.get("key"), (String) filterFields.get("text")); + } else { + throw new OpConfigError("Unsupported value type [" + + filterFields.get("text").getClass().getSimpleName() + "] for 'match -> text' condition"); + } + } + + private Condition getRangeCondition(Map filterFields) { + logger.debug("[QdrantBaseOpDispenser] - Building 'range' filter condition"); + if (filterFields.get("value") instanceof Map) { + if (((Map) filterFields.get("value")).keySet().stream().noneMatch(key -> + key.equals("gte") || key.equals("lte") || key.equals("lt") || key.equals("gt"))) { + throw new OpConfigError("Only gte/gt/lte/lt is expected for range condition, but received: " + + ((Map) filterFields.get("value")).keySet()); + } + + Range.Builder rangeBuilder = Range.newBuilder(); + DatetimeRange.Builder datetimerangeBuilder = DatetimeRange.newBuilder(); + + ((Map) filterFields.get("value")).forEach((rKey, rValue) -> { + if (rValue != null) { + if (rValue instanceof Number) { + switch ((String) rKey) { + case "gte" -> rangeBuilder.setGte(((Number) rValue).doubleValue()); + case "gt" -> rangeBuilder.setGt(((Number) rValue).doubleValue()); + case "lte" -> rangeBuilder.setLte(((Number) rValue).doubleValue()); + case "lt" -> rangeBuilder.setLt(((Number) rValue).doubleValue()); + } + } else if (rValue instanceof String) { + // This is now a https://qdrant.tech/documentation/concepts/filtering/#datetime-range type + long rVal = Instant.parse(String.valueOf(rValue)).getEpochSecond(); + Timestamp.Builder timestampBuilder = Timestamp.newBuilder().setSeconds(rVal); + switch ((String) rKey) { + case "gte" -> datetimerangeBuilder.setGte(timestampBuilder); + case "gt" -> datetimerangeBuilder.setGt(timestampBuilder); + case "lte" -> datetimerangeBuilder.setLte(timestampBuilder); + case "lt" -> datetimerangeBuilder.setLt(timestampBuilder); + } + } else { + logger.warn("Unsupported value [{}] ignored for 'range' filter condition.", rValue); + } + } + }); + if (datetimerangeBuilder.hasGt() || datetimerangeBuilder.hasGte() + || datetimerangeBuilder.hasLte() || datetimerangeBuilder.hasLt()) { + return datetimeRange((String) filterFields.get("key"), datetimerangeBuilder.build()); + } else { + // we assume here this is Range type + if (rangeBuilder.hasGt() || rangeBuilder.hasGte() || rangeBuilder.hasLte() || rangeBuilder.hasLt()) + return range((String) filterFields.get("key"), rangeBuilder.build()); + } + } else { + throw new OpConfigError("Unsupported value type [" + filterFields.get("value").getClass().getSimpleName() + + "] for 'range' condition. Needs a list containing gt/gte/lt/lte"); + } + return null; + } + + private Condition getGeoBoundingBoxCondition(Map filterFields) { + logger.debug("[QdrantBaseOpDispenser] - Building 'geo_bounding_box' filter condition"); + if (filterFields.get("value") instanceof Map) { + Map valueMap = (Map) filterFields.get("value"); + if (valueMap.keySet().stream().noneMatch(key -> key.equals("bottom_right") || key.equals("top_left"))) { + throw new OpConfigError("Both top_left & bottom_right are expected for geo_bounding_box condition, " + + "but received: " + valueMap.keySet()); + } + + valueMap.forEach((rKey, rValue) -> { + if (rValue instanceof Map) { + if (((Map) rValue).keySet().stream().noneMatch(geoLatLon -> + geoLatLon.equals("lat") || geoLatLon.equals("lon"))) { + throw new OpConfigError("Both 'top_left' & 'bottom_right' for 'geo_bounding_box' are expected" + + " to have both 'lat' & 'lon' fields"); + } + } else { + throw new OpConfigError("Unsupported value [" + rValue + "] ignored for 'geo_bounding_box' filter condition."); + } + }); + double topLeftLat = (((Map) valueMap.get("top_left")).get("lat")).doubleValue(); + double topLeftLon = (((Map) valueMap.get("top_left")).get("lon")).doubleValue(); + double bottomRightLat = (((Map) valueMap.get("bottom_right")).get("lat")).doubleValue(); + double bottomRightLon = (((Map) valueMap.get("bottom_right")).get("lon")).doubleValue(); + + return Condition.newBuilder() + .setField(Points.FieldCondition.newBuilder() + .setKey((String) filterFields.get("key")) + .setGeoBoundingBox(Points.GeoBoundingBox.newBuilder() + .setTopLeft(Points.GeoPoint.newBuilder() + .setLat(topLeftLat) + .setLon(topLeftLon) + .build()) + .setBottomRight(Points.GeoPoint.newBuilder() + .setLat(bottomRightLat) + .setLon(bottomRightLon) + .build()) + .build()) + .build()) + .build(); + } else { + throw new OpConfigError("Unsupported value type [" + filterFields.get("value").getClass().getSimpleName() + "]" + + " for 'geo_bounding_box' condition. Needs a map containing 'top_left' & 'bottom_right' with 'lat' & 'lon'"); + } + } + + private Condition getGeoRadiusCondition(Map filterFields) { + logger.debug("[QdrantBaseOpDispenser] - Building 'geo_radius' filter condition"); + if (filterFields.get("value") instanceof Map) { + Map valueMap = (Map) filterFields.get("value"); + if (valueMap.keySet().stream().noneMatch(key -> key.equals("center") || key.equals("radius"))) { + throw new OpConfigError("Both 'center' & 'radius' are expected for 'geo_radius' condition, " + + "but received: " + valueMap.keySet()); + } + + valueMap.forEach((rKey, rValue) -> { + if (rKey.equals("center")) { + if (rValue instanceof Map) { + if (((Map) rValue).keySet().stream().noneMatch(geoLatLon -> + geoLatLon.equals("lat") || geoLatLon.equals("lon"))) { + throw new OpConfigError("Both 'lat' & 'lon' within 'center' are expected" + + " for the 'geo_radius' condition"); + } + } else { + throw new OpConfigError("Unsupported value [" + rValue + "] ignored for 'geo_radius'" + + " -> 'center' filter condition."); + } + } + }); + double centerLat = (((Map) valueMap.get("center")).get("lat")).doubleValue(); + double centerLon = (((Map) valueMap.get("center")).get("lon")).doubleValue(); + float radius = ((Number) valueMap.get("radius")).floatValue(); + + return Condition.newBuilder() + .setField(Points.FieldCondition.newBuilder() + .setKey((String) filterFields.get("key")) + .setGeoRadius(Points.GeoRadius.newBuilder() + .setCenter(Points.GeoPoint.newBuilder() + .setLat(centerLat) + .setLon(centerLon) + .build()) + .setRadius(radius) + .build()) + .build()) + .build(); + } else { + throw new OpConfigError("Unsupported value type [" + filterFields.get("value").getClass().getSimpleName() + "]" + + " for 'geo_radius' condition. Needs a map containing 'center' ('lat' & 'lon') and 'radius' fields"); + } + } + + private Condition getGeoPolygonCondition(Map filterFields) { + logger.debug("[QdrantBaseOpDispenser] - Building 'geo_polygon' filter condition"); + if (filterFields.get("value") instanceof Map) { + Map valueMap = (Map) filterFields.get("value"); + if (valueMap.keySet().stream().noneMatch(key -> key.equals("exterior_points") || key.equals("interior_points"))) { + throw new OpConfigError("Both 'exterior_points' & 'interior_points' with lat/lon array is required" + + " for 'geo_polygon' filter condition"); + } + Points.GeoLineString.Builder exteriorPoints = Points.GeoLineString.newBuilder(); + List extPoints = new ArrayList<>(); + Points.GeoLineString.Builder interiorPoints = Points.GeoLineString.newBuilder(); + List intPoints = new ArrayList<>(); + valueMap.forEach((gpKey, gpVal) -> { + switch ((String) gpKey) { + case "exterior_points" -> { + ((List>) gpVal).forEach((endEp) -> { + extPoints.add(Points.GeoPoint.newBuilder() + .setLat((endEp).get("lat").doubleValue()) + .setLon((endEp).get("lon").doubleValue()) + .build()); + }); + } + case "interior_points" -> { + ((List>) gpVal).forEach((endIp) -> { + intPoints.add(Points.GeoPoint.newBuilder() + .setLat((endIp).get("lat").doubleValue()) + .setLon((endIp).get("lon").doubleValue()) + .build()); + }); + } + } + }); + exteriorPoints.addAllPoints(extPoints); + interiorPoints.addAllPoints(intPoints); + return geoPolygon((String) filterFields.get("key"), exteriorPoints.build(), List.of(interiorPoints.build())); + } else { + throw new OpConfigError("Unsupported type [" + filterFields.get("value").getClass().getSimpleName() + + "] passed for 'geo_polygon' filter condition"); + } + } + + private Condition getValuesCountCondition(Map filterFields) { + logger.debug("[QdrantBaseOpDispenser] - Building 'values_count' filter condition"); + if (filterFields.get("value") instanceof Map) { + Map valueMap = (Map) filterFields.get("value"); + if (valueMap.keySet().stream().noneMatch(key -> key.equals("gte") || key.equals("gt") + || key.equals("lte") || key.equals("lt"))) { + throw new OpConfigError("Only 'gte', 'gt', 'lte' or 'lt' is expected for 'values_count' condition, " + + "but received: " + valueMap.keySet()); + } + Points.ValuesCount.Builder valCntBuilder = Points.ValuesCount.newBuilder(); + valueMap.forEach((vcKey, vcValue) -> { + if (vcValue != null) { + switch (vcKey) { + case "gte" -> valCntBuilder.setGte(((Number) vcValue).longValue()); + case "gt" -> valCntBuilder.setGt(((Number) vcValue).longValue()); + case "lte" -> valCntBuilder.setLte(((Number) vcValue).longValue()); + case "lt" -> valCntBuilder.setLt(((Number) vcValue).longValue()); + } + } + }); + + return Condition.newBuilder() + .setField(Points.FieldCondition.newBuilder() + .setKey((String) filterFields.get("key")) + .setValuesCount(valCntBuilder) + .build()) + .build(); + } else { + throw new OpConfigError("Unsupported type [" + filterFields.get("value").getClass().getSimpleName() + "]" + + " for 'values_count' filter condition"); + } + } + + private Condition getIsNullCondition(Map filterFields) { + logger.debug("[QdrantBaseOpDispenser] - Building 'is_null' filter condition"); + return Condition.newBuilder() + .setIsNull(Points.IsNullCondition.newBuilder() + .setKey((String) filterFields.get("key")) + .build()) + .build(); + } + + private Condition getIsEmptyCondition(Map filterFields) { + logger.debug("[QdrantBaseOpDispenser] - Building 'is_empty' filter condition"); + return Condition.newBuilder() + .setIsEmpty(Points.IsEmptyCondition.newBuilder() + .setKey((String) filterFields.get("key")) + .build()) + .build(); + } + + /** + * This {@link nested} is only valid within a 'must' filter condition. + * + * @param filterFields + * @return + * @see Nested Object Filter + * @see code + */ + private Condition getNestedCondition(Map filterFields) { + logger.debug("[QdrantBaseOpDispenser] - Building 'nested' filter condition"); + if (filterFields.get("nested") instanceof List) { + List mustClauseList = new ArrayList<>(); + Filter.Builder nestedFilter = Filter.newBuilder(); + + ((List>) filterFields.get("nested")).forEach((nList) -> { + mustClauseList.add(getMatchFilterCondition(nList)); + }); + nestedFilter.addAllMust(mustClauseList); + return Condition.newBuilder() + .setNested(Points.NestedCondition.newBuilder() + .setKey((String) filterFields.get("key")) + .setFilter(nestedFilter) + .build()) + .build(); + } else { + throw new OpConfigError("Unsupported type [" + filterFields.get("nested").getClass().getSimpleName() + "]" + + " for 'nested' filter condition"); + } + } + + private Condition getHasIdCondition(Map filterFields) { + if (filterFields.get("value") instanceof List) { + List pointIds = new ArrayList<>(); + ((List) filterFields.get("value")).forEach((pId) -> { + if (pId instanceof Number) { + pointIds.add(PointIdFactory.id(((Number) pId).longValue())); + } else if (pId instanceof String) { + pointIds.add(Points.PointId.newBuilder().setUuid(pId.toString()).build()); + } else { + throw new OpConfigError("Unsupported type for 'id' specified [" + pId.getClass().getSimpleName() + "]"); + } + }); + return Condition.newBuilder() + .setHasId(Points.HasIdCondition.newBuilder() + .addAllHasId(pointIds) + .build()) + .build(); + } else { + throw new OpConfigError("Unsupported type [" + filterFields.get("value").getClass().getSimpleName() + "]" + + " for 'has_id' filter condition"); + } + } } diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCountPointsOpDispenser.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCountPointsOpDispenser.java index f0f4edbed..8c14b2954 100644 --- a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCountPointsOpDispenser.java +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCountPointsOpDispenser.java @@ -22,6 +22,7 @@ import io.nosqlbench.adapter.qdrant.ops.QdrantCountPointsOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.qdrant.client.QdrantClient; import io.qdrant.client.grpc.Points.CountPoints; +import io.qdrant.client.grpc.Points.Filter; import java.util.function.LongFunction; @@ -36,6 +37,10 @@ public class QdrantCountPointsOpDispenser extends QdrantBaseOpDispenser ebF = l -> CountPoints.newBuilder().setCollectionName(targetF.apply(l)); + LongFunction filterBuilder = getFilterFromOp(op); + final LongFunction filterF = ebF; + ebF = l -> filterF.apply(l).setFilter(filterBuilder.apply(l)); + final LongFunction lastF = ebF; return l -> lastF.apply(l).build(); } diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreatePayloadIndexOpDispenser.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreatePayloadIndexOpDispenser.java index 75406f86c..b69afb7fa 100644 --- a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreatePayloadIndexOpDispenser.java +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreatePayloadIndexOpDispenser.java @@ -18,14 +18,18 @@ package io.nosqlbench.adapter.qdrant.opdispensers; import io.nosqlbench.adapter.qdrant.QdrantDriverAdapter; import io.nosqlbench.adapter.qdrant.ops.QdrantBaseOp; -import io.nosqlbench.adapter.qdrant.ops.QdrantPayloadIndexOp; +import io.nosqlbench.adapter.qdrant.ops.QdrantCreatePayloadIndexOp; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.qdrant.client.QdrantClient; -import io.qdrant.client.grpc.Collections.PayloadIndexParams; +import io.qdrant.client.grpc.Points.CreateFieldIndexCollection; +import io.qdrant.client.grpc.Points.FieldType; +import io.qdrant.client.grpc.Points.WriteOrdering; +import io.qdrant.client.grpc.Points.WriteOrderingType; +import java.util.Optional; import java.util.function.LongFunction; -public class QdrantCreatePayloadIndexOpDispenser extends QdrantBaseOpDispenser { +public class QdrantCreatePayloadIndexOpDispenser extends QdrantBaseOpDispenser { public QdrantCreatePayloadIndexOpDispenser( QdrantDriverAdapter adapter, ParsedOp op, @@ -34,21 +38,44 @@ public class QdrantCreatePayloadIndexOpDispenser extends QdrantBaseOpDispenser

getParamFunc( + public LongFunction getParamFunc( LongFunction clientF, ParsedOp op, LongFunction targetF) { - LongFunction ebF = - l -> PayloadIndexParams.newBuilder().setField(null, targetF.apply(l)); - return l -> ebF.apply(l).build(); + LongFunction ebF = + l -> CreateFieldIndexCollection.newBuilder().setCollectionName(targetF.apply(l)); + // https://github.com/qdrant/java-client/blob/v1.9.1/src/main/java/io/qdrant/client/QdrantClient.java#L2240-L2248 + + ebF = op.enhanceFuncOptionally(ebF, "field_name", String.class, CreateFieldIndexCollection.Builder::setFieldName); + + LongFunction fieldTypeF = op.getAsRequiredFunction("field_type", String.class); + final LongFunction ftF = ebF; + ebF = l -> ftF.apply(l).setFieldType(FieldType.valueOf(fieldTypeF.apply(l))); + + Optional> writeOrderingF = op.getAsOptionalFunction("ordering", String.class); + if (writeOrderingF.isPresent()) { + LongFunction woF = ebF; + LongFunction writeOrdrF = buildWriteOrderingFunc(writeOrderingF.get()); + ebF = l -> woF.apply(l).setOrdering(writeOrdrF.apply(l)); + } + ebF = op.enhanceFuncOptionally(ebF, "wait", Boolean.class, CreateFieldIndexCollection.Builder::setWait); + + final LongFunction lastF = ebF; + return l -> lastF.apply(l).build(); } @Override - public LongFunction> createOpFunc( - LongFunction paramF, + public LongFunction> createOpFunc( + LongFunction paramF, LongFunction clientF, ParsedOp op, LongFunction targetF) { - return l -> new QdrantPayloadIndexOp(clientF.apply(l), paramF.apply(l)); + return l -> new QdrantCreatePayloadIndexOp(clientF.apply(l), paramF.apply(l)); + } + + private LongFunction buildWriteOrderingFunc(LongFunction stringLongFunction) { + return l -> { + return WriteOrdering.newBuilder().setType(WriteOrderingType.valueOf(stringLongFunction.apply(l))).build(); + }; } } diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantSearchPointsOpDispenser.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantSearchPointsOpDispenser.java index 61a8419d8..c54fd50d5 100644 --- a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantSearchPointsOpDispenser.java +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantSearchPointsOpDispenser.java @@ -19,7 +19,7 @@ package io.nosqlbench.adapter.qdrant.opdispensers; import io.nosqlbench.adapter.qdrant.QdrantDriverAdapter; import io.nosqlbench.adapter.qdrant.ops.QdrantBaseOp; import io.nosqlbench.adapter.qdrant.ops.QdrantSearchPointsOp; -import io.nosqlbench.adapter.qdrant.pojo.SearchPointsHelper; +import io.nosqlbench.adapter.qdrant.pojos.SearchPointsHelper; import io.nosqlbench.adapters.api.templating.ParsedOp; import io.nosqlbench.nb.api.errors.OpConfigError; import io.qdrant.client.QdrantClient; @@ -74,7 +74,7 @@ public class QdrantSearchPointsOpDispenser extends QdrantBaseOpDispenser detailsOfNamedVectorsF.apply(l) .setVectorName(searchPointsHelperF.apply(l).getVectorName()) .addAllVector(searchPointsHelperF.apply(l).getVectorValues()); - //.setSparseIndices(searchPointsHelperF.apply(l).getSparseIndices()); throws NPE at their driver and hence below + //.setSparseIndices(searchPointsHelperF.apply(l).getSparseIndices()); throws NPE at Qdrant driver and hence below final LongFunction sparseIndicesF = ebF; ebF = l -> { SearchPoints.Builder builder = sparseIndicesF.apply(l); @@ -106,7 +106,16 @@ public class QdrantSearchPointsOpDispenser extends QdrantBaseOpDispenser withVectorFunc.apply(l).setWithVectors(builtWithVector.apply(l)); } - // TODO - Implement filter, params + Optional> optionalParams = op.getAsOptionalFunction("params", Map.class); + if (optionalParams.isPresent()) { + LongFunction paramsF = ebF; + LongFunction params = buildSearchParams(optionalParams.get()); + ebF = l -> paramsF.apply(l).setParams(params.apply(l)); + } + + LongFunction filterBuilder = getFilterFromOp(op); + final LongFunction filterF = ebF; + ebF = l -> filterF.apply(l).setFilter(filterBuilder.apply(l)); final LongFunction lastF = ebF; return l -> lastF.apply(l).build(); @@ -216,4 +225,37 @@ public class QdrantSearchPointsOpDispenser extends QdrantBaseOpDispenser buildSearchParams(LongFunction mapLongFunction) { + return l -> { + SearchParams.Builder searchParamsBuilder = SearchParams.newBuilder(); + mapLongFunction.apply(l).forEach((key, val) -> { + if ("hnsw_config".equals(key)) { + searchParamsBuilder.setHnswEf(((Number) val).longValue()); + } + if ("exact".equals(key)) { + searchParamsBuilder.setExact((Boolean) val); + } + if ("indexed_only".equals(key)) { + searchParamsBuilder.setIndexedOnly((Boolean) val); + } + if ("quantization".equals(key)) { + QuantizationSearchParams.Builder qsBuilder = QuantizationSearchParams.newBuilder(); + ((Map) val).forEach((qKey, qVal) -> { + if ("ignore".equals(qKey)) { + qsBuilder.setIgnore((Boolean) qVal); + } + if ("rescore".equals(qKey)) { + qsBuilder.setRescore((Boolean) qVal); + } + if ("oversampling".equals(qKey)) { + qsBuilder.setOversampling(((Number) qVal).doubleValue()); + } + }); + searchParamsBuilder.setQuantization(qsBuilder); + } + }); + return searchParamsBuilder.build(); + }; + } } diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCreatePayloadIndexOp.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCreatePayloadIndexOp.java index ac6b5d2aa..b8431e184 100644 --- a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCreatePayloadIndexOp.java +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCreatePayloadIndexOp.java @@ -16,17 +16,35 @@ package io.nosqlbench.adapter.qdrant.ops; -import io.nosqlbench.adapter.qdrant.pojos.CreatePayloadIndexRequest; import io.qdrant.client.QdrantClient; +import io.qdrant.client.grpc.Collections.PayloadSchemaType; +import io.qdrant.client.grpc.Points.CreateFieldIndexCollection; +import io.qdrant.client.grpc.Points.UpdateResult; -public class QdrantCreatePayloadIndexOp extends QdrantBaseOp { - public QdrantCreatePayloadIndexOp(QdrantClient client, CreatePayloadIndexRequest request) { +import java.time.Duration; + +public class QdrantCreatePayloadIndexOp extends QdrantBaseOp { + public QdrantCreatePayloadIndexOp(QdrantClient client, CreateFieldIndexCollection request) { super(client, request); } @Override public Object applyOp(long value) { - //client.createPayloadIndexAsync(PayloadIndexParams.get); - return null; + UpdateResult response; + try { + response = client.createPayloadIndexAsync( + request.getCollectionName(), + request.getFieldName(), + PayloadSchemaType.forNumber(request.getFieldTypeValue()), + request.getFieldIndexParams(), + request.getWait(), + request.getOrdering().getType(), + Duration.ofSeconds(60000)) + .get(); + logger.debug("[QdrantCreatePayloadIndexOp] Response is {}", response); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + return response; } } diff --git a/nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml b/nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml index 547c15373..169207b2c 100644 --- a/nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml +++ b/nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml @@ -178,9 +178,113 @@ blocks: relevancy.accept({relevant_indices_TEMPLATE(filetype)},actual_indices); // because we are "verifying" although this needs to be reorganized return true; + # More complex filtering available. See 'count_points' below for an example filter structure + + create_payload_index: + ops: + create_payload_index_op: + create_payload_index: "TEMPLATE(collection)" + field_name: "field17" + field_type: "Keyword" + ordering: "Strong" + wait: true + # https://github.com/qdrant/qdrant/blob/v1.9.2/lib/api/src/grpc/proto/collections.proto#L395-L400 count_vectors: ops: count_points_op: count_points: "TEMPLATE(collection)" exact: true + filter: + # More complex filtering logic could be provided as follows + # - clause: "must" + # condition: "match" + # key: "field1" + # value: "abc1" + # - clause: "must_not" + # condition: "match_any" + # key: "field2" + # value: + # - "abc2" + # - "abc3" + # - clause: "should" + # condition: "range" + # key: "field3" + # # any one of below + # value: + # gte: 10 + # lte: 20 + # gt: null + # lt: null + # - clause: "must" + # condition: "nested" + # key: "field4" + # nested: + # - condition: "match" + # key: "field5[].whatsup" + # value: "ni24maddy" + # - condition: "match" + # key: "field6" + # value: true + # - clause: "should" + # condition: "has_id" + # # no other keys are supported for this type + # key: "id" + # value: + # - 1 + # - 2 + # - 3 + # - clause: "should" + # condition: "match" + # key: "field7" + # # special case of match is text + # text: "abc7" + # - clause: "should" + # condition: "geo_bounding_box" + # key: "field8" + # value: + # top_left: + # lat: 40.7128 + # lon: -74.0060 + # bottom_right: + # lat: 40.7128 + # lon: -74.0060 + # - clause: "must_not" + # condition: "geo_radius" + # key: "field9" + # value: + # center: + # lat: 40.7128 + # lon: -74.0060 + # radius: 100.0 + # - clause: "must" + # condition: "geo_polygon" + # key: "field10" + # value: + # exterior_points: + # - lat: 30.7128 + # lon: -34.0060 + # interior_points: + # - lat: 42.7128 + # lon: -54.0060 + # - clause: "should" + # condition: "values_count" + # key: "field11" + # # Any one of below + # value: + # gte: 1 + # lte: 10 + # gt: null + # lt: null + # - clause: "must_not" + # condition: "is_empty" + # key: "field12" + # - clause: "must" + # condition: "is_null" + # key: "field13" + # - clause: "must" + # condition: "match_except" + # key: "field14" + # value: + # - 1 + # - 2