Updates to Filter building search points, count points and create payload index

nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantBaseOpDispenser.java

@@ -16,16 +16,31 @@
 
 package io.nosqlbench.adapter.qdrant.opdispensers;
 
+import com.google.protobuf.Timestamp;
 import io.nosqlbench.adapter.qdrant.QdrantDriverAdapter;
 import io.nosqlbench.adapter.qdrant.QdrantSpace;
 import io.nosqlbench.adapter.qdrant.ops.QdrantBaseOp;
 import io.nosqlbench.adapters.api.activityimpl.BaseOpDispenser;
 import io.nosqlbench.adapters.api.activityimpl.uniform.DriverAdapter;
 import io.nosqlbench.adapters.api.templating.ParsedOp;
+import io.nosqlbench.nb.api.errors.OpConfigError;
+import io.qdrant.client.PointIdFactory;
 import io.qdrant.client.QdrantClient;
+import io.qdrant.client.grpc.Points;
+import io.qdrant.client.grpc.Points.Condition;
+import io.qdrant.client.grpc.Points.DatetimeRange;
+import io.qdrant.client.grpc.Points.Filter;
+import io.qdrant.client.grpc.Points.Range;
 
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.function.LongFunction;
 
+import static io.qdrant.client.ConditionFactory.*;
+
 public abstract class QdrantBaseOpDispenser<T> extends BaseOpDispenser<QdrantBaseOp<T>, QdrantSpace> {
 
     protected final LongFunction<QdrantSpace> qdrantSpaceFunction;
@@ -61,4 +76,459 @@ public abstract class QdrantBaseOpDispenser<T> extends BaseOpDispenser<QdrantBas
     public QdrantBaseOp<T> getOp(long value) {
         return opF.apply(value);
     }
+
+    /**
+     * Builds the complete {@link Filter.Builder} from the {@code ParsedOp}
+     *
+     * @param {@link ParsedOp}
+     * @return {@code LongFunction<Filter.Builder>}
+     * @see <a href="https://qdrant.tech/documentation/concepts/filtering">Filtering</a>
+     */
+    protected LongFunction<Filter.Builder> getFilterFromOp(ParsedOp op) {
+        Optional<LongFunction<List>> filterFunc = op.getAsOptionalFunction("filter", List.class);
+        return filterFunc.<LongFunction<Filter.Builder>>map(filterListLongF -> l -> {
+            Filter.Builder filterBuilder = Filter.newBuilder();
+            List<Map<String, Object>> filters = filterListLongF.apply(l);
+            return constructFilterBuilder(filters, filterBuilder);
+        }).orElse(null);
+    }
+
+    private Filter.Builder constructFilterBuilder(List<Map<String, Object>> filters, Filter.Builder filterBuilder) {
+        List<Condition> mustClauseList = new ArrayList<>();
+        List<Condition> mustNotClauseList = new ArrayList<>();
+        List<Condition> shouldClauseList = new ArrayList<>();
+        for (Map<String, Object> filterFields : filters) {
+            switch ((String) filterFields.get("clause")) {
+                case "must" -> {
+                    logger.debug("[QdrantBaseOpDispenser] - Building 'must' filter clause");
+                    switch (filterFields.get("condition").toString()) {
+                        case "match" -> //filterBuilder.addMust(getMatchFilterCondition(filterFields));
+                            mustClauseList.add(getMatchFilterCondition(filterFields));
+                        case "match_any" -> mustClauseList.add(getMatchAnyExceptCondition(filterFields, "match_any"));
+                        case "match_except" ->
+                            mustClauseList.add(getMatchAnyExceptCondition(filterFields, "match_except"));
+                        case "text" -> mustClauseList.add(getMatchTextCondition(filterFields));
+                        case "range" -> mustClauseList.add(getRangeCondition(filterFields));
+                        case "geo_bounding_box" -> mustClauseList.add(getGeoBoundingBoxCondition(filterFields));
+                        case "geo_radius" -> mustClauseList.add(getGeoRadiusCondition(filterFields));
+                        case "geo_polygon" -> mustClauseList.add(getGeoPolygonCondition(filterFields));
+                        case "values_count" -> mustClauseList.add(getValuesCountCondition(filterFields));
+                        case "is_empty" -> mustClauseList.add(getIsEmptyCondition(filterFields));
+                        case "is_null" -> mustClauseList.add(getIsNullCondition(filterFields));
+                        case "has_id" -> mustClauseList.add(getHasIdCondition(filterFields));
+                        case "nested" -> mustClauseList.add(getNestedCondition(filterFields));
+                        default ->
+                            logger.warn("Filter condition '{}' is not supported", filterFields.get("condition").toString());
+                    }
+                }
+                case "must_not" -> {
+                    logger.debug("[QdrantBaseOpDispenser] - Building 'must_not' filter clause");
+                    switch (filterFields.get("condition").toString()) {
+                        case "match" -> mustNotClauseList.add(getMatchFilterCondition(filterFields));
+                        case "match_any" ->
+                            mustNotClauseList.add(getMatchAnyExceptCondition(filterFields, "match_any"));
+                        case "match_except" ->
+                            mustNotClauseList.add(getMatchAnyExceptCondition(filterFields, "match_except"));
+                        case "range" -> mustNotClauseList.add(getRangeCondition(filterFields));
+                        case "geo_bounding_box" -> mustNotClauseList.add(getGeoBoundingBoxCondition(filterFields));
+                        case "geo_radius" -> mustNotClauseList.add(getGeoRadiusCondition(filterFields));
+                        case "values_count" -> mustNotClauseList.add(getValuesCountCondition(filterFields));
+                        case "is_empty" -> mustNotClauseList.add(getIsEmptyCondition(filterFields));
+                        case "is_null" -> mustNotClauseList.add(getIsNullCondition(filterFields));
+                        case "has_id" -> mustClauseList.add(getHasIdCondition(filterFields));
+                        default ->
+                            logger.warn("Filter condition '{}' is not supported", filterFields.get("condition").toString());
+                    }
+                }
+                case "should" -> {
+                    logger.debug("[QdrantBaseOpDispenser] - Building 'should' filter clause");
+                    switch (filterFields.get("condition").toString()) {
+                        case "match" -> shouldClauseList.add(getMatchFilterCondition(filterFields));
+                        case "match_any" -> shouldClauseList.add(getMatchAnyExceptCondition(filterFields, "match_any"));
+                        case "match_except" ->
+                            shouldClauseList.add(getMatchAnyExceptCondition(filterFields, "match_except"));
+                        case "range" -> shouldClauseList.add(getRangeCondition(filterFields));
+                        case "geo_bounding_box" -> shouldClauseList.add(getGeoBoundingBoxCondition(filterFields));
+                        case "geo_radius" -> shouldClauseList.add(getGeoRadiusCondition(filterFields));
+                        case "values_count" -> shouldClauseList.add(getValuesCountCondition(filterFields));
+                        case "is_empty" -> shouldClauseList.add(getIsEmptyCondition(filterFields));
+                        case "is_null" -> shouldClauseList.add(getIsNullCondition(filterFields));
+                        case "has_id" -> mustClauseList.add(getHasIdCondition(filterFields));
+                        default ->
+                            logger.warn("Filter condition '{}' is not supported", filterFields.get("condition").toString());
+                    }
+                }
+                default -> logger.error("Clause '{}' is not supported", filterFields.get("clause"));
+            }
+        }
+        if (!mustClauseList.isEmpty()) {
+            filterBuilder.addAllMust(mustClauseList);
+        }
+        if (!mustNotClauseList.isEmpty()) {
+            filterBuilder.addAllMustNot(mustNotClauseList);
+        }
+        if (!shouldClauseList.isEmpty()) {
+            filterBuilder.addAllShould(shouldClauseList);
+        }
+        return filterBuilder;
+    }
+
+    private Condition getMatchFilterCondition(Map<String, Object> filterFields) {
+        logger.debug("[QdrantBaseOpDispenser] - Building 'match' filter condition");
+        if (filterFields.get("value") instanceof String) {
+            return matchKeyword((String) filterFields.get("key"), (String) filterFields.get("value"));
+        } else if (filterFields.get("value") instanceof Number) {
+            return match((String) filterFields.get("key"), ((Number) filterFields.get("value")).intValue());
+        } else if (filterFields.get("value") instanceof Boolean) {
+            return match((String) filterFields.get("key"), ((Boolean) filterFields.get("value")));
+        } else if (filterFields.containsKey("text")) {
+            // special case of 'match'
+            // https://qdrant.tech/documentation/concepts/filtering/#full-text-match
+            return getMatchTextCondition(filterFields);
+        } else {
+            throw new OpConfigError("Unsupported value type [" + filterFields.get("value").getClass().getSimpleName() + "] for 'match' condition");
+        }
+    }
+
+    private Condition getMatchAnyExceptCondition(Map<String, Object> filterFields, String filterCondition) {
+        logger.debug("[QdrantBaseOpDispenser] - Building 'match_any'/'match_except' filter condition");
+        if (filterFields.get("value") instanceof List) {
+            switch (((List<?>) filterFields.get("value")).getFirst().getClass().getSimpleName()) {
+                case "String" -> {
+                    if ("match_any".equals(filterCondition)) {
+                        return matchKeywords(filterFields.get("key").toString(), (List<String>) filterFields.get("value"));
+                    } else if ("match_except".equals(filterCondition)) {
+                        return matchExceptKeywords(filterFields.get("key").toString(), (List<String>) filterFields.get("value"));
+                    } else {
+                        throw new OpConfigError("Unsupported filter condition [" + filterCondition + "]");
+                    }
+                }
+                case "Long" -> {
+                    if ("match_any".equals(filterCondition)) {
+                        return matchValues(filterFields.get("key").toString(), (List<Long>) filterFields.get("value"));
+                    } else if ("match_except".equals(filterCondition)) {
+                        return matchExceptValues(filterFields.get("key").toString(), (List<Long>) filterFields.get("value"));
+                    } else {
+                        throw new OpConfigError("Unsupported filter condition [" + filterCondition + "]");
+                    }
+                }
+                case "Integer" -> {
+                    List<Long> convertedIntToLongValues = new ArrayList<>();
+                    for (Integer intVal : (List<Integer>) filterFields.get("value")) {
+                        convertedIntToLongValues.add(intVal.longValue());
+                    }
+                    if ("match_any".equals(filterCondition)) {
+                        return matchValues(filterFields.get("key").toString(), convertedIntToLongValues);
+                    } else if ("match_except".equals(filterCondition)) {
+                        return matchExceptValues(filterFields.get("key").toString(), convertedIntToLongValues);
+                    } else {
+                        throw new OpConfigError("Unsupported filter condition [" + filterCondition + "]");
+                    }
+                }
+                default -> throw new OpConfigError("Unsupported value type [" +
+                    filterFields.get("value").getClass().getSimpleName() +
+                    "] within value list for 'match_any'/'match_except' condition.\n" + filterFields.get("value"));
+            }
+        } else {
+            throw new OpConfigError("Unsupported value type [" + filterFields.get("value").getClass().getSimpleName() +
+                "] for 'match_any'/'match_except' condition");
+        }
+    }
+
+    private Condition getMatchTextCondition(Map<String, Object> filterFields) {
+        logger.debug("[QdrantBaseOpDispenser] - Building 'match_text' filter condition");
+        if (filterFields.get("text") instanceof String) {
+            return matchText((String) filterFields.get("key"), (String) filterFields.get("text"));
+        } else {
+            throw new OpConfigError("Unsupported value type [" +
+                filterFields.get("text").getClass().getSimpleName() + "] for 'match -> text' condition");
+        }
+    }
+
+    private Condition getRangeCondition(Map<String, Object> filterFields) {
+        logger.debug("[QdrantBaseOpDispenser] - Building 'range' filter condition");
+        if (filterFields.get("value") instanceof Map) {
+            if (((Map<String, ?>) filterFields.get("value")).keySet().stream().noneMatch(key ->
+                key.equals("gte") || key.equals("lte") || key.equals("lt") || key.equals("gt"))) {
+                throw new OpConfigError("Only gte/gt/lte/lt is expected for range condition, but received: "
+                    + ((Map<String, ?>) filterFields.get("value")).keySet());
+            }
+
+            Range.Builder rangeBuilder = Range.newBuilder();
+            DatetimeRange.Builder datetimerangeBuilder = DatetimeRange.newBuilder();
+
+            ((Map<?, ?>) filterFields.get("value")).forEach((rKey, rValue) -> {
+                if (rValue != null) {
+                    if (rValue instanceof Number) {
+                        switch ((String) rKey) {
+                            case "gte" -> rangeBuilder.setGte(((Number) rValue).doubleValue());
+                            case "gt" -> rangeBuilder.setGt(((Number) rValue).doubleValue());
+                            case "lte" -> rangeBuilder.setLte(((Number) rValue).doubleValue());
+                            case "lt" -> rangeBuilder.setLt(((Number) rValue).doubleValue());
+                        }
+                    } else if (rValue instanceof String) {
+                        // This is now a https://qdrant.tech/documentation/concepts/filtering/#datetime-range type
+                        long rVal = Instant.parse(String.valueOf(rValue)).getEpochSecond();
+                        Timestamp.Builder timestampBuilder = Timestamp.newBuilder().setSeconds(rVal);
+                        switch ((String) rKey) {
+                            case "gte" -> datetimerangeBuilder.setGte(timestampBuilder);
+                            case "gt" -> datetimerangeBuilder.setGt(timestampBuilder);
+                            case "lte" -> datetimerangeBuilder.setLte(timestampBuilder);
+                            case "lt" -> datetimerangeBuilder.setLt(timestampBuilder);
+                        }
+                    } else {
+                        logger.warn("Unsupported value [{}] ignored for 'range' filter condition.", rValue);
+                    }
+                }
+            });
+            if (datetimerangeBuilder.hasGt() || datetimerangeBuilder.hasGte()
+                || datetimerangeBuilder.hasLte() || datetimerangeBuilder.hasLt()) {
+                return datetimeRange((String) filterFields.get("key"), datetimerangeBuilder.build());
+            } else {
+                // we assume here this is Range type
+                if (rangeBuilder.hasGt() || rangeBuilder.hasGte() || rangeBuilder.hasLte() || rangeBuilder.hasLt())
+                    return range((String) filterFields.get("key"), rangeBuilder.build());
+            }
+        } else {
+            throw new OpConfigError("Unsupported value type [" + filterFields.get("value").getClass().getSimpleName() +
+                "] for 'range' condition. Needs a list containing gt/gte/lt/lte");
+        }
+        return null;
+    }
+
+    private Condition getGeoBoundingBoxCondition(Map<String, Object> filterFields) {
+        logger.debug("[QdrantBaseOpDispenser] - Building 'geo_bounding_box' filter condition");
+        if (filterFields.get("value") instanceof Map) {
+            Map<String, ?> valueMap = (Map<String, ?>) filterFields.get("value");
+            if (valueMap.keySet().stream().noneMatch(key -> key.equals("bottom_right") || key.equals("top_left"))) {
+                throw new OpConfigError("Both top_left & bottom_right are expected for geo_bounding_box condition, " +
+                    "but received: " + valueMap.keySet());
+            }
+
+            valueMap.forEach((rKey, rValue) -> {
+                if (rValue instanceof Map) {
+                    if (((Map<String, Number>) rValue).keySet().stream().noneMatch(geoLatLon ->
+                        geoLatLon.equals("lat") || geoLatLon.equals("lon"))) {
+                        throw new OpConfigError("Both 'top_left' & 'bottom_right' for 'geo_bounding_box' are expected" +
+                            " to have both 'lat' & 'lon' fields");
+                    }
+                } else {
+                    throw new OpConfigError("Unsupported value [" + rValue + "] ignored for 'geo_bounding_box' filter condition.");
+                }
+            });
+            double topLeftLat = (((Map<String, Number>) valueMap.get("top_left")).get("lat")).doubleValue();
+            double topLeftLon = (((Map<String, Number>) valueMap.get("top_left")).get("lon")).doubleValue();
+            double bottomRightLat = (((Map<String, Number>) valueMap.get("bottom_right")).get("lat")).doubleValue();
+            double bottomRightLon = (((Map<String, Number>) valueMap.get("bottom_right")).get("lon")).doubleValue();
+
+            return Condition.newBuilder()
+                .setField(Points.FieldCondition.newBuilder()
+                    .setKey((String) filterFields.get("key"))
+                    .setGeoBoundingBox(Points.GeoBoundingBox.newBuilder()
+                        .setTopLeft(Points.GeoPoint.newBuilder()
+                            .setLat(topLeftLat)
+                            .setLon(topLeftLon)
+                            .build())
+                        .setBottomRight(Points.GeoPoint.newBuilder()
+                            .setLat(bottomRightLat)
+                            .setLon(bottomRightLon)
+                            .build())
+                        .build())
+                    .build())
+                .build();
+        } else {
+            throw new OpConfigError("Unsupported value type [" + filterFields.get("value").getClass().getSimpleName() + "]" +
+                " for 'geo_bounding_box' condition. Needs a map containing 'top_left' & 'bottom_right' with 'lat' & 'lon'");
+        }
+    }
+
+    private Condition getGeoRadiusCondition(Map<String, Object> filterFields) {
+        logger.debug("[QdrantBaseOpDispenser] - Building 'geo_radius' filter condition");
+        if (filterFields.get("value") instanceof Map) {
+            Map<String, ?> valueMap = (Map<String, ?>) filterFields.get("value");
+            if (valueMap.keySet().stream().noneMatch(key -> key.equals("center") || key.equals("radius"))) {
+                throw new OpConfigError("Both 'center' & 'radius' are expected for 'geo_radius' condition, " +
+                    "but received: " + valueMap.keySet());
+            }
+
+            valueMap.forEach((rKey, rValue) -> {
+                if (rKey.equals("center")) {
+                    if (rValue instanceof Map) {
+                        if (((Map<String, Number>) rValue).keySet().stream().noneMatch(geoLatLon ->
+                            geoLatLon.equals("lat") || geoLatLon.equals("lon"))) {
+                            throw new OpConfigError("Both 'lat' & 'lon' within 'center' are expected" +
+                                " for the 'geo_radius' condition");
+                        }
+                    } else {
+                        throw new OpConfigError("Unsupported value [" + rValue + "] ignored for 'geo_radius'" +
+                            " -> 'center' filter condition.");
+                    }
+                }
+            });
+            double centerLat = (((Map<String, Number>) valueMap.get("center")).get("lat")).doubleValue();
+            double centerLon = (((Map<String, Number>) valueMap.get("center")).get("lon")).doubleValue();
+            float radius = ((Number) valueMap.get("radius")).floatValue();
+
+            return Condition.newBuilder()
+                .setField(Points.FieldCondition.newBuilder()
+                    .setKey((String) filterFields.get("key"))
+                    .setGeoRadius(Points.GeoRadius.newBuilder()
+                        .setCenter(Points.GeoPoint.newBuilder()
+                            .setLat(centerLat)
+                            .setLon(centerLon)
+                            .build())
+                        .setRadius(radius)
+                        .build())
+                    .build())
+                .build();
+        } else {
+            throw new OpConfigError("Unsupported value type [" + filterFields.get("value").getClass().getSimpleName() + "]" +
+                " for 'geo_radius' condition. Needs a map containing 'center' ('lat' & 'lon') and 'radius' fields");
+        }
+    }
+
+    private Condition getGeoPolygonCondition(Map<String, Object> filterFields) {
+        logger.debug("[QdrantBaseOpDispenser] - Building 'geo_polygon' filter condition");
+        if (filterFields.get("value") instanceof Map) {
+            Map<String, ?> valueMap = (Map<String, ?>) filterFields.get("value");
+            if (valueMap.keySet().stream().noneMatch(key -> key.equals("exterior_points") || key.equals("interior_points"))) {
+                throw new OpConfigError("Both 'exterior_points' & 'interior_points' with lat/lon array is required" +
+                    " for 'geo_polygon' filter condition");
+            }
+            Points.GeoLineString.Builder exteriorPoints = Points.GeoLineString.newBuilder();
+            List<Points.GeoPoint> extPoints = new ArrayList<>();
+            Points.GeoLineString.Builder interiorPoints = Points.GeoLineString.newBuilder();
+            List<Points.GeoPoint> intPoints = new ArrayList<>();
+            valueMap.forEach((gpKey, gpVal) -> {
+                switch ((String) gpKey) {
+                    case "exterior_points" -> {
+                        ((List<Map<String, Number>>) gpVal).forEach((endEp) -> {
+                            extPoints.add(Points.GeoPoint.newBuilder()
+                                .setLat((endEp).get("lat").doubleValue())
+                                .setLon((endEp).get("lon").doubleValue())
+                                .build());
+                        });
+                    }
+                    case "interior_points" -> {
+                        ((List<Map<String, Number>>) gpVal).forEach((endIp) -> {
+                            intPoints.add(Points.GeoPoint.newBuilder()
+                                .setLat((endIp).get("lat").doubleValue())
+                                .setLon((endIp).get("lon").doubleValue())
+                                .build());
+                        });
+                    }
+                }
+            });
+            exteriorPoints.addAllPoints(extPoints);
+            interiorPoints.addAllPoints(intPoints);
+            return geoPolygon((String) filterFields.get("key"), exteriorPoints.build(), List.of(interiorPoints.build()));
+        } else {
+            throw new OpConfigError("Unsupported type [" + filterFields.get("value").getClass().getSimpleName() +
+                "] passed for 'geo_polygon' filter condition");
+        }
+    }
+
+    private Condition getValuesCountCondition(Map<String, Object> filterFields) {
+        logger.debug("[QdrantBaseOpDispenser] - Building 'values_count' filter condition");
+        if (filterFields.get("value") instanceof Map) {
+            Map<String, ?> valueMap = (Map<String, ?>) filterFields.get("value");
+            if (valueMap.keySet().stream().noneMatch(key -> key.equals("gte") || key.equals("gt")
+                || key.equals("lte") || key.equals("lt"))) {
+                throw new OpConfigError("Only 'gte', 'gt', 'lte' or 'lt' is expected for 'values_count' condition, " +
+                    "but received: " + valueMap.keySet());
+            }
+            Points.ValuesCount.Builder valCntBuilder = Points.ValuesCount.newBuilder();
+            valueMap.forEach((vcKey, vcValue) -> {
+                if (vcValue != null) {
+                    switch (vcKey) {
+                        case "gte" -> valCntBuilder.setGte(((Number) vcValue).longValue());
+                        case "gt" -> valCntBuilder.setGt(((Number) vcValue).longValue());
+                        case "lte" -> valCntBuilder.setLte(((Number) vcValue).longValue());
+                        case "lt" -> valCntBuilder.setLt(((Number) vcValue).longValue());
+                    }
+                }
+            });
+
+            return Condition.newBuilder()
+                .setField(Points.FieldCondition.newBuilder()
+                    .setKey((String) filterFields.get("key"))
+                    .setValuesCount(valCntBuilder)
+                    .build())
+                .build();
+        } else {
+            throw new OpConfigError("Unsupported type [" + filterFields.get("value").getClass().getSimpleName() + "]" +
+                " for 'values_count' filter condition");
+        }
+    }
+
+    private Condition getIsNullCondition(Map<String, Object> filterFields) {
+        logger.debug("[QdrantBaseOpDispenser] - Building 'is_null' filter condition");
+        return Condition.newBuilder()
+            .setIsNull(Points.IsNullCondition.newBuilder()
+                .setKey((String) filterFields.get("key"))
+                .build())
+            .build();
+    }
+
+    private Condition getIsEmptyCondition(Map<String, Object> filterFields) {
+        logger.debug("[QdrantBaseOpDispenser] - Building 'is_empty' filter condition");
+        return Condition.newBuilder()
+            .setIsEmpty(Points.IsEmptyCondition.newBuilder()
+                .setKey((String) filterFields.get("key"))
+                .build())
+            .build();
+    }
+
+    /**
+     * This {@link nested} is only valid within a 'must' filter condition.
+     *
+     * @param filterFields
+     * @return
+     * @see <a href="https://qdrant.tech/documentation/concepts/filtering/#nested-object-filter">Nested Object Filter</a>
+     * @see <a href="https://github.com/qdrant/java-client/blob/v1.9.1/src/main/java/io/qdrant/client/ConditionFactory.java#L220-L249">code</a>
+     */
+    private Condition getNestedCondition(Map<String, Object> filterFields) {
+        logger.debug("[QdrantBaseOpDispenser] - Building 'nested' filter condition");
+        if (filterFields.get("nested") instanceof List) {
+            List<Condition> mustClauseList = new ArrayList<>();
+            Filter.Builder nestedFilter = Filter.newBuilder();
+
+            ((List<Map<String, Object>>) filterFields.get("nested")).forEach((nList) -> {
+                mustClauseList.add(getMatchFilterCondition(nList));
+            });
+            nestedFilter.addAllMust(mustClauseList);
+            return Condition.newBuilder()
+                .setNested(Points.NestedCondition.newBuilder()
+                    .setKey((String) filterFields.get("key"))
+                    .setFilter(nestedFilter)
+                    .build())
+                .build();
+        } else {
+            throw new OpConfigError("Unsupported type [" + filterFields.get("nested").getClass().getSimpleName() + "]" +
+                " for 'nested' filter condition");
+        }
+    }
+
+    private Condition getHasIdCondition(Map<String, Object> filterFields) {
+        if (filterFields.get("value") instanceof List) {
+            List<Points.PointId> pointIds = new ArrayList<>();
+            ((List<Object>) filterFields.get("value")).forEach((pId) -> {
+                if (pId instanceof Number) {
+                    pointIds.add(PointIdFactory.id(((Number) pId).longValue()));
+                } else if (pId instanceof String) {
+                    pointIds.add(Points.PointId.newBuilder().setUuid(pId.toString()).build());
+                } else {
+                    throw new OpConfigError("Unsupported type for 'id' specified [" + pId.getClass().getSimpleName() + "]");
+                }
+            });
+            return Condition.newBuilder()
+                .setHasId(Points.HasIdCondition.newBuilder()
+                    .addAllHasId(pointIds)
+                    .build())
+                .build();
+        } else {
+            throw new OpConfigError("Unsupported type [" + filterFields.get("value").getClass().getSimpleName() + "]" +
+                " for 'has_id' filter condition");
+        }
+    }
 }

nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCountPointsOpDispenser.java

@@ -22,6 +22,7 @@ import io.nosqlbench.adapter.qdrant.ops.QdrantCountPointsOp;
 import io.nosqlbench.adapters.api.templating.ParsedOp;
 import io.qdrant.client.QdrantClient;
 import io.qdrant.client.grpc.Points.CountPoints;
+import io.qdrant.client.grpc.Points.Filter;
 
 import java.util.function.LongFunction;
 
@@ -36,6 +37,10 @@ public class QdrantCountPointsOpDispenser extends QdrantBaseOpDispenser<CountPoi
         LongFunction<CountPoints.Builder> ebF =
             l -> CountPoints.newBuilder().setCollectionName(targetF.apply(l));
 
+        LongFunction<Filter.Builder> filterBuilder = getFilterFromOp(op);
+        final LongFunction<CountPoints.Builder> filterF = ebF;
+        ebF = l -> filterF.apply(l).setFilter(filterBuilder.apply(l));
+
         final LongFunction<CountPoints.Builder> lastF = ebF;
         return l -> lastF.apply(l).build();
     }

nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreatePayloadIndexOpDispenser.java

@@ -18,14 +18,18 @@ package io.nosqlbench.adapter.qdrant.opdispensers;
 
 import io.nosqlbench.adapter.qdrant.QdrantDriverAdapter;
 import io.nosqlbench.adapter.qdrant.ops.QdrantBaseOp;
-import io.nosqlbench.adapter.qdrant.ops.QdrantPayloadIndexOp;
+import io.nosqlbench.adapter.qdrant.ops.QdrantCreatePayloadIndexOp;
 import io.nosqlbench.adapters.api.templating.ParsedOp;
 import io.qdrant.client.QdrantClient;
-import io.qdrant.client.grpc.Collections.PayloadIndexParams;
+import io.qdrant.client.grpc.Points.CreateFieldIndexCollection;
+import io.qdrant.client.grpc.Points.FieldType;
+import io.qdrant.client.grpc.Points.WriteOrdering;
+import io.qdrant.client.grpc.Points.WriteOrderingType;
 
+import java.util.Optional;
 import java.util.function.LongFunction;
 
-public class QdrantCreatePayloadIndexOpDispenser extends QdrantBaseOpDispenser<PayloadIndexParams> {
+public class QdrantCreatePayloadIndexOpDispenser extends QdrantBaseOpDispenser<CreateFieldIndexCollection> {
     public QdrantCreatePayloadIndexOpDispenser(
         QdrantDriverAdapter adapter,
         ParsedOp op,
@@ -34,21 +38,44 @@ public class QdrantCreatePayloadIndexOpDispenser extends QdrantBaseOpDispenser<P
     }
 
     @Override
-    public LongFunction<PayloadIndexParams> getParamFunc(
+    public LongFunction<CreateFieldIndexCollection> getParamFunc(
         LongFunction<QdrantClient> clientF,
         ParsedOp op,
         LongFunction<String> targetF) {
-        LongFunction<PayloadIndexParams.Builder> ebF =
-            l -> PayloadIndexParams.newBuilder().setField(null, targetF.apply(l));
-        return l -> ebF.apply(l).build();
+        LongFunction<CreateFieldIndexCollection.Builder> ebF =
+            l -> CreateFieldIndexCollection.newBuilder().setCollectionName(targetF.apply(l));
+        // https://github.com/qdrant/java-client/blob/v1.9.1/src/main/java/io/qdrant/client/QdrantClient.java#L2240-L2248
+
+        ebF = op.enhanceFuncOptionally(ebF, "field_name", String.class, CreateFieldIndexCollection.Builder::setFieldName);
+
+        LongFunction<String> fieldTypeF = op.getAsRequiredFunction("field_type", String.class);
+        final LongFunction<CreateFieldIndexCollection.Builder> ftF = ebF;
+        ebF = l -> ftF.apply(l).setFieldType(FieldType.valueOf(fieldTypeF.apply(l)));
+
+        Optional<LongFunction<String>> writeOrderingF = op.getAsOptionalFunction("ordering", String.class);
+        if (writeOrderingF.isPresent()) {
+            LongFunction<CreateFieldIndexCollection.Builder> woF = ebF;
+            LongFunction<WriteOrdering> writeOrdrF = buildWriteOrderingFunc(writeOrderingF.get());
+            ebF = l -> woF.apply(l).setOrdering(writeOrdrF.apply(l));
+        }
+        ebF = op.enhanceFuncOptionally(ebF, "wait", Boolean.class, CreateFieldIndexCollection.Builder::setWait);
+
+        final LongFunction<CreateFieldIndexCollection.Builder> lastF = ebF;
+        return l -> lastF.apply(l).build();
     }
 
     @Override
-    public LongFunction<QdrantBaseOp<PayloadIndexParams>> createOpFunc(
-        LongFunction<PayloadIndexParams> paramF,
+    public LongFunction<QdrantBaseOp<CreateFieldIndexCollection>> createOpFunc(
+        LongFunction<CreateFieldIndexCollection> paramF,
         LongFunction<QdrantClient> clientF,
         ParsedOp op,
         LongFunction<String> targetF) {
-        return l -> new QdrantPayloadIndexOp(clientF.apply(l), paramF.apply(l));
+        return l -> new QdrantCreatePayloadIndexOp(clientF.apply(l), paramF.apply(l));
+    }
+
+    private LongFunction<WriteOrdering> buildWriteOrderingFunc(LongFunction<String> stringLongFunction) {
+        return l -> {
+            return WriteOrdering.newBuilder().setType(WriteOrderingType.valueOf(stringLongFunction.apply(l))).build();
+        };
     }
 }

nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantSearchPointsOpDispenser.java

@@ -19,7 +19,7 @@ package io.nosqlbench.adapter.qdrant.opdispensers;
 import io.nosqlbench.adapter.qdrant.QdrantDriverAdapter;
 import io.nosqlbench.adapter.qdrant.ops.QdrantBaseOp;
 import io.nosqlbench.adapter.qdrant.ops.QdrantSearchPointsOp;
-import io.nosqlbench.adapter.qdrant.pojo.SearchPointsHelper;
+import io.nosqlbench.adapter.qdrant.pojos.SearchPointsHelper;
 import io.nosqlbench.adapters.api.templating.ParsedOp;
 import io.nosqlbench.nb.api.errors.OpConfigError;
 import io.qdrant.client.QdrantClient;
@@ -74,7 +74,7 @@ public class QdrantSearchPointsOpDispenser extends QdrantBaseOpDispenser<SearchP
         ebF = l -> detailsOfNamedVectorsF.apply(l)
             .setVectorName(searchPointsHelperF.apply(l).getVectorName())
             .addAllVector(searchPointsHelperF.apply(l).getVectorValues());
-        //.setSparseIndices(searchPointsHelperF.apply(l).getSparseIndices()); throws NPE at their driver and hence below
+        //.setSparseIndices(searchPointsHelperF.apply(l).getSparseIndices()); throws NPE at Qdrant driver and hence below
         final LongFunction<SearchPoints.Builder> sparseIndicesF = ebF;
         ebF = l -> {
             SearchPoints.Builder builder = sparseIndicesF.apply(l);
@@ -106,7 +106,16 @@ public class QdrantSearchPointsOpDispenser extends QdrantBaseOpDispenser<SearchP
             ebF = l -> withVectorFunc.apply(l).setWithVectors(builtWithVector.apply(l));
         }
 
-        // TODO - Implement filter, params
+        Optional<LongFunction<Map>> optionalParams = op.getAsOptionalFunction("params", Map.class);
+        if (optionalParams.isPresent()) {
+            LongFunction<SearchPoints.Builder> paramsF = ebF;
+            LongFunction<SearchParams> params = buildSearchParams(optionalParams.get());
+            ebF = l -> paramsF.apply(l).setParams(params.apply(l));
+        }
+
+        LongFunction<Filter.Builder> filterBuilder = getFilterFromOp(op);
+        final LongFunction<SearchPoints.Builder> filterF = ebF;
+        ebF = l -> filterF.apply(l).setFilter(filterBuilder.apply(l));
 
         final LongFunction<SearchPoints.Builder> lastF = ebF;
         return l -> lastF.apply(l).build();
@@ -216,4 +225,37 @@ public class QdrantSearchPointsOpDispenser extends QdrantBaseOpDispenser<SearchP
             }
         };
     }
+
+    private LongFunction<SearchParams> buildSearchParams(LongFunction<Map> mapLongFunction) {
+        return l -> {
+            SearchParams.Builder searchParamsBuilder = SearchParams.newBuilder();
+            mapLongFunction.apply(l).forEach((key, val) -> {
+                if ("hnsw_config".equals(key)) {
+                    searchParamsBuilder.setHnswEf(((Number) val).longValue());
+                }
+                if ("exact".equals(key)) {
+                    searchParamsBuilder.setExact((Boolean) val);
+                }
+                if ("indexed_only".equals(key)) {
+                    searchParamsBuilder.setIndexedOnly((Boolean) val);
+                }
+                if ("quantization".equals(key)) {
+                    QuantizationSearchParams.Builder qsBuilder = QuantizationSearchParams.newBuilder();
+                    ((Map<String, Object>) val).forEach((qKey, qVal) -> {
+                        if ("ignore".equals(qKey)) {
+                            qsBuilder.setIgnore((Boolean) qVal);
+                        }
+                        if ("rescore".equals(qKey)) {
+                            qsBuilder.setRescore((Boolean) qVal);
+                        }
+                        if ("oversampling".equals(qKey)) {
+                            qsBuilder.setOversampling(((Number) qVal).doubleValue());
+                        }
+                    });
+                    searchParamsBuilder.setQuantization(qsBuilder);
+                }
+            });
+            return searchParamsBuilder.build();
+        };
+    }
 }

nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCreatePayloadIndexOp.java

@@ -16,17 +16,35 @@
 
 package io.nosqlbench.adapter.qdrant.ops;
 
-import io.nosqlbench.adapter.qdrant.pojos.CreatePayloadIndexRequest;
 import io.qdrant.client.QdrantClient;
+import io.qdrant.client.grpc.Collections.PayloadSchemaType;
+import io.qdrant.client.grpc.Points.CreateFieldIndexCollection;
+import io.qdrant.client.grpc.Points.UpdateResult;
 
-public class QdrantCreatePayloadIndexOp extends QdrantBaseOp<CreatePayloadIndexRequest> {
-    public QdrantCreatePayloadIndexOp(QdrantClient client, CreatePayloadIndexRequest request) {
+import java.time.Duration;
+
+public class QdrantCreatePayloadIndexOp extends QdrantBaseOp<CreateFieldIndexCollection> {
+    public QdrantCreatePayloadIndexOp(QdrantClient client, CreateFieldIndexCollection request) {
         super(client, request);
     }
 
     @Override
     public Object applyOp(long value) {
-        //client.createPayloadIndexAsync(PayloadIndexParams.get);
-        return null;
+        UpdateResult response;
+        try {
+            response = client.createPayloadIndexAsync(
+                    request.getCollectionName(),
+                    request.getFieldName(),
+                    PayloadSchemaType.forNumber(request.getFieldTypeValue()),
+                    request.getFieldIndexParams(),
+                    request.getWait(),
+                    request.getOrdering().getType(),
+                    Duration.ofSeconds(60000))
+                .get();
+            logger.debug("[QdrantCreatePayloadIndexOp] Response is {}", response);
+        } catch (Exception ex) {
+            throw new RuntimeException(ex);
+        }
+        return response;
     }
 }

nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml

@@ -178,9 +178,113 @@ blocks:
           relevancy.accept({relevant_indices_TEMPLATE(filetype)},actual_indices);
           // because we are "verifying" although this needs to be reorganized
           return true;
+        # More complex filtering available. See 'count_points' below for an example filter structure
+
+  create_payload_index:
+    ops:
+      create_payload_index_op:
+        create_payload_index: "TEMPLATE(collection)"
+        field_name: "field17"
+        field_type: "Keyword"
+        ordering: "Strong"
+        wait: true
+        # https://github.com/qdrant/qdrant/blob/v1.9.2/lib/api/src/grpc/proto/collections.proto#L395-L400
 
   count_vectors:
     ops:
       count_points_op:
         count_points: "TEMPLATE(collection)"
         exact: true
+        filter:
+        # More complex filtering logic could be provided as follows
+        #  - clause: "must"
+        #    condition: "match"
+        #    key: "field1"
+        #    value: "abc1"
+        #  - clause: "must_not"
+        #    condition: "match_any"
+        #    key: "field2"
+        #    value:
+        #      - "abc2"
+        #      - "abc3"
+        #  - clause: "should"
+        #    condition: "range"
+        #    key: "field3"
+        #    # any one of below
+        #    value:
+        #      gte: 10
+        #      lte: 20
+        #      gt: null
+        #      lt: null
+        #  - clause: "must"
+        #    condition: "nested"
+        #    key: "field4"
+        #    nested:
+        #      - condition: "match"
+        #        key: "field5[].whatsup"
+        #        value: "ni24maddy"
+        #      - condition: "match"
+        #        key: "field6"
+        #        value: true
+        #  - clause: "should"
+        #    condition: "has_id"
+        #    # no other keys are supported for this type
+        #    key: "id"
+        #    value:
+        #      - 1
+        #      - 2
+        #      - 3
+        #  - clause: "should"
+        #    condition: "match"
+        #    key: "field7"
+        #    # special case of match is text
+        #    text: "abc7"
+        #  - clause: "should"
+        #    condition: "geo_bounding_box"
+        #    key: "field8"
+        #    value:
+        #      top_left:
+        #        lat: 40.7128
+        #        lon: -74.0060
+        #      bottom_right:
+        #        lat: 40.7128
+        #        lon: -74.0060
+        #  - clause: "must_not"
+        #    condition: "geo_radius"
+        #    key: "field9"
+        #    value:
+        #      center:
+        #        lat: 40.7128
+        #        lon: -74.0060
+        #      radius: 100.0
+        #  - clause: "must"
+        #    condition: "geo_polygon"
+        #    key: "field10"
+        #    value:
+        #      exterior_points:
+        #        - lat: 30.7128
+        #          lon: -34.0060
+        #      interior_points:
+        #        - lat: 42.7128
+        #          lon: -54.0060
+        #  - clause: "should"
+        #    condition: "values_count"
+        #    key: "field11"
+        #    # Any one of below
+        #    value:
+        #      gte: 1
+        #      lte: 10
+        #      gt: null
+        #      lt: null
+        #  - clause: "must_not"
+        #    condition: "is_empty"
+        #    key: "field12"
+        #  - clause: "must"
+        #    condition: "is_null"
+        #    key: "field13"
+        #  - clause: "must"
+        #    condition: "match_except"
+        #    key: "field14"
+        #    value:
+        #      - 1
+        #      - 2