Merge pull request #1874 from nosqlbench/mwolters/predicates

Mwolters/predicates

.run/cql_vector2__drop__E5-BASE-V2.run.xml

@@ -11,4 +11,4 @@
     <option name="ALTERNATIVE_JRE_PATH" value="jdk21" />
     <method v="2" />
   </configuration>
-</component>
+</component>

.run/vectorsearch-consistency-levels.run.xml

@@ -12,4 +12,4 @@
     <option name="ALTERNATIVE_JRE_PATH" value="17" />
     <method v="2" />
   </configuration>
-</component>
+</component>

adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/hdf_to_cql/HdfDatasetToCqlPredicates.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023 nosqlbench
+ * Copyright (c) 2023-2024 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -12,6 +12,7 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
 package io.nosqlbench.datamappers.functions.hdf_to_cql;
@@ -22,6 +23,7 @@ import io.nosqlbench.nb.api.nbio.NBIO;
 import io.nosqlbench.virtdata.api.annotations.Categories;
 import io.nosqlbench.virtdata.api.annotations.Category;
 import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
+import io.nosqlbench.virtdata.library.hdf5.from_long.to_string.predicate_parser.DatasetParser;
 
 import java.util.function.LongFunction;
 

adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/hdf_to_cql/HdfPredicatesToCql.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package io.nosqlbench.datamappers.functions.hdf_to_cql;
+
+import io.jhdf.HdfFile;
+import io.jhdf.api.Dataset;
+import io.nosqlbench.nb.annotations.ServiceSelector;
+import io.nosqlbench.nb.api.nbio.NBIO;
+import io.nosqlbench.virtdata.api.annotations.Categories;
+import io.nosqlbench.virtdata.api.annotations.Category;
+import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
+import io.nosqlbench.virtdata.predicates.ast.PConjunction;
+import io.nosqlbench.virtdata.predicates.ast.POperator;
+import io.nosqlbench.virtdata.predicates.ast.PredicateExpr;
+import io.nosqlbench.virtdata.predicates.ast.PredicateTerm;
+import io.nosqlbench.virtdata.predicates.types.PredicateAdapter;
+import io.nosqlbench.virtdata.predicates.types.PredicateSerDes;
+
+import java.util.ServiceLoader;
+import java.util.function.LongFunction;
+import java.util.stream.Collectors;
+
+/**
+ * Binding function that accepts a long input value for the cycle and returns a string consisting of the
+ * CQL predicate parsed from a single record in an HDF5 dataset
+ */
+@ThreadSafeMapper
+@Categories(Category.experimental)
+public class HdfPredicatesToCql implements LongFunction<String>, PredicateAdapter {
+    private final HdfFile hdfFile;
+    private final Dataset dataset;
+    private final int recordCount;
+    private final PredicateSerDes serDes;
+    private String lastFieldName;
+
+    /**
+     * Create a new binding function that accepts a long input value for the cycle and returns a string
+     * @param filename The HDF5 file to read the predicate dataset from
+     * @param datasetName The name of the dataset internal to the HDF5 file
+     * @param serDesType The type of serialization/deserialization to use for the predicate
+     */
+    public HdfPredicatesToCql(String filename, String datasetName, String serDesType) {
+        hdfFile = new HdfFile(NBIO.all().search(filename).one().asPath());
+        dataset = hdfFile.getDatasetByPath(datasetName);
+        recordCount = dataset.getDimensions()[0];
+        serDes = ServiceSelector.of(serDesType, ServiceLoader.load(PredicateSerDes.class)).getOne();
+    }
+
+    @Override
+    public String apply(long l) {
+        long[] sliceOffset = {(l % recordCount)};
+        int[] sliceDimensions = {1};
+        return getPredicate(serDes.unserialize(((String[])dataset.getData(sliceOffset, sliceDimensions))[0]));
+    }
+
+    @Override
+    public String getPredicate(PredicateExpr model) {
+        StringBuilder sb = new StringBuilder().append("WHERE ");
+        sb.append(switch (model.getConjunction()) {
+            case PConjunction.none -> renderTerm((PredicateTerm) model);
+            case PConjunction.and -> renderTermsAnd(model);
+            case PConjunction.or -> renderTermsOr(model);
+        });
+        return sb.toString();
+    }
+
+    private String renderTerm(PredicateTerm pt) {
+        return pt.field.name + " " + pt.operator.symbol() + " " + pt.comparator.formattedValue();
+    }
+
+    private String renderTermsAnd(PredicateExpr pae) {
+        return pae.getTerms().stream().map(this::renderTerm).collect(Collectors.joining(" and "));
+    }
+
+    private String renderTermsOr(PredicateExpr poe) {
+        /*
+         * There is no OR term in CQL, so we need to convert it to IN. For each record we need to check the following:
+         *   1. Does the field name match
+         *   2. is the operator eq
+         * If either of these are not true this is an invalid expression for CQL
+         */
+        lastFieldName = poe.getTerms().getFirst().field.name;
+        return lastFieldName + " " + "IN(" +
+            poe.getTerms().stream().map(this::validateOrConditions).collect(Collectors.joining(",")) + ")";
+    }
+
+    private String validateOrConditions(PredicateTerm term) {
+        if ((term.field.name.equalsIgnoreCase(lastFieldName)) && term.operator.equals(POperator.eq)) {
+            return term.comparator.formattedValue();
+        } else {
+            throw new RuntimeException("OR term invalid for CQL: " + term);
+        }
+    }
+}

adapter-cqld4/src/test/java/io/nosqlbench/datamappers/functions/hdf_to_cql/DefaultDatasetParserTest.java

@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2023 nosqlbench
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.nosqlbench.datamappers.functions.hdf_to_cql;
-
-import org.junit.jupiter.api.Test;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-public class DefaultDatasetParserTest {
-    String test1 = "{\"conditions\": {\"and\": [{\"a\": {\"match\": {\"value\": 53}}}]}}";
-    String test2 = "{\"conditions\": {\"and\": [{\"a\": {\"match\": {\"value\": \"thirteen\"}}}, {\"b\": {\"match\": {\"value\": \"fifty-four\"}}}]}}";
-    String test3 = "{\"conditions\": {\"and\": [{\"a\": {\"match\": {\"value\": 13}}}, {\"b\": {\"match\": {\"value\": 54}}},  {\"a\": {\"match\": {\"value\": 154}}}]}}";
-    String test4 = "{\"conditions\": {\"or\": [{\"a\": {\"match\": {\"value\": 9}}}, {\"a\": {\"match\": {\"value\": 71}}}]}}";
-    String test5 = "{\"conditions\": {\"or\": [{\"a\": {\"match\": {\"value\": 9}}}, {\"a\": {\"match\": {\"value\": 71}}}, {\"a\": {\"match\": {\"value\": 7}}}]}}";
-    String test6 = "{\"conditions\": {\"or\": [{\"b\": {\"match\": {\"value\": \"foo\"}}}, {\"b\": {\"match\": {\"value\": \"bar\"}}}]}}";
-
-
-    @Test
-    public void testParse() {
-        DefaultDatasetParser parser = new DefaultDatasetParser();
-        String parsed = parser.parse(test1);
-        assertEquals("WHERE a=53", parsed);
-
-        parsed = parser.parse(test2);
-        assertEquals("WHERE a='thirteen' and b='fifty-four'", parsed);
-
-        parsed = parser.parse(test3);
-        assertEquals("WHERE a=13 and b=54 and a=154", parsed);
-
-        parsed = parser.parse(test4);
-        assertEquals("WHERE a IN(9,71)", parsed);
-
-        parsed = parser.parse(test5);
-        assertEquals("WHERE a IN(9,71,7)", parsed);
-
-        parsed = parser.parse(test6);
-        assertEquals("WHERE b IN('foo','bar')", parsed);
-    }
-
-}

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/library/hdf5/from_long/to_string/predicate_parser/DatasetParser.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023 nosqlbench
+ * Copyright (c) 2023-2024 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -12,9 +12,14 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
-package io.nosqlbench.datamappers.functions.hdf_to_cql;
+package io.nosqlbench.virtdata.library.hdf5.from_long.to_string.predicate_parser;
+
+import io.nosqlbench.virtdata.library.hdf5.from_long.to_string.predicate_parser.from_json.NoopDatasetParser;
+import io.nosqlbench.virtdata.library.hdf5.from_long.to_string.predicate_parser.from_json.to_cql.CqlDatasetParser;
+import io.nosqlbench.virtdata.library.hdf5.from_long.to_string.predicate_parser.from_json.to_cql.JAWDatasetParser;
 
 /**
  * This interface is used to parse the raw JSON from the HDF dataset into a CQL predicate.
@@ -28,7 +33,7 @@ public interface DatasetParser {
      */
     static DatasetParser parserFactory(String parsername) {
         return switch (parsername) {
-            case "default" -> new DefaultDatasetParser();
+            case "cql" -> new CqlDatasetParser();
             case "noop" -> new NoopDatasetParser();
             case "jaw" -> new JAWDatasetParser();
             default -> throw new RuntimeException("Unknown parser name: " + parsername);

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/library/hdf5/from_long/to_string/predicate_parser/from_json/NoopDatasetParser.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023 nosqlbench
+ * Copyright (c) 2023-2024 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -12,9 +12,13 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
-package io.nosqlbench.datamappers.functions.hdf_to_cql;
+package io.nosqlbench.virtdata.library.hdf5.from_long.to_string.predicate_parser.from_json;
+
+import io.nosqlbench.virtdata.library.hdf5.from_long.to_string.predicate_parser.DatasetParser;
+
 /**
  * This class is used to parse the raw JSON from the HDF dataset into a CQL predicate. This implementation
  * accepts a string consisting of the desired CQL predicate as translated from the original jsonl files and

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/library/hdf5/from_long/to_string/predicate_parser/from_json/to_cql/CqlDatasetParser.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023 nosqlbench
+ * Copyright (c) 2023-2024 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -12,22 +12,24 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
-package io.nosqlbench.datamappers.functions.hdf_to_cql;
+package io.nosqlbench.virtdata.library.hdf5.from_long.to_string.predicate_parser.from_json.to_cql;
 
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
+import io.nosqlbench.virtdata.library.hdf5.from_long.to_string.predicate_parser.DatasetParser;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 /**
  * This class is used to parse the raw JSON from the HDF dataset into a CQL predicate. This is the default
- * implementation. It accepts a JSON string of the form found at https://github.com/qdrant/ann-filtering-benchmark-datasets
+ * implementation. It accepts a JSON string of the form found at <a href="https://github.com/qdrant/ann-filtering-benchmark-datasets">...</a>
  * and converts it into a CQL predicate in String form
  */
-public class DefaultDatasetParser implements DatasetParser {
+public class CqlDatasetParser implements DatasetParser {
     private static final String WHERE = "WHERE";
     private static final String MATCH = "match";
     private static final String AND = "and";
@@ -41,7 +43,7 @@ public class DefaultDatasetParser implements DatasetParser {
     private static final String COMMA = ",";
     private static final String LEFT_PAREN = "(";
     private static final String RIGHT_PAREN = ")";
-    private static final Logger logger = LogManager.getLogger(DefaultDatasetParser.class);
+    private static final Logger logger = LogManager.getLogger(CqlDatasetParser.class);
 
     @Override
     public String parse(String raw) {

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/library/hdf5/from_long/to_string/predicate_parser/from_json/to_cql/JAWDatasetParser.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023 nosqlbench
+ * Copyright (c) 2023-2024 nosqlbench
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -12,9 +12,13 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
  */
 
-package io.nosqlbench.datamappers.functions.hdf_to_cql;
+package io.nosqlbench.virtdata.library.hdf5.from_long.to_string.predicate_parser.from_json.to_cql;
+
+import io.nosqlbench.virtdata.library.hdf5.from_long.to_string.predicate_parser.DatasetParser;
+
 /**
  * This class is used to parse the raw JSON from the HDF dataset into a CQL predicate. This implementation
  * accepts a string consisting of the desired CQL predicate as translated from the original jsonl files

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/library/hdf5/from_long/to_string/predicate_parser/predicates.md

@@ -0,0 +1,48 @@
+# Description
+
+The predicates parser is a way of adding predicates/filters/lmiting expressions to the vector queries generated by the bindings used in nb tests of various vector databases. The predicates will be stored as a dataset within an hdf5 file, which supports "file system like" data storage such as this. The possibility for expanding to either other data sources or other storage formats exists for the future but the initial implementation will be json within hdf5.
+
+# Usage
+Usage will be adapter dependent, as the bindings necessary to support different adapter types will vary.
+
+## CQL
+ops:
+  select_ann_limit:
+    raw: |
+      SELECT * FROM TEMPLATE(keyspace,baselines).TEMPLATE(table,vectors) {query_predicates} ORDER BY value ANN OF {test_floatlist} LIMIT TEMPLATE(select_limit,100);
+
+query_predicates: HdfDatasetToCqlPredicates("testdata/TEMPLATE(dataset).hdf5", "/predicates", "defaultCql")
+
+In this case the parser needs to return a string representation of the predicates that limit the result set returned by the query using the expected cql syntax.
+
+## Pinecone filter
+op1:
+  
+  query: "test-index"
+  
+  vector: "0.8602578079921012,0.12103044768221516,0.7737329191858439,0.4521093269320254,0.29351661477669416,0.4261807015226558,0.14131665592103335,0.882370813029422,0.4412833140430886,0.9916525700115515"
+  
+  namespace: "example_namespace"
+  
+  top_k: 10
+  
+  include_values: true
+  
+  include_metadata: true
+  
+  filter:
+  
+    filterfield: {filterfield_predicate}
+  
+    operator: {operator_predicate}
+  
+    comparator: {comparator_predicate}
+
+filterfield_predicate: HdfDatasetToPcFilterPredicates("testdata/TEMPLATE(dataset).hdf5", "/predicates", "pcfilter", "field")
+
+operator_predicate: HdfDatasetToPcFilterPredicates("testdata/TEMPLATE(dataset).hdf5", "/predicates", "pcfilter", "operator")
+
+comparator_predicate: HdfDatasetToPcFilterPredicates("testdata/TEMPLATE(dataset).hdf5", "/predicates", "pcfilter", "comparator")
+
+The same parser instance should be used across the three predicate portions with the desired field an additional parameter in order to avoid having to reparse for each binding.
+

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/predicates/JsonPredicateSerDes.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.predicates;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
+import io.nosqlbench.nb.annotations.Service;
+import io.nosqlbench.virtdata.predicates.ast.PredicateExpr;
+import io.nosqlbench.virtdata.predicates.ast.PredicateExprDefaultImpl;
+import io.nosqlbench.virtdata.predicates.types.PredicateSerDes;
+
+@Service(value=PredicateSerDes.class,selector = "json-serdes")
+public class JsonPredicateSerDes implements PredicateSerDes {
+    private final static Gson gson = new GsonBuilder().setPrettyPrinting().create();
+    private final static TypeToken<PredicateExprDefaultImpl> pexprType = TypeToken.get(PredicateExprDefaultImpl.class);
+
+    @Override
+    public PredicateExpr unserialize(String predicateData) {
+        PredicateExpr expr = gson.fromJson(predicateData, pexprType);
+        return expr;
+    }
+
+    @Override
+    public String serialize(PredicateExpr predicateExpr) {
+        return gson.toJson(predicateExpr);
+    }
+}

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/predicates/adapter/ExamplePredicateAdapter.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.predicates.adapter;
+
+import io.nosqlbench.virtdata.predicates.ast.*;
+import io.nosqlbench.virtdata.predicates.types.PredicateAdapter;
+
+import java.util.stream.Collectors;
+
+/**
+ * <P>This predicate adapter is not based on any real type of system.
+ * It provides the ability to render a string-form predicate clause
+ * for an imaginary system. This can be used for integrated testing
+ * and validation of high level configurations in the
+ * hands of users, like the diag adapter.</P>
+ *
+ * <P>The vernacular for this is simply JSON in the most obvious format,
+ * using verbs and nouns which are familiar from classic SQL systems.</P>
+ */
+public class ExamplePredicateAdapter implements PredicateAdapter {
+
+    @Override
+    public String getPredicate(PredicateExpr model) {
+        StringBuilder sb = new StringBuilder();
+        String fragment = switch (model.getConjunction()) {
+            case PConjunction.none -> renderTerm(model);
+            case PConjunction.and -> renderTermsAnd(model);
+            case PConjunction.or -> renderTermsOr(model);
+        };
+        sb.append(fragment);
+
+        return sb.toString();
+    }
+
+    private String renderTerm(PredicateExpr pe) {
+        PredicateTerm pt = pe.getTerms().get(0);
+        String value = pt.field.name + " " + pt.operator.name() + " " + pt.comparator.value;
+        return value;
+    }
+
+    private String renderTermsAnd(PredicateExpr pae) {
+        String value = pae.getTerms().stream().map(this::renderTerm).collect(Collectors.joining(" and "));
+        return value;
+    }
+
+    private String renderTermsOr(PredicateExpr poe) {
+        String value = poe.getTerms().stream().map(this::renderTerm).collect(Collectors.joining(" or "));
+        return value;
+    }
+}

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/predicates/ast/PComparator.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.predicates.ast;
+
+public class PComparator {
+    public Object value;
+
+    public PComparator(Object value) {
+        this.value = value;
+    }
+
+    public String formattedValue() {
+        return switch (value.getClass().getSimpleName()) {
+            case "String" -> "'" + value + "'";
+            default -> value.toString();
+        };
+    }
+}

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/predicates/ast/PConjunction.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.predicates.ast;
+
+public enum PConjunction {
+    none,
+    and,
+    or
+}

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/predicates/ast/PField.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.predicates.ast;
+
+public class PField {
+    public String name;
+
+    public PField(String name) {
+        this.name = name;
+    }
+}

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/predicates/ast/POperator.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.predicates.ast;
+
+public enum POperator {
+    eq,
+    gt,
+    lt,
+    lte,
+    gte;
+
+    public String symbol() {
+        return switch (this) {
+            case eq -> "=";
+            case gt -> ">";
+            case lt -> "<";
+            case lte -> "<=";
+            case gte -> ">=";
+        };
+    }
+}

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/predicates/ast/PredicateExpr.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.predicates.ast;
+
+import java.util.List;
+import java.util.Map;
+
+public interface PredicateExpr {
+    PConjunction getConjunction();
+
+    List<PredicateTerm> getTerms();
+}

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/predicates/ast/PredicateExprDefaultImpl.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package io.nosqlbench.virtdata.predicates.ast;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class PredicateExprDefaultImpl implements PredicateExpr {
+
+    public final List<PredicateTerm> terms = new ArrayList<>();
+    public PConjunction conjunction = PConjunction.none;
+
+    public PredicateExprDefaultImpl()  {
+    }
+    public PredicateExprDefaultImpl(List<PredicateTerm> terms) {
+        this.terms.addAll(terms);
+    }
+
+    public PredicateExprDefaultImpl term(PredicateTerm term) {
+        this.terms.add(term);
+        return this;
+    }
+
+    @Override
+    public PConjunction getConjunction() {
+        return this.conjunction;
+    }
+
+    @Override
+    public List<PredicateTerm> getTerms() {
+        return terms;
+    }
+}

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/predicates/ast/PredicateTerm.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.predicates.ast;
+
+public class PredicateTerm extends PredicateExprDefaultImpl {
+    public PredicateTerm(PField field, POperator operator, PComparator value) {
+        this.field = field;
+        this.operator = operator;
+        this.comparator = value;
+    }
+
+    public PField field;
+    public POperator operator;
+    public PComparator comparator;
+
+    public String toString() {
+        return field.name + " " + operator.name() + " " + comparator.value;
+    }
+}

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/predicates/package-info.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <P>The predicates module defines a few key elements which work together to
+ * allow driver adapters to share a common structure for predicate forms.
+ * These include:
+ * <UL>
+ *     <LI>{@link io.nosqlbench.virtdata.predicates.ast.PredicateExpr} - an
+ *     abstract syntax which captures the range of predicate forms which are supported.</LI>
+ *     <LI>{@link io.nosqlbench.virtdata.predicates.types.PredicateAdapter} - an
+ *     adapter type which can allow individual driver adapters to create a compatible
+ *     predicate clause or expression object from a
+ *     {@link io.nosqlbench.virtdata.predicates.ast.PredicateExpr}</LI>
+ *     <LI>{@link io.nosqlbench.virtdata.predicates.types.PredicateSerDes}
+ *     - serialization and deserialization functionality which allows
+ *     the core machinery of NB to read the common format to and from the
+ *     {@link io.nosqlbench.virtdata.predicates.ast.PredicateExpr} form.</LI>
+ * </UL>
+ * </P>
+ */
+package io.nosqlbench.virtdata.predicates;

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/predicates/types/PredicateAdapter.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.predicates.types;
+
+import io.nosqlbench.virtdata.predicates.ast.PredicateExpr;
+
+/**
+ * PredicateAdapters know how to read the predicate abstract syntax
+ * representation (in the form of a {@link PredicateExpr} and all it entails)
+ * and render a protocol-specific form. (adapter or system vernacular).
+ * The initial version presumed that a string form suffice for most cases,
+ * although this interface should be generified or specialized when needed to
+ * support other native in-memory representations, such as those which are
+ * based on native driver APIs and their direct types.
+ */
+public interface PredicateAdapter {
+    String getPredicate(PredicateExpr model);
+}

virtdata-lib-hdf5/src/main/java/io/nosqlbench/virtdata/predicates/types/PredicateSerDes.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.predicates.types;
+
+import io.nosqlbench.virtdata.predicates.ast.PredicateExpr;
+
+public interface PredicateSerDes {
+    public PredicateExpr unserialize(String predicateData);
+    public String serialize(PredicateExpr predicateExpr);
+}

virtdata-lib-hdf5/src/test/java/io/nosqlbench/virtdata/predicates/adapter/ExamplePredicateAdapterTest.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.predicates.adapter;
+
+import io.nosqlbench.virtdata.predicates.ast.*;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class ExamplePredicateAdapterTest {
+
+    public static PredicateExpr exampleExpr =
+        new PredicateExprDefaultImpl()
+            .term(
+                new PredicateTerm(new PField("username"), POperator.eq,new PComparator("joe"))
+            );
+    @Test
+    public void testBasicAdapterExample() {
+        ExamplePredicateAdapter epa = new ExamplePredicateAdapter();
+        String nativeForm = epa.getPredicate(exampleExpr);
+        assertThat(nativeForm)
+            .isEqualTo("username eq joe");
+    }
+
+}

virtdata-lib-hdf5/src/test/java/io/nosqlbench/virtdata/predicates/adapter/JsonPredicateSerDesTest.java

@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) 2024 nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package io.nosqlbench.virtdata.predicates.adapter;
+
+import io.nosqlbench.virtdata.predicates.JsonPredicateSerDes;
+import io.nosqlbench.virtdata.predicates.ast.PConjunction;
+import io.nosqlbench.virtdata.predicates.ast.POperator;
+import io.nosqlbench.virtdata.predicates.ast.PredicateExpr;
+import io.nosqlbench.virtdata.predicates.ast.PredicateTerm;
+import io.nosqlbench.virtdata.predicates.types.PredicateSerDes;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class JsonPredicateSerDesTest {
+
+    public static final String test1 = """
+    {
+        "conjunction": "none",
+        "terms": [
+            {
+                "field": {"name": "firstname"},
+                "operator": "eq",
+                "comparator": {"value": "Mark"}
+            }
+        ]
+    }
+    """;
+    public static final String test2 = """
+    {
+        "conjunction": "and",
+        "terms": [
+            {
+                "field": {"name": "firstname"},
+                "operator": "eq",
+                "comparator": {"value": "Mark"}
+            },
+            {
+                "field": {"name": "lastname"},
+                "operator": "eq",
+                "comparator": {"value": "Wolters"}
+            }
+        ]
+    }
+    """;
+
+    public static final String test3 = """
+    {
+        "conjunction": "or",
+        "terms": [
+            {
+                "field": {"name": "highprice"},
+                "operator": "gt",
+                "comparator": {"value": 1000}
+            },
+            {
+                "field": {"name": "lowprice"},
+                "operator": "lt",
+                "comparator": {"value": 1}
+            }
+        ]
+    }
+    """;
+
+    private static final PredicateSerDes instance = new JsonPredicateSerDes();
+
+    @Test
+    public void testUnserialize() {
+        PredicateExpr result1 = instance.unserialize(test1);
+        assertEquals(PConjunction.none, result1.getConjunction());
+        List<PredicateTerm> terms = result1.getTerms();
+        assertEquals(1, terms.size());
+        assertEquals("firstname", terms.getFirst().field.name);
+        assertEquals("Mark", terms.getFirst().comparator.value);
+        assertEquals(POperator.eq, terms.getFirst().operator);
+
+        PredicateExpr result2 = instance.unserialize(test2);
+        assertEquals(PConjunction.and, result2.getConjunction());
+        List<PredicateTerm> terms2 = result2.getTerms();
+        assertEquals(2, terms2.size());
+        assertEquals("firstname", terms2.getFirst().field.name);
+        assertEquals("Mark", terms2.getFirst().comparator.value);
+        assertEquals(POperator.eq, terms2.getFirst().operator);
+        assertEquals("lastname", terms2.get(1).field.name);
+        assertEquals("Wolters", terms2.get(1).comparator.value);
+        assertEquals(POperator.eq, terms2.get(1).operator);
+
+
+        PredicateExpr result3 = instance.unserialize(test3);
+        assertEquals(PConjunction.or, result3.getConjunction());
+        List<PredicateTerm> terms3 = result3.getTerms();
+        assertEquals(2, terms3.size());
+        assertEquals("highprice", terms3.getFirst().field.name);
+        assertEquals(1000.0, terms3.getFirst().comparator.value);
+        assertEquals(POperator.gt, terms3.getFirst().operator);
+        assertEquals("lowprice", terms3.get(1).field.name);
+        assertEquals(1.0, terms3.get(1).comparator.value);
+        assertEquals(POperator.lt, terms3.get(1).operator);
+
+    }
+
+    @Test
+    public void testSerialize() {
+        // PredicateExpr predicateExpr = null;
+        // PredicateSerDes instance = new JsonPredicateSerDes();
+        // String expResult = "";
+        // String result = instance.serialize(predicateExpr);
+        // assertEquals(expResult, result);
+    }
+}