Jshook/vectors (#2210)

* vector data support

* update monitor loop

* license headers

nb-annotations/src/main/java/io/nosqlbench/nb/annotations/ServiceProcessor.java

@@ -36,7 +36,7 @@ import java.util.stream.Collectors;
  * <pre>classes/META-INF/services/servicename</pre> file for each
  * implemented and annotated service name.
  */
-@SupportedSourceVersion(SourceVersion.RELEASE_24)
+@SupportedSourceVersion(SourceVersion.RELEASE_25)
 public class ServiceProcessor extends AbstractProcessor {
     public final static String SERVICE_NAME = Service.class.getCanonicalName();
 

nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/api/processors/FunctionDocInfoProcessor.java

@@ -41,7 +41,7 @@ import static io.nosqlbench.virtdata.api.processors.ProcessorClassNames.ThreadSa
  * manifests. It simply calls listener interfaces to do the rest of the work.
  */
 @SupportedOptions({"title"})
-@SupportedSourceVersion(SourceVersion.RELEASE_24)
+@SupportedSourceVersion(SourceVersion.RELEASE_25)
 @SupportedAnnotationTypes({
         ThreadSafeMapper,
         PerThreadMapper
@@ -117,25 +117,19 @@ public class FunctionDocInfoProcessor extends AbstractProcessor {
             }
             // apply method types
 
-            boolean foundApply=false;
-            Element applyMethodElem = null;
-            Element applyInClassElem = classElem;
-            while (applyMethodElem==null && applyInClassElem!=null) {
-                for (Element candidateApplyElem : applyInClassElem.getEnclosedElements()) {
-                    if (candidateApplyElem.getKind() == ElementKind.METHOD) {
-                        if (candidateApplyElem.getSimpleName().toString().startsWith("apply")) {
-                            applyMethodElem = candidateApplyElem;
-                            break;
-                        }
+            // Build class hierarchy starting with current class
+            List<TypeElement> classHierarchy = getClassHierarchy((TypeElement) classElem);
 
-                    }
-                }
-                if (applyMethodElem!=null) {
+            // Search for apply method in class hierarchy
+            Element applyMethodElem = null;
+            for (TypeElement typeInHierarchy : classHierarchy) {
+                applyMethodElem = findApplyMethodInType(typeInHierarchy);
+                if (applyMethodElem != null) {
                     break;
                 }
-                applyInClassElem = elementUtils.getTypeElement(((TypeElement) applyInClassElem).getSuperclass().toString());
             }
-            if (applyMethodElem==null) {
+
+            if (applyMethodElem == null) {
                 messenger.printMessage(Diagnostic.Kind.ERROR, "Unable to enumerate input and output types for " + simpleClassName);
                 return false;
             }
@@ -226,6 +220,45 @@ public class FunctionDocInfoProcessor extends AbstractProcessor {
         return ctorDoc.replaceAll("(?m)^ ", "");
     }
 
+    /**
+     * Build an ordered list of types in the class hierarchy, starting with the given type
+     * and walking up to its parent classes (excluding Object).
+     */
+    private List<TypeElement> getClassHierarchy(TypeElement startType) {
+        List<TypeElement> hierarchy = new ArrayList<>();
+        TypeElement current = startType;
+
+        while (current != null) {
+            hierarchy.add(current);
+
+            // Get the superclass
+            if (current.getSuperclass() != null &&
+                !current.getSuperclass().toString().equals("java.lang.Object") &&
+                !current.getSuperclass().toString().equals("none")) {
+                current = elementUtils.getTypeElement(current.getSuperclass().toString());
+            } else {
+                current = null;
+            }
+        }
+
+        return hierarchy;
+    }
+
+    /**
+     * Find an apply method in the given type element.
+     * Returns the first method whose name starts with "apply", or null if none found.
+     */
+    private Element findApplyMethodInType(TypeElement type) {
+        for (Element element : type.getEnclosedElements()) {
+            if (element.getKind() == ElementKind.METHOD) {
+                if (element.getSimpleName().toString().startsWith("apply")) {
+                    return element;
+                }
+            }
+        }
+        return null;
+    }
+
     private static class StdoutListener implements FuncEnumerator.Listener {
         @Override
         public void onFunctionModel(DocForFunc functionDoc) {

nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/api/processors/FunctionManifestProcessor.java

@@ -40,7 +40,7 @@ import static io.nosqlbench.virtdata.api.processors.ProcessorClassNames.ThreadSa
  * enumerate candidate functions without requiring them to have a no-args constructor.
  */
 @SupportedOptions({"title"})
-@SupportedSourceVersion(SourceVersion.RELEASE_24)
+@SupportedSourceVersion(SourceVersion.RELEASE_25)
 @SupportedAnnotationTypes({
         ThreadSafeMapper,
         PerThreadMapper

nb-virtdata/virtdata-lang/pom.xml

@@ -43,7 +43,7 @@
             <plugin>
                 <groupId>org.antlr</groupId>
                 <artifactId>antlr4-maven-plugin</artifactId>
-                <version>4.13.1</version>
+                <version>4.13.2</version>
                 <configuration>
                     <sourceDirectory>src/main/java/io/nosqlbench/virtdata/lang/grammars</sourceDirectory>
                     <arguments>

nb-virtdata/virtdata-lib-vectors/pom.xml

@@ -47,6 +47,12 @@
             <version>${revision}</version>
         </dependency>
 
+        <dependency>
+            <groupId>io.nosqlbench</groupId>
+            <artifactId>vectordata</artifactId>
+            <version>0.1.8</version>
+        </dependency>
+
 <!--        <dependency>-->
 <!--            <groupId>org.matheclipse</groupId>-->
 <!--            <artifactId>matheclipse-core</artifactId>-->

nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/BaseVectors.java

@@ -0,0 +1,54 @@
+package io.nosqlbench.virtdata.lib.vectors.vectordata;
+
+/*
+ * Copyright (c) nosqlbench
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import io.nosqlbench.vectordata.spec.datasets.types.FloatVectors;
+import io.nosqlbench.virtdata.api.annotations.Categories;
+import io.nosqlbench.virtdata.api.annotations.Category;
+import io.nosqlbench.virtdata.api.annotations.Example;
+import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
+
+/*
+* Random access to vector data from a hosted location. */
+@ThreadSafeMapper
+@Categories(Category.vectors)
+public class BaseVectors extends CoreVectors<float[]> {
+
+    @Example({"BaseVectors('exampledataset:exampleprofile')",
+        "Find and download vectordata for base vectors, prebuffering (by default) before resuming"})
+    public BaseVectors(String datasetAndProfile) {
+        super(datasetAndProfile,true);
+    }
+
+    @Example({"BaseVectors('exampledataset:exampleprofile',false)",
+    "Find and download vectordata for base vectors, with demand-paged access"})
+    public BaseVectors(String datasetAndProfile, boolean prebuffer) {
+        super (datasetAndProfile, prebuffer);
+    }
+
+    @Override
+    protected FloatVectors getRandomAccessData() {
+        return super.tdv.getBaseVectors().orElseThrow(()->new RuntimeException("Cannot get base vectors from data view " + super.tdv.getName()));
+    }
+
+    @Override
+    public float[] apply(long value) {
+        return super.apply(value);
+    }
+}

nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/CoreVectors.java

@@ -0,0 +1,57 @@
+package io.nosqlbench.virtdata.lib.vectors.vectordata;
+
+/*
+ * Copyright (c) nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import io.nosqlbench.nbdatatools.api.concurrent.ProgressIndicatingFuture;
+import io.nosqlbench.vectordata.VectorTestData;
+import io.nosqlbench.vectordata.discovery.TestDataView;
+import io.nosqlbench.vectordata.downloader.Catalog;
+import io.nosqlbench.vectordata.downloader.DatasetEntry;
+import io.nosqlbench.vectordata.spec.datasets.types.DatasetView;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.LongFunction;
+
+public abstract class CoreVectors<T> implements LongFunction<T> {
+
+    protected final TestDataView tdv;
+    protected final DatasetView<T> dataset;
+
+    public CoreVectors(String datasetAndProfile, boolean prebuffer) {
+        Catalog catalog = VectorTestData.catalogs().configure().catalog();
+        DatasetEntry datasetEntry = catalog.findExact(datasetAndProfile)
+            .orElseThrow(() -> new RuntimeException("Cannot find dataset and/or profile '" + datasetAndProfile + "'"));
+        tdv = datasetEntry.select().profile(datasetAndProfile);
+        dataset = getRandomAccessData();
+
+        if (prebuffer) {
+            CompletableFuture<Void> pbfuture = dataset.prebuffer();
+            if (pbfuture instanceof ProgressIndicatingFuture<Void> indicator) {
+                    indicator.monitorProgress(1000);
+            }
+        }
+    }
+
+    protected abstract DatasetView<T> getRandomAccessData();
+
+    @Override
+    public T apply(long value) {
+        return dataset.get(value);
+    }
+
+}

nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/NeighborDistances.java

@@ -0,0 +1,51 @@
+package io.nosqlbench.virtdata.lib.vectors.vectordata;
+
+/*
+ * Copyright (c) nosqlbench
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import io.nosqlbench.vectordata.spec.datasets.types.FloatVectors;
+import io.nosqlbench.virtdata.api.annotations.Categories;
+import io.nosqlbench.virtdata.api.annotations.Category;
+import io.nosqlbench.virtdata.api.annotations.Example;
+import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
+
+@ThreadSafeMapper
+@Categories(Category.vectors)
+public class NeighborDistances extends CoreVectors<float[]> {
+
+    @Example({"NeighborDistances('exampledataset:exampleprofile',false)",
+        "Find and download vectordata for neighbor distances, prebuffering (by default) before resuming"})
+    public NeighborDistances(String datasetAndProfile) {
+        super(datasetAndProfile, true);
+    }
+    @Example({"NeighborDistances('exampledataset:exampleprofile',false)",
+        "Find and download vectordata for neighbor distances, with demand-paged access"})
+    public NeighborDistances(String datasetAndProfile, boolean prebuffer) {
+        super (datasetAndProfile, prebuffer);
+    }
+
+    @Override
+    protected FloatVectors getRandomAccessData() {
+        return super.tdv.getNeighborDistances().orElseThrow(()->new RuntimeException("Cannot get neighbor distances from data view " + super.tdv.getName()));
+    }
+
+    @Override
+    public float[] apply(long value) {
+        return super.apply(value);
+    }
+}

nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/NeighborIndices.java

@@ -0,0 +1,52 @@
+package io.nosqlbench.virtdata.lib.vectors.vectordata;
+
+/*
+ * Copyright (c) nosqlbench
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import io.nosqlbench.vectordata.spec.datasets.types.FloatVectors;
+import io.nosqlbench.vectordata.spec.datasets.types.IntVectors;
+import io.nosqlbench.virtdata.api.annotations.Categories;
+import io.nosqlbench.virtdata.api.annotations.Category;
+import io.nosqlbench.virtdata.api.annotations.Example;
+import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
+
+@ThreadSafeMapper
+@Categories(Category.vectors)
+public class NeighborIndices extends CoreVectors<int[]> {
+
+    @Example({"NeighborIndices('exampledataset:exampleprofile',false)",
+        "Find and download vectordata for neighbor indices, prebuffering (by default) before resuming"})
+    public NeighborIndices(String datasetAndProfile) {
+        super(datasetAndProfile, true);
+    }
+    @Example({"NeighborIndices('exampledataset:exampleprofile',false)",
+        "Find and download vectordata for neighbor indices, with demand-paged access"})
+    public NeighborIndices(String datasetAndProfile, boolean prebuffer) {
+        super (datasetAndProfile, prebuffer);
+    }
+
+    @Override
+    protected IntVectors getRandomAccessData() {
+        return super.tdv.getNeighborIndices().orElseThrow(()->new RuntimeException("Cannot get neighbor indices from data view " + super.tdv.getName()));
+    }
+
+    @Override
+    public int[] apply(long value) {
+        return super.apply(value);
+    }
+}

nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/QueryVectors.java

@@ -0,0 +1,52 @@
+package io.nosqlbench.virtdata.lib.vectors.vectordata;
+
+/*
+ * Copyright (c) nosqlbench
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import io.nosqlbench.vectordata.spec.datasets.types.FloatVectors;
+import io.nosqlbench.virtdata.api.annotations.Categories;
+import io.nosqlbench.virtdata.api.annotations.Category;
+import io.nosqlbench.virtdata.api.annotations.Example;
+import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
+
+@ThreadSafeMapper
+@Categories(Category.vectors)
+public class QueryVectors extends CoreVectors<float[]> {
+
+    @Example({"QueryVectors('exampledataset:exampleprofile',false)",
+        "Find and download vectordata for query vectors, prebuffering (by default) before resuming"})
+    public QueryVectors(String datasetAndProfile) {
+        super(datasetAndProfile, true);
+    }
+    @Example({"QueryVectors('exampledataset:exampleprofile',false)",
+        "Find and download vectordata for query vectors, with demand-paged access"})
+    public QueryVectors(String datasetAndProfile, boolean prebuffer) {
+        super (datasetAndProfile, prebuffer);
+    }
+
+
+    @Override
+    protected FloatVectors getRandomAccessData() {
+        return super.tdv.getQueryVectors().orElseThrow(()->new RuntimeException("Cannot get query vectors from data view " + super.tdv.getName()));
+    }
+
+    @Override
+    public float[] apply(long value) {
+        return super.apply(value);
+    }
+}

nb-virtdata/virtdata-lib-vectors/src/test/java/io/nosqlbench/virtdata/lib/vectors/vectordata/BaseVectorsTest.java

@@ -0,0 +1,37 @@
+package io.nosqlbench.virtdata.lib.vectors.vectordata;
+
+/*
+ * Copyright (c) nosqlbench
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class BaseVectorsTest {
+
+    @Test
+    @Disabled
+    public void testExampleDataset() {
+        BaseVectors bv1 = new BaseVectors("example1");
+        float[] v23 = bv1.apply(23L);
+        System.out.println("v23:" + Arrays.toString(v23));
+    }
+}