refactor(virtdata): migrate vector readers to nb-vectordata

nb-virtdata/virtdata-lib-vectors/pom.xml

@@ -55,8 +55,8 @@
 
         <dependency>
             <groupId>io.nosqlbench</groupId>
-            <artifactId>datatools-vectordata</artifactId>
-            <version>0.1.25</version>
+            <artifactId>nb-vectordata</artifactId>
+            <version>${revision}</version>
         </dependency>
 
         <!--        <dependency>-->

nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java

@@ -22,9 +22,10 @@ import io.nosqlbench.nb.annotations.Service;
 import io.nosqlbench.nb.api.expr.ExprFunctionProvider;
 import io.nosqlbench.nb.api.expr.annotations.ExprExample;
 import io.nosqlbench.nb.api.expr.annotations.ExprFunctionSpec;
-import io.nosqlbench.vectordata.VectorTestData;
-import io.nosqlbench.vectordata.discovery.vector.TestDataView;
-import io.nosqlbench.vectordata.spec.datasets.types.*;
+import io.nosqlbench.vectordata.Catalog;
+import io.nosqlbench.vectordata.CatalogSources;
+import io.nosqlbench.vectordata.TestDataView;
+import io.nosqlbench.vectordata.VectorReader;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -43,7 +44,7 @@ public class VectorDataExprs implements ExprFunctionProvider {
         if (!datasetNameAndProfile.contains(":")) {
             logger.warn("datasetNameAndProfile missing profile:" + datasetNameAndProfile);
         }
-        return VectorTestData.catalogs().configure().catalog().profile(datasetNameAndProfile);
+        return Catalog.of(CatalogSources.defaults()).openProfile(datasetNameAndProfile);
     }
     @ExprExample(args = {"\"airports:demo\""}, expectNotNull = true)
     @ExprExample(args = {"\"airports:demo\""}, matches = ".+" )
@@ -52,9 +53,8 @@ public class VectorDataExprs implements ExprFunctionProvider {
         synopsis = "baseVectors(\"dataset:profile\")",
         description = "Return the BaseVectors associated with the dataset profile."
     )
-    public BaseVectors baseVectors(String datasetNameAndProfile) {
-        return dataset(datasetNameAndProfile).getBaseVectors().orElseThrow(() ->
-            new RuntimeException("Base vectors are not defined for dataset profile '" + datasetNameAndProfile + "'"));
+    public VectorReader<float[]> baseVectors(String datasetNameAndProfile) {
+        return dataset(datasetNameAndProfile).baseVectors();
     }
 
     @ExprExample(args = {"\"airports:demo\""}, expectNotNull = true)
@@ -64,9 +64,8 @@ public class VectorDataExprs implements ExprFunctionProvider {
         synopsis = "queryVectors(\"dataset:profile\")",
         description = "Return the QueryVectors associated with the dataset profile."
     )
-    public QueryVectors queryVectors(String datasetNameAndProfile) {
-        return dataset(datasetNameAndProfile).getQueryVectors().orElseThrow(() ->
-            new RuntimeException("Query vectors are not defined for dataset profile '" + datasetNameAndProfile + "'"));
+    public VectorReader<float[]> queryVectors(String datasetNameAndProfile) {
+        return dataset(datasetNameAndProfile).queryVectors();
     }
 
     @ExprExample(args = {"\"airports:demo\""}, expectNotNull = true)
@@ -76,9 +75,8 @@ public class VectorDataExprs implements ExprFunctionProvider {
         synopsis = "neighborDistances(\"dataset:profile\")",
         description = "Return the NeighborDistances associated with the dataset profile."
     )
-    public NeighborDistances neighborDistances(String datasetNameAndProfile) {
-        return dataset(datasetNameAndProfile).getNeighborDistances().orElseThrow(() ->
-            new RuntimeException("Neighbor distances are not defined for dataset profile '" + datasetNameAndProfile + "'"));
+    public VectorReader<float[]> neighborDistances(String datasetNameAndProfile) {
+        return dataset(datasetNameAndProfile).neighborDistances();
     }
 
     @ExprExample(args = {"\"airports:demo\""}, expectNotNull = true)
@@ -88,9 +86,8 @@ public class VectorDataExprs implements ExprFunctionProvider {
         synopsis = "neighborIndices(\"dataset:profile\")",
         description = "Return the NeighborIndices associated with the dataset profile."
     )
-    public NeighborIndices neighborIndices(String datasetNameAndProfile) {
-        return dataset(datasetNameAndProfile).getNeighborIndices().orElseThrow(() ->
-            new RuntimeException("Neighbor indices are not defined for dataset profile '" + datasetNameAndProfile + "'"));
+    public VectorReader<int[]> neighborIndices(String datasetNameAndProfile) {
+        return dataset(datasetNameAndProfile).neighborIndices();
     }
 
 }

nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/BaseVectors.java

@@ -18,7 +18,7 @@ package io.nosqlbench.virtdata.lib.vectors.vectordata;
  */
 
 
-import io.nosqlbench.vectordata.spec.datasets.types.FloatVectors;
+import io.nosqlbench.vectordata.VectorReader;
 import io.nosqlbench.virtdata.api.annotations.Categories;
 import io.nosqlbench.virtdata.api.annotations.Category;
 import io.nosqlbench.virtdata.api.annotations.Example;
@@ -43,8 +43,8 @@ public class BaseVectors extends CoreVectors<float[]> {
     }
 
     @Override
-    protected FloatVectors getRandomAccessData() {
-        return super.tdv.getBaseVectors().orElseThrow(()->new RuntimeException("Cannot get base vectors from data view " + super.tdv.getName()));
+    protected VectorReader<float[]> getRandomAccessData() {
+        return super.tdv.baseVectors();
     }
 
     @Override

nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/CoreVectors.java

@@ -18,35 +18,28 @@ package io.nosqlbench.virtdata.lib.vectors.vectordata;
  */
 
 
-import io.nosqlbench.nbdatatools.api.concurrent.ProgressIndicatingFuture;
-import io.nosqlbench.vectordata.VectorTestData;
-import io.nosqlbench.vectordata.discovery.ProfileSelector;
-import io.nosqlbench.vectordata.discovery.vector.TestDataView;
-import io.nosqlbench.vectordata.downloader.Catalog;
-import io.nosqlbench.vectordata.downloader.DatasetEntry;
-import io.nosqlbench.vectordata.spec.datasets.types.DatasetView;
-import java.util.concurrent.CompletableFuture;
+import io.nosqlbench.vectordata.Catalog;
+import io.nosqlbench.vectordata.CatalogSources;
+import io.nosqlbench.vectordata.TestDataView;
+import io.nosqlbench.vectordata.VectorReader;
 import java.util.function.LongFunction;
 
 public abstract class CoreVectors<T> implements LongFunction<T> {
 
     protected final TestDataView tdv;
-    protected final DatasetView<T> dataset;
+    protected final VectorReader<T> dataset;
 
     public CoreVectors(String datasetAndProfile, boolean prebuffer) {
-        Catalog catalog = VectorTestData.catalogs().configure().catalog();
-        tdv = catalog.profile(datasetAndProfile);
+        Catalog catalog = Catalog.of(CatalogSources.defaults());
+        tdv = catalog.openProfile(datasetAndProfile);
         dataset = getRandomAccessData();
 
         if (prebuffer) {
-            CompletableFuture<Void> pbfuture = dataset.prebuffer();
-            if (pbfuture instanceof ProgressIndicatingFuture<Void> indicator) {
-                    indicator.monitorProgress(1000);
-            }
+            dataset.prebuffer();
         }
     }
 
-    protected abstract DatasetView<T> getRandomAccessData();
+    protected abstract VectorReader<T> getRandomAccessData();
 
     @Override
     public T apply(long value) {

nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/NeighborDistances.java

@@ -18,7 +18,7 @@ package io.nosqlbench.virtdata.lib.vectors.vectordata;
  */
 
 
-import io.nosqlbench.vectordata.spec.datasets.types.FloatVectors;
+import io.nosqlbench.vectordata.VectorReader;
 import io.nosqlbench.virtdata.api.annotations.Categories;
 import io.nosqlbench.virtdata.api.annotations.Category;
 import io.nosqlbench.virtdata.api.annotations.Example;
@@ -40,8 +40,8 @@ public class NeighborDistances extends CoreVectors<float[]> {
     }
 
     @Override
-    protected FloatVectors getRandomAccessData() {
-        return super.tdv.getNeighborDistances().orElseThrow(()->new RuntimeException("Cannot get neighbor distances from data view " + super.tdv.getName()));
+    protected VectorReader<float[]> getRandomAccessData() {
+        return super.tdv.neighborDistances();
     }
 
     @Override

nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/NeighborIndices.java

@@ -18,8 +18,7 @@ package io.nosqlbench.virtdata.lib.vectors.vectordata;
  */
 
 
-import io.nosqlbench.vectordata.spec.datasets.types.FloatVectors;
-import io.nosqlbench.vectordata.spec.datasets.types.IntVectors;
+import io.nosqlbench.vectordata.VectorReader;
 import io.nosqlbench.virtdata.api.annotations.Categories;
 import io.nosqlbench.virtdata.api.annotations.Category;
 import io.nosqlbench.virtdata.api.annotations.Example;
@@ -41,8 +40,8 @@ public class NeighborIndices extends CoreVectors<int[]> {
     }
 
     @Override
-    protected IntVectors getRandomAccessData() {
-        return super.tdv.getNeighborIndices().orElseThrow(()->new RuntimeException("Cannot get neighbor indices from data view " + super.tdv.getName()));
+    protected VectorReader<int[]> getRandomAccessData() {
+        return super.tdv.neighborIndices();
     }
 
     @Override

nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/QueryVectors.java

@@ -18,7 +18,7 @@ package io.nosqlbench.virtdata.lib.vectors.vectordata;
  */
 
 
-import io.nosqlbench.vectordata.spec.datasets.types.FloatVectors;
+import io.nosqlbench.vectordata.VectorReader;
 import io.nosqlbench.virtdata.api.annotations.Categories;
 import io.nosqlbench.virtdata.api.annotations.Category;
 import io.nosqlbench.virtdata.api.annotations.Example;
@@ -41,8 +41,8 @@ public class QueryVectors extends CoreVectors<float[]> {
 
 
     @Override
-    protected FloatVectors getRandomAccessData() {
-        return super.tdv.getQueryVectors().orElseThrow(()->new RuntimeException("Cannot get query vectors from data view " + super.tdv.getName()));
+    protected VectorReader<float[]> getRandomAccessData() {
+        return super.tdv.queryVectors();
     }
 
     @Override

nb-virtdata/virtdata-lib-vectors/src/test/java/io/nosqlbench/virtdata/lib/vectors/vectordata/BaseVectorsTest.java

@@ -18,22 +18,47 @@ package io.nosqlbench.virtdata.lib.vectors.vectordata;
  */
 
 
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.io.TempDir;
 
-import java.util.Arrays;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.file.Files;
+import java.nio.file.Path;
 
 import static org.junit.jupiter.api.Assertions.*;
 
 @Tag("unit")
 public class BaseVectorsTest {
 
+    @TempDir Path temporary;
+
     @Test
-    @Disabled
-    public void testExampleDataset() {
-        BaseVectors bv1 = new BaseVectors("example1");
-        float[] v23 = bv1.apply(23L);
-        System.out.println("v23:" + Arrays.toString(v23));
+    public void testFixtureDataset() throws Exception {
+        Path dataset = Files.createDirectories(temporary.resolve("example"));
+        ByteBuffer values = ByteBuffer.allocate(24).order(ByteOrder.LITTLE_ENDIAN);
+        values.putInt(2).putFloat(1f).putFloat(2f).putInt(2).putFloat(3f).putFloat(4f);
+        Files.write(dataset.resolve("base.fvec"), values.array());
+        Files.writeString(dataset.resolve("dataset.yaml"), """
+            name: example
+            profiles:
+              demo:
+                base: base.fvec
+            """);
+        Path catalog = temporary.resolve("catalog.yaml");
+        Files.writeString(catalog, """
+            datasets:
+              - name: example
+                path: example/dataset.yaml
+            """);
+        String prior = System.getProperty("vectordata.catalog");
+        System.setProperty("vectordata.catalog", catalog.toString());
+        try {
+            BaseVectors vectors = new BaseVectors("example:demo", true);
+            assertArrayEquals(new float[] {3f, 4f}, vectors.apply(1));
+        } finally {
+            if (prior == null) System.clearProperty("vectordata.catalog"); else System.setProperty("vectordata.catalog", prior);
+        }
     }
 }