feat(virtdata): survey a ground truth's attainable recall before anything is loaded

groundTruthCoverage reads the ground-truth facet alone and reports how many queries have all k neighbors, how many have none, and the recall@k ceiling a perfect search would score. requireAttainableRecall refuses a run below a threshold before a million rows are loaded for a number that could never be read, naming the ceiling and the profiles that would read. On tessera, 100k mixed ceils at 0.1428, 300k-uniform-2-1e-3 at 0.988, and 1m-uniform-2-1e-3 at 1.0.

nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java

@@ -37,6 +37,7 @@ import io.nosqlbench.vectordata.anode.MNode;
 import io.nosqlbench.vectordata.binding.Layout;
 import io.nosqlbench.vectordata.records.RecordFacet;
 import io.nosqlbench.virtdata.lib.vectors.vectordata.CqlColumns;
+import io.nosqlbench.virtdata.lib.vectors.vectordata.GroundTruthCoverage;
 import io.nosqlbench.virtdata.lib.vectors.vectordata.PredicateClause;
 import io.nosqlbench.virtdata.lib.vectors.vectordata.PrefetchMeter;
 import io.nosqlbench.virtdata.lib.vectors.vectordata.WindowedReader;
@@ -280,6 +281,30 @@ public class VectorDataExprs implements ExprFunctionProvider {
         return CqlColumns.columns(layout, sample);
     }
 
+    @ExprExample(args = {"\"airports:demo\"", "\"neighbor_indices\""}, matches = ".+")
+    @ExprFunctionSpec(
+        name = "groundTruthCoverage",
+        synopsis = "groundTruthCoverage(\"dataset:profile\", \"ground_truth_facet\")",
+        description = "Survey a ground-truth facet before anything is loaded: how many of its queries have all k neighbors, how many have none, and the recall@k ceiling a perfect search would score under the strict recall measure — a negative entry is a sentinel for a neighbor that does not exist, which a filtered ground truth over a small slice carries whenever a predicate matches fewer than k rows. Fields: queries, k, fullRows, emptyRows, ceiling, isComplete."
+    )
+    public GroundTruthCoverage groundTruthCoverage(String datasetNameAndProfile, String groundTruthFacet) {
+        return GroundTruthCoverage.of(datasetNameAndProfile + ":" + groundTruthFacet, dataset(datasetNameAndProfile).openFacet(groundTruthFacet));
+    }
+
+    @ExprExample(args = {"\"airports:demo\"", "\"neighbor_indices\"", "0.95"}, matches = ".+")
+    @ExprExample(args = {"\"airports:demo\"", "\"neighbor_indices\"", "\"0\""}, matches = ".+")
+    @ExprFunctionSpec(
+        name = "requireAttainableRecall",
+        synopsis = "requireAttainableRecall(\"dataset:profile\", \"ground_truth_facet\", minimum)",
+        description = "The coverage of groundTruthCoverage, refused when its recall@k ceiling is below minimum: a run whose strict recall could never be read is stopped before it loads anything, with a message naming the ceiling and the profiles that would read. Pass 0 to let a smoke run through and read recall_attainable instead."
+    )
+    public GroundTruthCoverage requireAttainableRecall(String datasetNameAndProfile, String groundTruthFacet, Object minimum) {
+        double floor = minimum instanceof Number number ? number.doubleValue() : Double.parseDouble(String.valueOf(minimum).trim());
+        GroundTruthCoverage coverage = groundTruthCoverage(datasetNameAndProfile, groundTruthFacet);
+        System.err.printf("[vectordata] %s%n", coverage);
+        return coverage.require(floor);
+    }
+
     @ExprExample(args = {"\"airports:demo\"", "\"metadata_predicates\"", "\"metadata_content\""}, matches = ".+")
     @ExprFunctionSpec(
         name = "predicateForms",

nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/GroundTruthCoverage.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.lib.vectors.vectordata;
+
+import io.nosqlbench.vectordata.VectorReader;
+
+/// What a ground-truth facet can pay out: for each query row, how many
+/// of its `k` entries name a neighbor that exists — a negative entry is
+/// a sentinel for one that does not, which a filtered search over a
+/// small slice leaves behind whenever a predicate matches fewer than
+/// `k` rows. `ceiling` is the recall@k a perfect search would score
+/// against this facet under the strict `recall` measure, so a run that
+/// reads below it has found something and a run that reads at it has
+/// found everything there was.
+///
+/// Computed once, before anything is loaded, from the facet alone: a
+/// few megabytes read, against the hours a load can cost. A regime
+/// that gates on it never mistakes a padded ground truth for a search
+/// that missed.
+public record GroundTruthCoverage(String facet, long queries, int k, long fullRows, long emptyRows, long attainableEntries, double ceiling) {
+
+    /// Surveys a ground-truth reader whose rows are `int[]` of `k`
+    /// neighbor ordinals, negative for a sentinel.
+    public static GroundTruthCoverage of(String facet, VectorReader<?> reader) {
+        long queries = reader.count();
+        int k = reader.dimension();
+        long full = 0, empty = 0, attainable = 0;
+        for (long ordinal = 0; ordinal < queries; ordinal++) {
+            Object row = reader.get(ordinal);
+            if (!(row instanceof int[] entries))
+                throw new IllegalArgumentException("ground truth facet '" + facet + "' holds " + (row == null ? "null" : row.getClass().getSimpleName()) + " rows, not int[]");
+            int present = 0;
+            for (int entry : entries) if (entry >= 0) present++;
+            present = Math.min(present, k);
+            attainable += present;
+            if (present >= k) full++;
+            if (present == 0) empty++;
+        }
+        double ceiling = queries == 0 || k == 0 ? 1.0 : (double) attainable / ((double) queries * k);
+        return new GroundTruthCoverage(facet, queries, k, full, empty, attainable, ceiling);
+    }
+
+    /// Whether every query can be answered in full: no row is padded.
+    public boolean isComplete() { return fullRows == queries; }
+
+    /// Refuses a ceiling below `minimum`, naming what the facet holds
+    /// and what to do about it, so a run whose recall could never be
+    /// read is stopped before it loads anything.
+    public GroundTruthCoverage require(double minimum) {
+        if (ceiling >= minimum) return this;
+        throw new IllegalStateException(String.format(
+            "%s: attainable recall@%d ceiling is %.4f, below the required %.2f — %d of %d queries have all %d neighbors, %d have none; "
+            + "a strict recall reading on this profile cannot rise above the ceiling. Choose a profile whose predicates match at least %d rows "
+            + "(a larger size, or a uniform predicate set with selectivity x base_count >= %d), or lower min_attainable for a smoke run "
+            + "and read recall_attainable instead.",
+            facet, k, ceiling, minimum, fullRows, queries, k, emptyRows, k, k));
+    }
+
+    @Override public String toString() {
+        return String.format("%s: %d queries, %d with all %d neighbors, %d with none; attainable recall@%d ceiling %.4f",
+            facet, queries, fullRows, k, emptyRows, k, ceiling);
+    }
+}

nb-virtdata/virtdata-lib-vectors/src/test/java/io/nosqlbench/virtdata/lib/vectors/vectordata/GroundTruthCoverageTest.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) nosqlbench
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.nosqlbench.virtdata.lib.vectors.vectordata;
+
+import io.nosqlbench.vectordata.CacheStats;
+import io.nosqlbench.vectordata.ElementType;
+import io.nosqlbench.vectordata.PrebufferProgress;
+import io.nosqlbench.vectordata.VectorReader;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/// The ceiling a padded ground truth imposes on strict recall, read
+/// from the facet before anything is loaded.
+@Tag("unit")
+class GroundTruthCoverageTest {
+
+    /// A ground-truth reader over fixed rows.
+    private static VectorReader<int[]> rows(List<int[]> rows) {
+        return new VectorReader<>() {
+            @Override public long count() { return rows.size(); }
+            @Override public int dimension() { return rows.get(0).length; }
+            @Override public int[] get(long ordinal) { return rows.get((int) ordinal); }
+            @Override public void prebuffer(PrebufferProgress progress) { }
+            @Override public boolean isComplete() { return true; }
+            @Override public CacheStats cacheStats() { return null; }
+            @Override public ElementType elementType() { return ElementType.I32; }
+        };
+    }
+
+    @Test void aFullFacetHasACeilingOfOne() {
+        GroundTruthCoverage coverage = GroundTruthCoverage.of("gt", rows(List.of(new int[] {1, 2, 3, 4}, new int[] {5, 6, 7, 8})));
+        assertEquals(1.0, coverage.ceiling(), 1e-9);
+        assertTrue(coverage.isComplete());
+        assertEquals(2, coverage.fullRows());
+        assertEquals(0, coverage.emptyRows());
+        assertSame(coverage, coverage.require(0.95), "a complete facet passes any gate up to one");
+    }
+
+    @Test void sentinelsLowerTheCeilingByTheNeighborsTheyStandFor() {
+        // 4 + 2 + 0 + 4 attainable of 16: half the neighbors exist.
+        GroundTruthCoverage coverage = GroundTruthCoverage.of("gt", rows(List.of(
+            new int[] {1, 2, 3, 4}, new int[] {5, 6, -1, -1}, new int[] {-1, -1, -1, -1}, new int[] {9, 10, 11, 12})));
+        assertEquals(0.625, coverage.ceiling(), 1e-9);
+        assertEquals(2, coverage.fullRows());
+        assertEquals(1, coverage.emptyRows());
+        assertEquals(10, coverage.attainableEntries());
+        assertFalse(coverage.isComplete());
+        IllegalStateException refused = assertThrows(IllegalStateException.class, () -> coverage.require(0.95));
+        assertTrue(refused.getMessage().contains("0.6250"), refused.getMessage());
+        assertTrue(refused.getMessage().contains("2 of 4 queries"), refused.getMessage());
+        assertTrue(refused.getMessage().contains("recall_attainable"), "names the measure that reads through padding: " + refused.getMessage());
+        assertSame(coverage, coverage.require(0.5), "a lowered gate lets a smoke run through");
+        assertTrue(coverage.toString().contains("ceiling 0.6250"), coverage.toString());
+    }
+}