From 6ae106a5eb67f2aec131eed4ac12277cd0ca7708 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 16 Sep 2026 07:04:49 +0000 Subject: [PATCH] feat(virtdata): survey a ground truth's attainable recall before anything is loaded groundTruthCoverage reads the ground-truth facet alone and reports how many queries have all k neighbors, how many have none, and the recall@k ceiling a perfect search would score. requireAttainableRecall refuses a run below a threshold before a million rows are loaded for a number that could never be read, naming the ceiling and the profiles that would read. On tessera, 100k mixed ceils at 0.1428, 300k-uniform-2-1e-3 at 0.988, and 1m-uniform-2-1e-3 at 1.0. --- .../exprs/lib/vectors/VectorDataExprs.java | 25 ++++++ .../vectordata/GroundTruthCoverage.java | 77 +++++++++++++++++++ .../vectordata/GroundTruthCoverageTest.java | 73 ++++++++++++++++++ 3 files changed, 175 insertions(+) create mode 100644 nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/GroundTruthCoverage.java create mode 100644 nb-virtdata/virtdata-lib-vectors/src/test/java/io/nosqlbench/virtdata/lib/vectors/vectordata/GroundTruthCoverageTest.java diff --git a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java index f8311dbad..5544ae311 100644 --- a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java +++ b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/exprs/lib/vectors/VectorDataExprs.java @@ -37,6 +37,7 @@ import io.nosqlbench.vectordata.anode.MNode; import io.nosqlbench.vectordata.binding.Layout; import io.nosqlbench.vectordata.records.RecordFacet; import io.nosqlbench.virtdata.lib.vectors.vectordata.CqlColumns; +import io.nosqlbench.virtdata.lib.vectors.vectordata.GroundTruthCoverage; import io.nosqlbench.virtdata.lib.vectors.vectordata.PredicateClause; import io.nosqlbench.virtdata.lib.vectors.vectordata.PrefetchMeter; import io.nosqlbench.virtdata.lib.vectors.vectordata.WindowedReader; @@ -280,6 +281,30 @@ public class VectorDataExprs implements ExprFunctionProvider { return CqlColumns.columns(layout, sample); } + @ExprExample(args = {"\"airports:demo\"", "\"neighbor_indices\""}, matches = ".+") + @ExprFunctionSpec( + name = "groundTruthCoverage", + synopsis = "groundTruthCoverage(\"dataset:profile\", \"ground_truth_facet\")", + description = "Survey a ground-truth facet before anything is loaded: how many of its queries have all k neighbors, how many have none, and the recall@k ceiling a perfect search would score under the strict recall measure — a negative entry is a sentinel for a neighbor that does not exist, which a filtered ground truth over a small slice carries whenever a predicate matches fewer than k rows. Fields: queries, k, fullRows, emptyRows, ceiling, isComplete." + ) + public GroundTruthCoverage groundTruthCoverage(String datasetNameAndProfile, String groundTruthFacet) { + return GroundTruthCoverage.of(datasetNameAndProfile + ":" + groundTruthFacet, dataset(datasetNameAndProfile).openFacet(groundTruthFacet)); + } + + @ExprExample(args = {"\"airports:demo\"", "\"neighbor_indices\"", "0.95"}, matches = ".+") + @ExprExample(args = {"\"airports:demo\"", "\"neighbor_indices\"", "\"0\""}, matches = ".+") + @ExprFunctionSpec( + name = "requireAttainableRecall", + synopsis = "requireAttainableRecall(\"dataset:profile\", \"ground_truth_facet\", minimum)", + description = "The coverage of groundTruthCoverage, refused when its recall@k ceiling is below minimum: a run whose strict recall could never be read is stopped before it loads anything, with a message naming the ceiling and the profiles that would read. Pass 0 to let a smoke run through and read recall_attainable instead." + ) + public GroundTruthCoverage requireAttainableRecall(String datasetNameAndProfile, String groundTruthFacet, Object minimum) { + double floor = minimum instanceof Number number ? number.doubleValue() : Double.parseDouble(String.valueOf(minimum).trim()); + GroundTruthCoverage coverage = groundTruthCoverage(datasetNameAndProfile, groundTruthFacet); + System.err.printf("[vectordata] %s%n", coverage); + return coverage.require(floor); + } + @ExprExample(args = {"\"airports:demo\"", "\"metadata_predicates\"", "\"metadata_content\""}, matches = ".+") @ExprFunctionSpec( name = "predicateForms", diff --git a/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/GroundTruthCoverage.java b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/GroundTruthCoverage.java new file mode 100644 index 000000000..f062c0a7e --- /dev/null +++ b/nb-virtdata/virtdata-lib-vectors/src/main/java/io/nosqlbench/virtdata/lib/vectors/vectordata/GroundTruthCoverage.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.virtdata.lib.vectors.vectordata; + +import io.nosqlbench.vectordata.VectorReader; + +/// What a ground-truth facet can pay out: for each query row, how many +/// of its `k` entries name a neighbor that exists — a negative entry is +/// a sentinel for one that does not, which a filtered search over a +/// small slice leaves behind whenever a predicate matches fewer than +/// `k` rows. `ceiling` is the recall@k a perfect search would score +/// against this facet under the strict `recall` measure, so a run that +/// reads below it has found something and a run that reads at it has +/// found everything there was. +/// +/// Computed once, before anything is loaded, from the facet alone: a +/// few megabytes read, against the hours a load can cost. A regime +/// that gates on it never mistakes a padded ground truth for a search +/// that missed. +public record GroundTruthCoverage(String facet, long queries, int k, long fullRows, long emptyRows, long attainableEntries, double ceiling) { + + /// Surveys a ground-truth reader whose rows are `int[]` of `k` + /// neighbor ordinals, negative for a sentinel. + public static GroundTruthCoverage of(String facet, VectorReader reader) { + long queries = reader.count(); + int k = reader.dimension(); + long full = 0, empty = 0, attainable = 0; + for (long ordinal = 0; ordinal < queries; ordinal++) { + Object row = reader.get(ordinal); + if (!(row instanceof int[] entries)) + throw new IllegalArgumentException("ground truth facet '" + facet + "' holds " + (row == null ? "null" : row.getClass().getSimpleName()) + " rows, not int[]"); + int present = 0; + for (int entry : entries) if (entry >= 0) present++; + present = Math.min(present, k); + attainable += present; + if (present >= k) full++; + if (present == 0) empty++; + } + double ceiling = queries == 0 || k == 0 ? 1.0 : (double) attainable / ((double) queries * k); + return new GroundTruthCoverage(facet, queries, k, full, empty, attainable, ceiling); + } + + /// Whether every query can be answered in full: no row is padded. + public boolean isComplete() { return fullRows == queries; } + + /// Refuses a ceiling below `minimum`, naming what the facet holds + /// and what to do about it, so a run whose recall could never be + /// read is stopped before it loads anything. + public GroundTruthCoverage require(double minimum) { + if (ceiling >= minimum) return this; + throw new IllegalStateException(String.format( + "%s: attainable recall@%d ceiling is %.4f, below the required %.2f — %d of %d queries have all %d neighbors, %d have none; " + + "a strict recall reading on this profile cannot rise above the ceiling. Choose a profile whose predicates match at least %d rows " + + "(a larger size, or a uniform predicate set with selectivity x base_count >= %d), or lower min_attainable for a smoke run " + + "and read recall_attainable instead.", + facet, k, ceiling, minimum, fullRows, queries, k, emptyRows, k, k)); + } + + @Override public String toString() { + return String.format("%s: %d queries, %d with all %d neighbors, %d with none; attainable recall@%d ceiling %.4f", + facet, queries, fullRows, k, emptyRows, k, ceiling); + } +} diff --git a/nb-virtdata/virtdata-lib-vectors/src/test/java/io/nosqlbench/virtdata/lib/vectors/vectordata/GroundTruthCoverageTest.java b/nb-virtdata/virtdata-lib-vectors/src/test/java/io/nosqlbench/virtdata/lib/vectors/vectordata/GroundTruthCoverageTest.java new file mode 100644 index 000000000..612cb5353 --- /dev/null +++ b/nb-virtdata/virtdata-lib-vectors/src/test/java/io/nosqlbench/virtdata/lib/vectors/vectordata/GroundTruthCoverageTest.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.virtdata.lib.vectors.vectordata; + +import io.nosqlbench.vectordata.CacheStats; +import io.nosqlbench.vectordata.ElementType; +import io.nosqlbench.vectordata.PrebufferProgress; +import io.nosqlbench.vectordata.VectorReader; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/// The ceiling a padded ground truth imposes on strict recall, read +/// from the facet before anything is loaded. +@Tag("unit") +class GroundTruthCoverageTest { + + /// A ground-truth reader over fixed rows. + private static VectorReader rows(List rows) { + return new VectorReader<>() { + @Override public long count() { return rows.size(); } + @Override public int dimension() { return rows.get(0).length; } + @Override public int[] get(long ordinal) { return rows.get((int) ordinal); } + @Override public void prebuffer(PrebufferProgress progress) { } + @Override public boolean isComplete() { return true; } + @Override public CacheStats cacheStats() { return null; } + @Override public ElementType elementType() { return ElementType.I32; } + }; + } + + @Test void aFullFacetHasACeilingOfOne() { + GroundTruthCoverage coverage = GroundTruthCoverage.of("gt", rows(List.of(new int[] {1, 2, 3, 4}, new int[] {5, 6, 7, 8}))); + assertEquals(1.0, coverage.ceiling(), 1e-9); + assertTrue(coverage.isComplete()); + assertEquals(2, coverage.fullRows()); + assertEquals(0, coverage.emptyRows()); + assertSame(coverage, coverage.require(0.95), "a complete facet passes any gate up to one"); + } + + @Test void sentinelsLowerTheCeilingByTheNeighborsTheyStandFor() { + // 4 + 2 + 0 + 4 attainable of 16: half the neighbors exist. + GroundTruthCoverage coverage = GroundTruthCoverage.of("gt", rows(List.of( + new int[] {1, 2, 3, 4}, new int[] {5, 6, -1, -1}, new int[] {-1, -1, -1, -1}, new int[] {9, 10, 11, 12}))); + assertEquals(0.625, coverage.ceiling(), 1e-9); + assertEquals(2, coverage.fullRows()); + assertEquals(1, coverage.emptyRows()); + assertEquals(10, coverage.attainableEntries()); + assertFalse(coverage.isComplete()); + IllegalStateException refused = assertThrows(IllegalStateException.class, () -> coverage.require(0.95)); + assertTrue(refused.getMessage().contains("0.6250"), refused.getMessage()); + assertTrue(refused.getMessage().contains("2 of 4 queries"), refused.getMessage()); + assertTrue(refused.getMessage().contains("recall_attainable"), "names the measure that reads through padding: " + refused.getMessage()); + assertSame(coverage, coverage.require(0.5), "a lowered gate lets a smoke run through"); + assertTrue(coverage.toString().contains("ceiling 0.6250"), coverage.toString()); + } +}