feat(vectordata): select profiles by selector on the group and catalog surfaces

A group answers what a selector reads of every profile, structure as loaded after inheritance and attributes its own, in size order: default first, then by base_count or the name read as a count, then naturally by name. A catalog open takes a selector and refuses a set, openProfiles returns the set, openProfile splits a spec by its head's shape, and a catalog entry resolves the same selection from its listed layout before anything is fetched, exposing the listed format version.
This commit is contained in:
Jonathan Shook committed 2026-09-16 01:40:28 +00:00
1 parent 9f3efebe85
commit b6ddfcc575
5 files changed
+418 -9

No files matched your search

@@ -39,18 +39,46 @@ public final class Catalog {
return new Catalog(entries, settings);
}
public Map<String, CatalogEntry> entries() { return entries; }
public TestDataView open(String dataset, String profile) {
/// Loads a dataset's manifest as a group, without choosing a profile.
public TestDataGroup openGroup(String dataset) {
CatalogEntry entry = entries.get(dataset);
if (entry == null) throw new VectorDataException("Dataset is not in configured catalogs: " + dataset);
if ("knn_entries.yaml".equals(entry.datasetType())) {
@SuppressWarnings("unchecked") Map<String, Map<String, Object>> profiles = (Map<String, Map<String, Object>>) entry.attributes().get("profiles");
return TestDataGroup.fromLegacyEntries(dataset, entry.manifest(), profiles, settings).profile(profile);
return TestDataGroup.fromLegacyEntries(dataset, entry.manifest(), profiles, settings);
}
return TestDataGroup.load(entry.manifest(), settings).profile(profile);
return TestDataGroup.load(entry.manifest(), settings);
}
public TestDataView openProfile(String datasetAndProfile) {
int separator = datasetAndProfile.lastIndexOf(':');
return separator < 0 ? open(datasetAndProfile, null) : open(datasetAndProfile.substring(0, separator), datasetAndProfile.substring(separator + 1));
/// Opens the one profile of a dataset a selector names. `selector`
/// is a bare name as it always was, or an expression such as
/// `size=10m,predicates=uniform-2`; `null` opens `default`. It must
/// name exactly one profile: a set fails here, and [#openProfiles]
/// is the surface that takes one.
public TestDataView open(String dataset, String selector) {
TestDataGroup group = openGroup(dataset);
return group.profile(selectOne(group, dataset, selector));
}
/// Opens every profile of a dataset a selector names, size-ordered.
/// `null` opens `default`; `profile=*` opens them all.
public List<TestDataView> openProfiles(String dataset, String selector) {
TestDataGroup group = openGroup(dataset);
List<String> selected;
try { selected = group.select(selector); }
catch (SelectionException e) { throw e.inDataset(dataset); }
List<TestDataView> views = new ArrayList<>();
for (String profile : selected) views.add(group.profile(profile));
return views;
}
/// Opens the one profile a dataset spec names — `<dataset>` or
/// `<dataset>:<selector>`, split by the head's shape as
/// [DatasetSpec] describes, so a selector may carry a colon.
public TestDataView openProfile(String spec) {
DatasetSpec parsed = DatasetSpec.parse(spec);
return open(parsed.head(), parsed.selectorText());
}
private static String selectOne(TestDataGroup group, String dataset, String selector) {
try { return group.selectOne(selector); }
catch (SelectionException e) { throw e.inDataset(dataset); }
}
private static void load(URI source, VectorDataSettings settings, Map<String, CatalogEntry> output) {
if (!isCatalogFile(source)) { loadDirectory(source, settings, output); return; }
@@ -15,8 +15,130 @@
*/
package io.nosqlbench.vectordata;
import io.nosqlbench.vectordata.internal.YamlData;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/** A dataset entry resolved from a catalog source. */
public record CatalogEntry(String name, URI manifest, String datasetType, Map<String, Object> attributes) { }
/// A dataset entry resolved from a catalog source. `attributes` is the
/// entry as listed; a canonical entry's `layout` carries the dataset's
/// `format_version`, `profile_tags`, and declared `profiles`, which is
/// what a consumer reads **before fetching** — to refuse a dataset
/// above its version, or to resolve a selector against the profiles on
/// offer, without downloading the manifest.
public record CatalogEntry(String name, URI manifest, String datasetType, Map<String, Object> attributes) {
/// The entry's declared layout, or an empty map when it lists none.
public Map<String, Object> layout() {
return attributes.get("layout") instanceof Map<?, ?> layout ? YamlData.map(layout, "layout") : Map.of();
}
/// The `format_version` the listing states for the dataset, or
/// [FormatVersion#BASE] when it states none — absent means 1, like
/// everywhere else.
public int formatVersion() {
Object stated = layout().get("format_version");
return stated == null ? FormatVersion.BASE : YamlData.integer(stated, "format_version");
}
/// The listed profile tag schema, naming tags in order; empty when
/// none is listed.
public Map<String, Object> profileTags() {
return layout().get("profile_tags") instanceof Map<?, ?> tags ? Collections.unmodifiableMap(YamlData.map(tags, "profile_tags")) : Map.of();
}
/// The profiles the listing declares, by name, as declared.
public Map<String, Map<String, Object>> profiles() {
Object listed = layout().get("profiles");
if (listed == null) listed = attributes.get("profiles");
Map<String, Map<String, Object>> profiles = new LinkedHashMap<>();
if (listed instanceof Map<?, ?> map)
map.forEach((key, value) -> profiles.put(String.valueOf(key), value instanceof Map<?, ?> ? YamlData.map(value, "profile " + key) : Map.of()));
return profiles;
}
/// The listed profile names, size-ordered as [TestDataGroup#profileNames] orders them.
public List<String> profileNames() {
Map<String, Map<String, Object>> profiles = profiles();
List<String> names = new ArrayList<>(profiles.keySet());
names.sort((a, b) -> {
int bySize = Long.compare(TestDataGroup.sortKey(a, declaredStructure(profiles, a).baseCount()),
TestDataGroup.sortKey(b, declaredStructure(profiles, b).baseCount()));
return bySize != 0 ? bySize : TestDataGroup.naturalCompare(a, b);
});
return names;
}
/// Everything a selector can read of every listed profile, with
/// `base_count` and `maxk` read through the `inherits` chain the
/// way the loader resolves them, so a catalog entry selects the
/// same profiles a loaded dataset does. Attributes are the
/// profile's own, in declaration order.
public List<ProfileFacts> profileFacts() {
Map<String, Map<String, Object>> profiles = profiles();
List<ProfileFacts> facts = new ArrayList<>();
for (String name : profileNames()) {
Map<String, Object> profile = profiles.get(name);
Structure structure = declaredStructure(profiles, name);
Map<String, Object> own = profile.get("attributes") instanceof Map<?, ?> declared ? YamlData.map(declared, "attributes") : Map.of();
facts.add(new ProfileFacts(name, structure.baseCount(), structure.maxk(), partition(profile), inherits(profile), own));
}
return facts;
}
/// The profiles a selector names in this entry, size-ordered;
/// `null` is `default`. A catalog entry is what a consumer reads
/// before fetching, so a selector resolves here the same way it
/// does against the loaded dataset.
public List<String> select(String selector) { return ProfileSelector.resolve(selector, profileFacts()); }
/// The one profile a selector names in this entry.
public String selectOne(String selector) { return ProfileSelector.resolveOne(selector, profileFacts()); }
/// `base_count` and `maxk` of a declared profile as the loader would
/// see them: its own, else its parent's. `maxk` crosses any parent;
/// `base_count` crosses only a named parent other than `default`,
/// because inheriting from `default` is a size step and the child's
/// count is its own. A partition profile is self-contained.
private record Structure(Long baseCount, Integer maxk) { }
private static Structure declaredStructure(Map<String, Map<String, Object>> profiles, String name) {
Map<String, Object> own = profiles.get(name);
if (own == null) return new Structure(null, null);
Long baseCount = longOrNull(own.get("base_count"));
Integer maxk = own.get("maxk") == null ? null : YamlData.integer(own.get("maxk"), "maxk");
String current = name;
// A chain is at most one hop per profile; a cycle stops here.
for (int hops = 0; hops < profiles.size(); hops++) {
Map<String, Object> profile = profiles.get(current);
if (profile == null || partition(profile) || "default".equals(current)) break;
String stated = inherits(profile);
String parent = stated != null && !stated.equals(current) && profiles.containsKey(stated) ? stated : "default";
Map<String, Object> above = profiles.get(parent);
if (above == null) break;
if (maxk == null && above.get("maxk") != null) maxk = YamlData.integer(above.get("maxk"), "maxk");
if (!"default".equals(parent) && baseCount == null) baseCount = longOrNull(above.get("base_count"));
current = parent;
}
return new Structure(baseCount, maxk);
}
private static boolean partition(Map<String, Object> profile) {
Object partition = profile.get("partition");
return Boolean.TRUE.equals(partition) || "true".equalsIgnoreCase(String.valueOf(partition));
}
private static String inherits(Map<String, Object> profile) {
Object raw = profile.containsKey("inherits") ? profile.get("inherits") : profile.get("extends");
return raw == null ? null : String.valueOf(raw);
}
private static Long longOrNull(Object value) {
if (value == null) return null;
if (value instanceof Number number) return number.longValue();
try { return Long.parseLong(String.valueOf(value).trim()); } catch (NumberFormatException e) { return null; }
}
}
@@ -177,11 +177,45 @@ public final class TestDataGroup {
/// declares none.
public Map<String, Object> profileTags() { return profileTags; }
/// Opens a profile by its exact name. A `null` or blank name opens
/// `default`, or the first profile when there is none.
/// `default`, or the first profile when there is none. For a
/// selector expression see [#selectOne] and [#select].
public TestDataView profile(String profile) {
String selected = profile == null || profile.isBlank() ? (profiles.containsKey("default") ? "default" : profiles.keySet().iterator().next()) : profile;
return new ManifestView(name, selected, resolve(selected).facets(), settings, attributes);
}
/// Profile names size-ordered: `default` first, then ascending by
/// `base_count` — or by the name read as a count, `10m` before
/// `100m` — with a natural tiebreak by name. The order every
/// selection follows.
public List<String> profileNames() {
List<String> names = new ArrayList<>(profiles.keySet());
names.sort((a, b) -> {
int bySize = Long.compare(sortKey(a, resolve(a).baseCount()), sortKey(b, resolve(b).baseCount()));
return bySize != 0 ? bySize : naturalCompare(a, b);
});
return names;
}
/// Everything a selector can read of every profile, in
/// [#profileNames] order: structure as loaded, after inheritance;
/// attributes the profile's own.
public List<ProfileFacts> profileFacts() {
List<ProfileFacts> facts = new ArrayList<>();
for (String profile : profileNames()) {
Map<String, Object> definition = profiles.get(profile);
Resolved resolved = resolve(profile);
Map<String, Object> own = definition.get("attributes") instanceof Map<?, ?> declared ? YamlData.map(declared, "attributes") : Map.of();
facts.add(new ProfileFacts(profile, resolved.baseCount(), resolved.maxk(), isPartition(definition), statedParent(profile, definition), own));
}
return facts;
}
/// The profiles a selector names, size-ordered. `null` is `default`,
/// `profile=*` is every profile, and a selector that matches
/// nothing fails listing what was on offer.
public List<String> select(String selector) { return ProfileSelector.resolve(selector, profileFacts()); }
/// The one profile a selector names, for a surface that takes a
/// single profile; more than one match fails.
public String selectOne(String selector) { return ProfileSelector.resolveOne(selector, profileFacts()); }
/// The format version this manifest's content requires: tagged if
/// it declares a tag schema or a profile names a parent other than
/// `default`, sharded if any profile declares a multi-file facet,
@@ -358,6 +392,30 @@ public final class TestDataGroup {
}
throw new VectorDataException("Unrecognized window interval: " + item);
}
/// The size sort key of a profile: `default` first, then its
/// `base_count`, else its name read as a count, else last.
static long sortKey(String profile, Long baseCount) {
if ("default".equals(profile)) return 0;
if (baseCount != null) return baseCount;
try { return DSWindow.parseNumberWithSuffix(profile); } catch (VectorDataException notACount) { return Long.MAX_VALUE - 1; }
}
/// Natural comparison: digit runs compare by value, so `label_2`
/// sorts before `label_10`, and text compares character by character.
static int naturalCompare(String a, String b) {
int i = 0, j = 0;
while (i < a.length() && j < b.length()) {
char ac = a.charAt(i), bc = b.charAt(j);
if (Character.isDigit(ac) && Character.isDigit(bc)) {
long an = 0; while (i < a.length() && Character.isDigit(a.charAt(i))) { an = an * 10 + (a.charAt(i) - '0'); i++; }
long bn = 0; while (j < b.length() && Character.isDigit(b.charAt(j))) { bn = bn * 10 + (b.charAt(j) - '0'); j++; }
if (an != bn) return Long.compare(an, bn);
} else {
if (ac != bc) return Character.compare(ac, bc);
i++; j++;
}
}
return Integer.compare(a.length() - i, b.length() - j);
}
private static boolean isYaml(URI source) { String path = source.getPath() == null ? "" : source.getPath().toLowerCase(); return path.endsWith(".yaml") || path.endsWith(".yml"); }
private static URI child(URI source, String name) { String text = source.toString(); return URI.create(text.endsWith("/") ? text + name : text + "/" + name); }
private static String directoryName(URI source) { String path = source.getPath(); int end = path.lastIndexOf('/'); String parent = path.substring(0, end); int start = parent.lastIndexOf('/'); return parent.substring(start + 1); }
@@ -0,0 +1,139 @@
/*
* Copyright (c) 2026 The NoSQLBench Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.vectordata;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
/// Selectors on the catalog surfaces: a single-profile open resolves
/// exactly one, the set surface returns every match size-ordered, a
/// spec splits by its head's shape, and a catalog entry resolves the
/// same selection before anything is fetched.
@Tag("unit")
class CatalogSelectionTest {
@TempDir Path temporary;
private VectorDataSettings settings() {
return VectorDataSettings.builder().cacheDirectory(temporary.resolve("cache")).build();
}
private Catalog catalog() throws Exception {
Path dataset = Files.createDirectories(temporary.resolve("tess"));
FixtureSupport.fvec(dataset, "base.fvec", new float[][] {{0f, 1f}, {1f, 2f}, {2f, 3f}, {3f, 4f}});
FixtureSupport.fvec(dataset, "query.fvec", new float[][] {{9f, 8f}});
FixtureSupport.ivec(dataset, "gt_2.ivecs", new int[][] {{1, 2}});
Files.writeString(dataset.resolve("dataset.yaml"), """
format_version: 3
name: tess
profile_tags:
size: ~
family: stratified
profiles:
default:
maxk: 10
base_vectors: base.fvec
query_vectors: query.fvec
2r:
inherits: default
base_count: 2
neighbor_indices: gt_2.ivecs
attributes: { size: 2r, family: stratified }
2r-uniform:
inherits: 2r
attributes: { size: 2r, family: uniform, selectivity: 0.01 }
3r:
inherits: default
base_count: 3
attributes: { size: 3r }
""");
Files.writeString(temporary.resolve("catalog.yaml"), """
datasets:
- name: tess
path: tess/dataset.yaml
dataset_type: dataset.yaml
layout:
format_version: 3
profile_tags: { size: ~, family: stratified }
profiles:
default: { maxk: 10, base_vectors: base.fvec, query_vectors: query.fvec }
2r: { inherits: default, base_count: 2, neighbor_indices: gt_2.ivecs, attributes: { size: 2r, family: stratified } }
2r-uniform: { inherits: 2r, attributes: { size: 2r, family: uniform, selectivity: 0.01 } }
3r: { inherits: default, base_count: 3, attributes: { size: 3r } }
- name: future
path: nowhere/dataset.yaml
layout: { format_version: 99 }
""");
return Catalog.of(CatalogSources.of(temporary.resolve("catalog.yaml").toUri()), settings());
}
@Test void aSingleOpenResolvesExactlyOneProfile() throws Exception {
Catalog catalog = catalog();
assertEquals("default", catalog.open("tess", null).profile(), "no selector is default");
assertEquals("2r", catalog.open("tess", "2R").profile(), "a bare name as it always was, case folded");
assertEquals("2r-uniform", catalog.open("tess", "family=uniform").profile(), "an expression that names one");
SelectionException ambiguous = assertThrows(SelectionException.class, () -> catalog.open("tess", "size=2r"));
assertEquals(SelectionException.Kind.AMBIGUOUS, ambiguous.kind());
assertTrue(ambiguous.getMessage().startsWith("dataset 'tess': "), ambiguous.getMessage());
assertEquals(List.of("2r", "2r-uniform"), ambiguous.matches());
SelectionException none = assertThrows(SelectionException.class, () -> catalog.open("tess", "size=9r"));
assertEquals(SelectionException.Kind.NO_MATCH, none.kind());
assertTrue(none.getMessage().contains("3r (base_count=3, maxk=10, inherits=default, size=3r)"), "the offer is listed as loaded: " + none.getMessage());
}
@Test void theSetSurfaceReturnsEveryMatchSizeOrdered() throws Exception {
Catalog catalog = catalog();
List<TestDataView> all = catalog.openProfiles("tess", "profile=*");
assertEquals(List.of("default", "2r", "2r-uniform", "3r"), all.stream().map(TestDataView::profile).toList());
assertEquals(List.of("2r", "2r-uniform"), catalog.openProfiles("tess", "base_count=2").stream().map(TestDataView::profile).toList(),
"an inherited base_count is as selectable as a declared one");
assertEquals(List.of("default"), catalog.openProfiles("tess", null).stream().map(TestDataView::profile).toList());
assertEquals(2, catalog.openProfiles("tess", "2r-uniform").get(0).baseVectors().count(), "each view is the profile as loaded");
}
@Test void aSpecSplitsByItsHeadsShape() throws Exception {
Catalog catalog = catalog();
assertEquals("2r", catalog.openProfile("tess:2r").profile());
assertEquals("default", catalog.openProfile("tess").profile());
assertEquals("2r-uniform", catalog.openProfile("tess:profile=^2r-.*$").profile(), "a selector may carry what a last-colon split would have cut");
SelectionException bad = assertThrows(SelectionException.class, () -> catalog.openProfile("tess:size>abc"));
assertEquals(SelectionException.Kind.SYNTAX, bad.kind());
assertTrue(bad.position() >= "tess:".length(), "the position is into the spec: " + bad.position());
}
@Test void aCatalogEntrySelectsBeforeFetching() throws Exception {
Catalog catalog = catalog();
CatalogEntry entry = catalog.entries().get("tess");
assertEquals(3, entry.formatVersion());
assertEquals(List.of("size", "family"), List.copyOf(entry.profileTags().keySet()));
assertEquals(List.of("default", "2r", "2r-uniform", "3r"), entry.profileNames());
assertEquals(List.of("2r", "2r-uniform"), entry.select("base_count=2"), "the declared form reads structure through the chain");
assertEquals("2r-uniform", entry.selectOne("selectivity=1e-2"));
assertEquals(List.of("default"), entry.select(null));
ProfileFacts uniform = entry.profileFacts().stream().filter(f -> f.name().equals("2r-uniform")).findFirst().orElseThrow();
assertEquals(2L, uniform.baseCount());
assertEquals(10, uniform.maxk(), "maxk crosses every step");
assertEquals(entry.select("family=uniform"), catalog.openGroup("tess").select("family=uniform"), "the entry and the loaded dataset agree");
assertEquals(99, catalog.entries().get("future").formatVersion(), "a consumer can refuse a listed dataset before fetching it");
assertEquals(1, new CatalogEntry("bare", temporary.toUri(), null, java.util.Map.of()).formatVersion(), "absent means 1");
}
}
@@ -158,6 +158,9 @@ class ProfileInheritanceTest {
TestDataView uniform = view(dir, "2r-uniform");
assertArrayEquals(new int[] {3, 4}, uniform.neighborIndices().get(0));
assertTrue(view(dir, "2r").facet("metadata_results").isEmpty(), "a layer holds no predicate group of its own");
List<ProfileFacts> facts = group(dir).profileFacts();
ProfileFacts noCount = facts.stream().filter(f -> f.name().equals("2r-uniform")).findFirst().orElseThrow();
assertEquals(2L, noCount.baseCount(), "a set without a count is at its layer's");
}
/// A results index is derived from `base_count` like the ground
@@ -248,6 +251,7 @@ class ProfileInheritanceTest {
inherits: 100
""");
assertArrayEquals(new int[] {3, 4}, view(dir, "child").neighborIndices().get(0), "a parent YAML read as a number is the name it spells");
assertEquals(List.of("default", "100", "child"), group(dir).profileNames());
}
// -- format_version 3: stated parents --
@@ -363,4 +367,62 @@ class ProfileInheritanceTest {
assertTrue(refused(dataset("format_version: 2\nprofile_tags:\n size: ~\n", " default:\n base_vectors: base.fvec\n")).getMessage().contains("understate"));
assertTrue(group(dataset(" default:\n base_vectors: base.fvec\n")).profileTags().isEmpty(), "a dataset without a schema declares none");
}
// -- what a selector reads --
/// Attributes never inherit: a child of a named parent selects on
/// its own attributes and on the structure it inherits.
@Test void attributesNeverInheritButStructureDoes() throws Exception {
Path dir = layered("""
default:
base_vectors: base.fvec
maxk: 100
1m:
inherits: default
base_count: 2
attributes:
family: sized
1m-sel:
inherits: 1m
query_vectors: query.fvec
attributes:
selectivity: 0.01
""");
TestDataGroup group = group(dir);
ProfileFacts child = group.profileFacts().stream().filter(f -> f.name().equals("1m-sel")).findFirst().orElseThrow();
assertEquals(2L, child.baseCount(), "structure is read after inheritance");
assertEquals(100, child.maxk());
assertEquals("1m", child.inherits());
assertEquals(Map.of("selectivity", 0.01), child.attributes(), "a parent's attributes are not the child's");
assertTrue(ProfileSelector.parse("family=sized").matches(group.profileFacts().stream().filter(f -> f.name().equals("1m")).findFirst().orElseThrow()));
assertFalse(ProfileSelector.parse("family=sized").matches(child));
assertTrue(ProfileSelector.parse("base_count=2,selectivity=1e-2").matches(child));
assertEquals(List.of("1m-sel"), group.select("selectivity=1e-2"));
assertEquals("1m", group.selectOne("1M"));
assertEquals(List.of("default", "1m", "1m-sel"), group.select("profile=*"));
assertEquals(List.of("default"), group.select(null));
}
@Test void profileNamesAreSizeOrdered() throws Exception {
Path dir = layered("""
default:
base_vectors: base.fvec
label_10:
partition: true
base_vectors: part.fvec
label_2:
partition: true
base_vectors: part.fvec
3r:
inherits: default
base_count: 3
2r:
inherits: default
base_count: 2
10m:
inherits: default
""");
assertEquals(List.of("default", "2r", "3r", "10m", "label_2", "label_10"), group(dir).profileNames(),
"default first, then by base_count or the name read as a count, then naturally by name");
}
}