mirror of
https://github.com/nosqlbench/nosqlbench.git
synced 2026-10-13 18:02:01 -05:00
feat(vectordata): select profiles by selector on the group and catalog surfaces
A group answers what a selector reads of every profile, structure as loaded after inheritance and attributes its own, in size order: default first, then by base_count or the name read as a count, then naturally by name. A catalog open takes a selector and refuses a set, openProfiles returns the set, openProfile splits a spec by its head's shape, and a catalog entry resolves the same selection from its listed layout before anything is fetched, exposing the listed format version.
This commit is contained in:
1 parent
9f3efebe85
commit
b6ddfcc575
5 files changed
+418
-9
No files matched your search
@@ -39,18 +39,46 @@ public final class Catalog {
|
||||
return new Catalog(entries, settings);
|
||||
}
|
||||
public Map<String, CatalogEntry> entries() { return entries; }
|
||||
public TestDataView open(String dataset, String profile) {
|
||||
/// Loads a dataset's manifest as a group, without choosing a profile.
|
||||
public TestDataGroup openGroup(String dataset) {
|
||||
CatalogEntry entry = entries.get(dataset);
|
||||
if (entry == null) throw new VectorDataException("Dataset is not in configured catalogs: " + dataset);
|
||||
if ("knn_entries.yaml".equals(entry.datasetType())) {
|
||||
@SuppressWarnings("unchecked") Map<String, Map<String, Object>> profiles = (Map<String, Map<String, Object>>) entry.attributes().get("profiles");
|
||||
return TestDataGroup.fromLegacyEntries(dataset, entry.manifest(), profiles, settings).profile(profile);
|
||||
return TestDataGroup.fromLegacyEntries(dataset, entry.manifest(), profiles, settings);
|
||||
}
|
||||
return TestDataGroup.load(entry.manifest(), settings).profile(profile);
|
||||
return TestDataGroup.load(entry.manifest(), settings);
|
||||
}
|
||||
public TestDataView openProfile(String datasetAndProfile) {
|
||||
int separator = datasetAndProfile.lastIndexOf(':');
|
||||
return separator < 0 ? open(datasetAndProfile, null) : open(datasetAndProfile.substring(0, separator), datasetAndProfile.substring(separator + 1));
|
||||
/// Opens the one profile of a dataset a selector names. `selector`
|
||||
/// is a bare name as it always was, or an expression such as
|
||||
/// `size=10m,predicates=uniform-2`; `null` opens `default`. It must
|
||||
/// name exactly one profile: a set fails here, and [#openProfiles]
|
||||
/// is the surface that takes one.
|
||||
public TestDataView open(String dataset, String selector) {
|
||||
TestDataGroup group = openGroup(dataset);
|
||||
return group.profile(selectOne(group, dataset, selector));
|
||||
}
|
||||
/// Opens every profile of a dataset a selector names, size-ordered.
|
||||
/// `null` opens `default`; `profile=*` opens them all.
|
||||
public List<TestDataView> openProfiles(String dataset, String selector) {
|
||||
TestDataGroup group = openGroup(dataset);
|
||||
List<String> selected;
|
||||
try { selected = group.select(selector); }
|
||||
catch (SelectionException e) { throw e.inDataset(dataset); }
|
||||
List<TestDataView> views = new ArrayList<>();
|
||||
for (String profile : selected) views.add(group.profile(profile));
|
||||
return views;
|
||||
}
|
||||
/// Opens the one profile a dataset spec names — `<dataset>` or
|
||||
/// `<dataset>:<selector>`, split by the head's shape as
|
||||
/// [DatasetSpec] describes, so a selector may carry a colon.
|
||||
public TestDataView openProfile(String spec) {
|
||||
DatasetSpec parsed = DatasetSpec.parse(spec);
|
||||
return open(parsed.head(), parsed.selectorText());
|
||||
}
|
||||
private static String selectOne(TestDataGroup group, String dataset, String selector) {
|
||||
try { return group.selectOne(selector); }
|
||||
catch (SelectionException e) { throw e.inDataset(dataset); }
|
||||
}
|
||||
private static void load(URI source, VectorDataSettings settings, Map<String, CatalogEntry> output) {
|
||||
if (!isCatalogFile(source)) { loadDirectory(source, settings, output); return; }
|
||||
|
||||
@@ -15,8 +15,130 @@
|
||||
*/
|
||||
package io.nosqlbench.vectordata;
|
||||
|
||||
import io.nosqlbench.vectordata.internal.YamlData;
|
||||
|
||||
import java.net.URI;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/** A dataset entry resolved from a catalog source. */
|
||||
public record CatalogEntry(String name, URI manifest, String datasetType, Map<String, Object> attributes) { }
|
||||
/// A dataset entry resolved from a catalog source. `attributes` is the
|
||||
/// entry as listed; a canonical entry's `layout` carries the dataset's
|
||||
/// `format_version`, `profile_tags`, and declared `profiles`, which is
|
||||
/// what a consumer reads **before fetching** — to refuse a dataset
|
||||
/// above its version, or to resolve a selector against the profiles on
|
||||
/// offer, without downloading the manifest.
|
||||
public record CatalogEntry(String name, URI manifest, String datasetType, Map<String, Object> attributes) {
|
||||
|
||||
/// The entry's declared layout, or an empty map when it lists none.
|
||||
public Map<String, Object> layout() {
|
||||
return attributes.get("layout") instanceof Map<?, ?> layout ? YamlData.map(layout, "layout") : Map.of();
|
||||
}
|
||||
|
||||
/// The `format_version` the listing states for the dataset, or
|
||||
/// [FormatVersion#BASE] when it states none — absent means 1, like
|
||||
/// everywhere else.
|
||||
public int formatVersion() {
|
||||
Object stated = layout().get("format_version");
|
||||
return stated == null ? FormatVersion.BASE : YamlData.integer(stated, "format_version");
|
||||
}
|
||||
|
||||
/// The listed profile tag schema, naming tags in order; empty when
|
||||
/// none is listed.
|
||||
public Map<String, Object> profileTags() {
|
||||
return layout().get("profile_tags") instanceof Map<?, ?> tags ? Collections.unmodifiableMap(YamlData.map(tags, "profile_tags")) : Map.of();
|
||||
}
|
||||
|
||||
/// The profiles the listing declares, by name, as declared.
|
||||
public Map<String, Map<String, Object>> profiles() {
|
||||
Object listed = layout().get("profiles");
|
||||
if (listed == null) listed = attributes.get("profiles");
|
||||
Map<String, Map<String, Object>> profiles = new LinkedHashMap<>();
|
||||
if (listed instanceof Map<?, ?> map)
|
||||
map.forEach((key, value) -> profiles.put(String.valueOf(key), value instanceof Map<?, ?> ? YamlData.map(value, "profile " + key) : Map.of()));
|
||||
return profiles;
|
||||
}
|
||||
|
||||
/// The listed profile names, size-ordered as [TestDataGroup#profileNames] orders them.
|
||||
public List<String> profileNames() {
|
||||
Map<String, Map<String, Object>> profiles = profiles();
|
||||
List<String> names = new ArrayList<>(profiles.keySet());
|
||||
names.sort((a, b) -> {
|
||||
int bySize = Long.compare(TestDataGroup.sortKey(a, declaredStructure(profiles, a).baseCount()),
|
||||
TestDataGroup.sortKey(b, declaredStructure(profiles, b).baseCount()));
|
||||
return bySize != 0 ? bySize : TestDataGroup.naturalCompare(a, b);
|
||||
});
|
||||
return names;
|
||||
}
|
||||
|
||||
/// Everything a selector can read of every listed profile, with
|
||||
/// `base_count` and `maxk` read through the `inherits` chain the
|
||||
/// way the loader resolves them, so a catalog entry selects the
|
||||
/// same profiles a loaded dataset does. Attributes are the
|
||||
/// profile's own, in declaration order.
|
||||
public List<ProfileFacts> profileFacts() {
|
||||
Map<String, Map<String, Object>> profiles = profiles();
|
||||
List<ProfileFacts> facts = new ArrayList<>();
|
||||
for (String name : profileNames()) {
|
||||
Map<String, Object> profile = profiles.get(name);
|
||||
Structure structure = declaredStructure(profiles, name);
|
||||
Map<String, Object> own = profile.get("attributes") instanceof Map<?, ?> declared ? YamlData.map(declared, "attributes") : Map.of();
|
||||
facts.add(new ProfileFacts(name, structure.baseCount(), structure.maxk(), partition(profile), inherits(profile), own));
|
||||
}
|
||||
return facts;
|
||||
}
|
||||
|
||||
/// The profiles a selector names in this entry, size-ordered;
|
||||
/// `null` is `default`. A catalog entry is what a consumer reads
|
||||
/// before fetching, so a selector resolves here the same way it
|
||||
/// does against the loaded dataset.
|
||||
public List<String> select(String selector) { return ProfileSelector.resolve(selector, profileFacts()); }
|
||||
|
||||
/// The one profile a selector names in this entry.
|
||||
public String selectOne(String selector) { return ProfileSelector.resolveOne(selector, profileFacts()); }
|
||||
|
||||
/// `base_count` and `maxk` of a declared profile as the loader would
|
||||
/// see them: its own, else its parent's. `maxk` crosses any parent;
|
||||
/// `base_count` crosses only a named parent other than `default`,
|
||||
/// because inheriting from `default` is a size step and the child's
|
||||
/// count is its own. A partition profile is self-contained.
|
||||
private record Structure(Long baseCount, Integer maxk) { }
|
||||
private static Structure declaredStructure(Map<String, Map<String, Object>> profiles, String name) {
|
||||
Map<String, Object> own = profiles.get(name);
|
||||
if (own == null) return new Structure(null, null);
|
||||
Long baseCount = longOrNull(own.get("base_count"));
|
||||
Integer maxk = own.get("maxk") == null ? null : YamlData.integer(own.get("maxk"), "maxk");
|
||||
String current = name;
|
||||
// A chain is at most one hop per profile; a cycle stops here.
|
||||
for (int hops = 0; hops < profiles.size(); hops++) {
|
||||
Map<String, Object> profile = profiles.get(current);
|
||||
if (profile == null || partition(profile) || "default".equals(current)) break;
|
||||
String stated = inherits(profile);
|
||||
String parent = stated != null && !stated.equals(current) && profiles.containsKey(stated) ? stated : "default";
|
||||
Map<String, Object> above = profiles.get(parent);
|
||||
if (above == null) break;
|
||||
if (maxk == null && above.get("maxk") != null) maxk = YamlData.integer(above.get("maxk"), "maxk");
|
||||
if (!"default".equals(parent) && baseCount == null) baseCount = longOrNull(above.get("base_count"));
|
||||
current = parent;
|
||||
}
|
||||
return new Structure(baseCount, maxk);
|
||||
}
|
||||
|
||||
private static boolean partition(Map<String, Object> profile) {
|
||||
Object partition = profile.get("partition");
|
||||
return Boolean.TRUE.equals(partition) || "true".equalsIgnoreCase(String.valueOf(partition));
|
||||
}
|
||||
|
||||
private static String inherits(Map<String, Object> profile) {
|
||||
Object raw = profile.containsKey("inherits") ? profile.get("inherits") : profile.get("extends");
|
||||
return raw == null ? null : String.valueOf(raw);
|
||||
}
|
||||
|
||||
private static Long longOrNull(Object value) {
|
||||
if (value == null) return null;
|
||||
if (value instanceof Number number) return number.longValue();
|
||||
try { return Long.parseLong(String.valueOf(value).trim()); } catch (NumberFormatException e) { return null; }
|
||||
}
|
||||
}
|
||||
@@ -177,11 +177,45 @@ public final class TestDataGroup {
|
||||
/// declares none.
|
||||
public Map<String, Object> profileTags() { return profileTags; }
|
||||
/// Opens a profile by its exact name. A `null` or blank name opens
|
||||
/// `default`, or the first profile when there is none.
|
||||
/// `default`, or the first profile when there is none. For a
|
||||
/// selector expression see [#selectOne] and [#select].
|
||||
public TestDataView profile(String profile) {
|
||||
String selected = profile == null || profile.isBlank() ? (profiles.containsKey("default") ? "default" : profiles.keySet().iterator().next()) : profile;
|
||||
return new ManifestView(name, selected, resolve(selected).facets(), settings, attributes);
|
||||
}
|
||||
/// Profile names size-ordered: `default` first, then ascending by
|
||||
/// `base_count` — or by the name read as a count, `10m` before
|
||||
/// `100m` — with a natural tiebreak by name. The order every
|
||||
/// selection follows.
|
||||
public List<String> profileNames() {
|
||||
List<String> names = new ArrayList<>(profiles.keySet());
|
||||
names.sort((a, b) -> {
|
||||
int bySize = Long.compare(sortKey(a, resolve(a).baseCount()), sortKey(b, resolve(b).baseCount()));
|
||||
return bySize != 0 ? bySize : naturalCompare(a, b);
|
||||
});
|
||||
return names;
|
||||
}
|
||||
/// Everything a selector can read of every profile, in
|
||||
/// [#profileNames] order: structure as loaded, after inheritance;
|
||||
/// attributes the profile's own.
|
||||
public List<ProfileFacts> profileFacts() {
|
||||
List<ProfileFacts> facts = new ArrayList<>();
|
||||
for (String profile : profileNames()) {
|
||||
Map<String, Object> definition = profiles.get(profile);
|
||||
Resolved resolved = resolve(profile);
|
||||
Map<String, Object> own = definition.get("attributes") instanceof Map<?, ?> declared ? YamlData.map(declared, "attributes") : Map.of();
|
||||
facts.add(new ProfileFacts(profile, resolved.baseCount(), resolved.maxk(), isPartition(definition), statedParent(profile, definition), own));
|
||||
}
|
||||
return facts;
|
||||
}
|
||||
/// The profiles a selector names, size-ordered. `null` is `default`,
|
||||
/// `profile=*` is every profile, and a selector that matches
|
||||
/// nothing fails listing what was on offer.
|
||||
public List<String> select(String selector) { return ProfileSelector.resolve(selector, profileFacts()); }
|
||||
/// The one profile a selector names, for a surface that takes a
|
||||
/// single profile; more than one match fails.
|
||||
public String selectOne(String selector) { return ProfileSelector.resolveOne(selector, profileFacts()); }
|
||||
|
||||
/// The format version this manifest's content requires: tagged if
|
||||
/// it declares a tag schema or a profile names a parent other than
|
||||
/// `default`, sharded if any profile declares a multi-file facet,
|
||||
@@ -358,6 +392,30 @@ public final class TestDataGroup {
|
||||
}
|
||||
throw new VectorDataException("Unrecognized window interval: " + item);
|
||||
}
|
||||
/// The size sort key of a profile: `default` first, then its
|
||||
/// `base_count`, else its name read as a count, else last.
|
||||
static long sortKey(String profile, Long baseCount) {
|
||||
if ("default".equals(profile)) return 0;
|
||||
if (baseCount != null) return baseCount;
|
||||
try { return DSWindow.parseNumberWithSuffix(profile); } catch (VectorDataException notACount) { return Long.MAX_VALUE - 1; }
|
||||
}
|
||||
/// Natural comparison: digit runs compare by value, so `label_2`
|
||||
/// sorts before `label_10`, and text compares character by character.
|
||||
static int naturalCompare(String a, String b) {
|
||||
int i = 0, j = 0;
|
||||
while (i < a.length() && j < b.length()) {
|
||||
char ac = a.charAt(i), bc = b.charAt(j);
|
||||
if (Character.isDigit(ac) && Character.isDigit(bc)) {
|
||||
long an = 0; while (i < a.length() && Character.isDigit(a.charAt(i))) { an = an * 10 + (a.charAt(i) - '0'); i++; }
|
||||
long bn = 0; while (j < b.length() && Character.isDigit(b.charAt(j))) { bn = bn * 10 + (b.charAt(j) - '0'); j++; }
|
||||
if (an != bn) return Long.compare(an, bn);
|
||||
} else {
|
||||
if (ac != bc) return Character.compare(ac, bc);
|
||||
i++; j++;
|
||||
}
|
||||
}
|
||||
return Integer.compare(a.length() - i, b.length() - j);
|
||||
}
|
||||
private static boolean isYaml(URI source) { String path = source.getPath() == null ? "" : source.getPath().toLowerCase(); return path.endsWith(".yaml") || path.endsWith(".yml"); }
|
||||
private static URI child(URI source, String name) { String text = source.toString(); return URI.create(text.endsWith("/") ? text + name : text + "/" + name); }
|
||||
private static String directoryName(URI source) { String path = source.getPath(); int end = path.lastIndexOf('/'); String parent = path.substring(0, end); int start = parent.lastIndexOf('/'); return parent.substring(start + 1); }
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
/*
|
||||
* Copyright (c) 2026 The NoSQLBench Authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.nosqlbench.vectordata;
|
||||
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/// Selectors on the catalog surfaces: a single-profile open resolves
|
||||
/// exactly one, the set surface returns every match size-ordered, a
|
||||
/// spec splits by its head's shape, and a catalog entry resolves the
|
||||
/// same selection before anything is fetched.
|
||||
@Tag("unit")
|
||||
class CatalogSelectionTest {
|
||||
@TempDir Path temporary;
|
||||
|
||||
private VectorDataSettings settings() {
|
||||
return VectorDataSettings.builder().cacheDirectory(temporary.resolve("cache")).build();
|
||||
}
|
||||
|
||||
private Catalog catalog() throws Exception {
|
||||
Path dataset = Files.createDirectories(temporary.resolve("tess"));
|
||||
FixtureSupport.fvec(dataset, "base.fvec", new float[][] {{0f, 1f}, {1f, 2f}, {2f, 3f}, {3f, 4f}});
|
||||
FixtureSupport.fvec(dataset, "query.fvec", new float[][] {{9f, 8f}});
|
||||
FixtureSupport.ivec(dataset, "gt_2.ivecs", new int[][] {{1, 2}});
|
||||
Files.writeString(dataset.resolve("dataset.yaml"), """
|
||||
format_version: 3
|
||||
name: tess
|
||||
profile_tags:
|
||||
size: ~
|
||||
family: stratified
|
||||
profiles:
|
||||
default:
|
||||
maxk: 10
|
||||
base_vectors: base.fvec
|
||||
query_vectors: query.fvec
|
||||
2r:
|
||||
inherits: default
|
||||
base_count: 2
|
||||
neighbor_indices: gt_2.ivecs
|
||||
attributes: { size: 2r, family: stratified }
|
||||
2r-uniform:
|
||||
inherits: 2r
|
||||
attributes: { size: 2r, family: uniform, selectivity: 0.01 }
|
||||
3r:
|
||||
inherits: default
|
||||
base_count: 3
|
||||
attributes: { size: 3r }
|
||||
""");
|
||||
Files.writeString(temporary.resolve("catalog.yaml"), """
|
||||
datasets:
|
||||
- name: tess
|
||||
path: tess/dataset.yaml
|
||||
dataset_type: dataset.yaml
|
||||
layout:
|
||||
format_version: 3
|
||||
profile_tags: { size: ~, family: stratified }
|
||||
profiles:
|
||||
default: { maxk: 10, base_vectors: base.fvec, query_vectors: query.fvec }
|
||||
2r: { inherits: default, base_count: 2, neighbor_indices: gt_2.ivecs, attributes: { size: 2r, family: stratified } }
|
||||
2r-uniform: { inherits: 2r, attributes: { size: 2r, family: uniform, selectivity: 0.01 } }
|
||||
3r: { inherits: default, base_count: 3, attributes: { size: 3r } }
|
||||
- name: future
|
||||
path: nowhere/dataset.yaml
|
||||
layout: { format_version: 99 }
|
||||
""");
|
||||
return Catalog.of(CatalogSources.of(temporary.resolve("catalog.yaml").toUri()), settings());
|
||||
}
|
||||
|
||||
@Test void aSingleOpenResolvesExactlyOneProfile() throws Exception {
|
||||
Catalog catalog = catalog();
|
||||
assertEquals("default", catalog.open("tess", null).profile(), "no selector is default");
|
||||
assertEquals("2r", catalog.open("tess", "2R").profile(), "a bare name as it always was, case folded");
|
||||
assertEquals("2r-uniform", catalog.open("tess", "family=uniform").profile(), "an expression that names one");
|
||||
SelectionException ambiguous = assertThrows(SelectionException.class, () -> catalog.open("tess", "size=2r"));
|
||||
assertEquals(SelectionException.Kind.AMBIGUOUS, ambiguous.kind());
|
||||
assertTrue(ambiguous.getMessage().startsWith("dataset 'tess': "), ambiguous.getMessage());
|
||||
assertEquals(List.of("2r", "2r-uniform"), ambiguous.matches());
|
||||
SelectionException none = assertThrows(SelectionException.class, () -> catalog.open("tess", "size=9r"));
|
||||
assertEquals(SelectionException.Kind.NO_MATCH, none.kind());
|
||||
assertTrue(none.getMessage().contains("3r (base_count=3, maxk=10, inherits=default, size=3r)"), "the offer is listed as loaded: " + none.getMessage());
|
||||
}
|
||||
|
||||
@Test void theSetSurfaceReturnsEveryMatchSizeOrdered() throws Exception {
|
||||
Catalog catalog = catalog();
|
||||
List<TestDataView> all = catalog.openProfiles("tess", "profile=*");
|
||||
assertEquals(List.of("default", "2r", "2r-uniform", "3r"), all.stream().map(TestDataView::profile).toList());
|
||||
assertEquals(List.of("2r", "2r-uniform"), catalog.openProfiles("tess", "base_count=2").stream().map(TestDataView::profile).toList(),
|
||||
"an inherited base_count is as selectable as a declared one");
|
||||
assertEquals(List.of("default"), catalog.openProfiles("tess", null).stream().map(TestDataView::profile).toList());
|
||||
assertEquals(2, catalog.openProfiles("tess", "2r-uniform").get(0).baseVectors().count(), "each view is the profile as loaded");
|
||||
}
|
||||
|
||||
@Test void aSpecSplitsByItsHeadsShape() throws Exception {
|
||||
Catalog catalog = catalog();
|
||||
assertEquals("2r", catalog.openProfile("tess:2r").profile());
|
||||
assertEquals("default", catalog.openProfile("tess").profile());
|
||||
assertEquals("2r-uniform", catalog.openProfile("tess:profile=^2r-.*$").profile(), "a selector may carry what a last-colon split would have cut");
|
||||
SelectionException bad = assertThrows(SelectionException.class, () -> catalog.openProfile("tess:size>abc"));
|
||||
assertEquals(SelectionException.Kind.SYNTAX, bad.kind());
|
||||
assertTrue(bad.position() >= "tess:".length(), "the position is into the spec: " + bad.position());
|
||||
}
|
||||
|
||||
@Test void aCatalogEntrySelectsBeforeFetching() throws Exception {
|
||||
Catalog catalog = catalog();
|
||||
CatalogEntry entry = catalog.entries().get("tess");
|
||||
assertEquals(3, entry.formatVersion());
|
||||
assertEquals(List.of("size", "family"), List.copyOf(entry.profileTags().keySet()));
|
||||
assertEquals(List.of("default", "2r", "2r-uniform", "3r"), entry.profileNames());
|
||||
assertEquals(List.of("2r", "2r-uniform"), entry.select("base_count=2"), "the declared form reads structure through the chain");
|
||||
assertEquals("2r-uniform", entry.selectOne("selectivity=1e-2"));
|
||||
assertEquals(List.of("default"), entry.select(null));
|
||||
ProfileFacts uniform = entry.profileFacts().stream().filter(f -> f.name().equals("2r-uniform")).findFirst().orElseThrow();
|
||||
assertEquals(2L, uniform.baseCount());
|
||||
assertEquals(10, uniform.maxk(), "maxk crosses every step");
|
||||
assertEquals(entry.select("family=uniform"), catalog.openGroup("tess").select("family=uniform"), "the entry and the loaded dataset agree");
|
||||
assertEquals(99, catalog.entries().get("future").formatVersion(), "a consumer can refuse a listed dataset before fetching it");
|
||||
assertEquals(1, new CatalogEntry("bare", temporary.toUri(), null, java.util.Map.of()).formatVersion(), "absent means 1");
|
||||
}
|
||||
}
|
||||
@@ -158,6 +158,9 @@ class ProfileInheritanceTest {
|
||||
TestDataView uniform = view(dir, "2r-uniform");
|
||||
assertArrayEquals(new int[] {3, 4}, uniform.neighborIndices().get(0));
|
||||
assertTrue(view(dir, "2r").facet("metadata_results").isEmpty(), "a layer holds no predicate group of its own");
|
||||
List<ProfileFacts> facts = group(dir).profileFacts();
|
||||
ProfileFacts noCount = facts.stream().filter(f -> f.name().equals("2r-uniform")).findFirst().orElseThrow();
|
||||
assertEquals(2L, noCount.baseCount(), "a set without a count is at its layer's");
|
||||
}
|
||||
|
||||
/// A results index is derived from `base_count` like the ground
|
||||
@@ -248,6 +251,7 @@ class ProfileInheritanceTest {
|
||||
inherits: 100
|
||||
""");
|
||||
assertArrayEquals(new int[] {3, 4}, view(dir, "child").neighborIndices().get(0), "a parent YAML read as a number is the name it spells");
|
||||
assertEquals(List.of("default", "100", "child"), group(dir).profileNames());
|
||||
}
|
||||
|
||||
// -- format_version 3: stated parents --
|
||||
@@ -363,4 +367,62 @@ class ProfileInheritanceTest {
|
||||
assertTrue(refused(dataset("format_version: 2\nprofile_tags:\n size: ~\n", " default:\n base_vectors: base.fvec\n")).getMessage().contains("understate"));
|
||||
assertTrue(group(dataset(" default:\n base_vectors: base.fvec\n")).profileTags().isEmpty(), "a dataset without a schema declares none");
|
||||
}
|
||||
|
||||
// -- what a selector reads --
|
||||
|
||||
/// Attributes never inherit: a child of a named parent selects on
|
||||
/// its own attributes and on the structure it inherits.
|
||||
@Test void attributesNeverInheritButStructureDoes() throws Exception {
|
||||
Path dir = layered("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
maxk: 100
|
||||
1m:
|
||||
inherits: default
|
||||
base_count: 2
|
||||
attributes:
|
||||
family: sized
|
||||
1m-sel:
|
||||
inherits: 1m
|
||||
query_vectors: query.fvec
|
||||
attributes:
|
||||
selectivity: 0.01
|
||||
""");
|
||||
TestDataGroup group = group(dir);
|
||||
ProfileFacts child = group.profileFacts().stream().filter(f -> f.name().equals("1m-sel")).findFirst().orElseThrow();
|
||||
assertEquals(2L, child.baseCount(), "structure is read after inheritance");
|
||||
assertEquals(100, child.maxk());
|
||||
assertEquals("1m", child.inherits());
|
||||
assertEquals(Map.of("selectivity", 0.01), child.attributes(), "a parent's attributes are not the child's");
|
||||
assertTrue(ProfileSelector.parse("family=sized").matches(group.profileFacts().stream().filter(f -> f.name().equals("1m")).findFirst().orElseThrow()));
|
||||
assertFalse(ProfileSelector.parse("family=sized").matches(child));
|
||||
assertTrue(ProfileSelector.parse("base_count=2,selectivity=1e-2").matches(child));
|
||||
assertEquals(List.of("1m-sel"), group.select("selectivity=1e-2"));
|
||||
assertEquals("1m", group.selectOne("1M"));
|
||||
assertEquals(List.of("default", "1m", "1m-sel"), group.select("profile=*"));
|
||||
assertEquals(List.of("default"), group.select(null));
|
||||
}
|
||||
|
||||
@Test void profileNamesAreSizeOrdered() throws Exception {
|
||||
Path dir = layered("""
|
||||
default:
|
||||
base_vectors: base.fvec
|
||||
label_10:
|
||||
partition: true
|
||||
base_vectors: part.fvec
|
||||
label_2:
|
||||
partition: true
|
||||
base_vectors: part.fvec
|
||||
3r:
|
||||
inherits: default
|
||||
base_count: 3
|
||||
2r:
|
||||
inherits: default
|
||||
base_count: 2
|
||||
10m:
|
||||
inherits: default
|
||||
""");
|
||||
assertEquals(List.of("default", "2r", "3r", "10m", "label_2", "label_10"), group(dir).profileNames(),
|
||||
"default first, then by base_count or the name read as a count, then naturally by name");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user